feat: recreateService transient-container helper + use it in deploy() + rollback()
Replaces the runOnServer("docker compose up -d --force-recreate") pattern with a one-shot transient docker:cli container that runs OUTSIDE kua-deploy lifecycle. Solves the self-recreate chicken-and-egg: when the target is kua-deploy itself, the recreate completes because the transient survives kua-deploy stopping (docker daemon does the actual work).
Secrets are fetched via kua-vault export, written to a 600-perm tempfile on /app/data, passed via --env-file (docker CLI reads it from kua-deploys perspective; never on the docker run command line). Tempfile is unlinked in finally{}.
Replaces: deploy() stateless recreate (force=true), deploy() stateful up (force=false), rollback() recreate (force=true with all-services svcList).
Build step keeps runOnServer (local exec on bruno) since build doesnt kill kua-deploy. envPrefix/kvPrefix vars retained for the build command.
This commit is contained in:
kua-deploy-split
parent
26804c692e
commit
9169c84381
147
server.js
147
server.js
|
|
@ -3,7 +3,7 @@ import fs from 'fs/promises';
|
|||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import http from 'http';
|
||||
import { exec as execCb, execFile as execFileCb } from 'child_process';
|
||||
import { exec as execCb, execFile as execFileCb, spawn } from 'child_process';
|
||||
import { promisify } from 'util';
|
||||
|
||||
const exec = promisify(execCb);
|
||||
|
|
@ -270,6 +270,99 @@ function composeEnvPrefix(server) {
|
|||
return tailscaleIp ? `TAILSCALE_IP=${tailscaleIp} ` : '';
|
||||
}
|
||||
|
||||
// recreateService — spawn a one-shot transient docker:cli container that runs
|
||||
// `docker compose up -d` against the host docker socket. The transient container
|
||||
// is OUTSIDE the lifecycle of the service it recreates, so even when the target
|
||||
// is kua-deploy itself, the recreate completes (the transient survives kua-deploy
|
||||
// being stopped/started; the docker daemon does the actual work).
|
||||
//
|
||||
// Bind paths MUST be identical between host and transient container
|
||||
// (e.g. -v /root/apps/X:/root/apps/X) so compose's path resolution matches host
|
||||
// reality. Secrets are pre-fetched via kua-vault into a private --env-file rather
|
||||
// than being passed on the docker run command line.
|
||||
//
|
||||
// Returns the same { ok, stdout, stderr, error? } shape as run()/runOnServer()
|
||||
// so call sites can swap with minimal change.
|
||||
async function recreateService({
|
||||
project, // compose project name (basename(deployDir) typically)
|
||||
deployDir, // absolute path to the deploy dir on the docker host
|
||||
services, // array of service names to recreate
|
||||
force = true, // pass --force-recreate
|
||||
vault = null, // { project, env } — if set, fetch secrets via kua-vault export
|
||||
server = 'bruno', // for TAILSCALE_IP env var
|
||||
composeFile = 'docker-compose.yml',
|
||||
timeout = 300000,
|
||||
} = {}) {
|
||||
if (!Array.isArray(services) || services.length === 0) {
|
||||
return { ok: true, stdout: '', stderr: '', error: null, skipped: true };
|
||||
}
|
||||
// Stage env file on the kua-deploy data volume so the docker CLI (running
|
||||
// inside kua-deploy) can read it. The transient container picks up vars via
|
||||
// --env-file processed at submit time — no host-side mount needed.
|
||||
const tmpName = `.env-recreate-${crypto.randomBytes(8).toString('hex')}`;
|
||||
const envFilePath = `/app/data/${tmpName}`;
|
||||
let envFileWritten = false;
|
||||
try {
|
||||
if (vault && vault.project) {
|
||||
const envEnv = vault.env || 'prod';
|
||||
// kua-vault export emits KEY=VALUE lines — directly compatible with --env-file
|
||||
const exportRes = await run(`kua-vault export --project ${vault.project} --env ${envEnv}`, { timeout: 30000 });
|
||||
if (!exportRes.ok) {
|
||||
return { ok: false, stdout: '', stderr: `kua-vault export ${vault.project}/${envEnv} failed: ${exportRes.stderr?.slice(-300) || exportRes.error}`, error: 'vault export failed' };
|
||||
}
|
||||
// Strip any non KEY=VALUE lines (e.g. status banners) and validate
|
||||
const envLines = exportRes.stdout
|
||||
.split('\n')
|
||||
.filter(l => /^[A-Z_][A-Z0-9_]*=/.test(l));
|
||||
if (envLines.length === 0) {
|
||||
return { ok: false, stdout: '', stderr: `kua-vault export returned no KEY=VALUE lines for ${vault.project}/${envEnv}`, error: 'empty vault export' };
|
||||
}
|
||||
await fs.writeFile(envFilePath, envLines.join('\n') + '\n', { mode: 0o600 });
|
||||
envFileWritten = true;
|
||||
}
|
||||
|
||||
// Build docker run args
|
||||
const runArgs = [
|
||||
'run', '--rm',
|
||||
'-v', '/var/run/docker.sock:/var/run/docker.sock',
|
||||
'-v', `${deployDir}:${deployDir}`,
|
||||
'-w', deployDir,
|
||||
];
|
||||
const tailscaleIp = tailscaleIpForServer(server);
|
||||
if (tailscaleIp) runArgs.push('-e', `TAILSCALE_IP=${tailscaleIp}`);
|
||||
if (envFileWritten) runArgs.push('--env-file', envFilePath);
|
||||
runArgs.push('docker:cli');
|
||||
// Compose command (transient container will run it)
|
||||
runArgs.push('docker', 'compose', '-p', project, '-f', composeFile, 'up', '-d', '--no-deps', '--remove-orphans');
|
||||
if (force) runArgs.push('--force-recreate');
|
||||
runArgs.push(...services);
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
const child = spawn('docker', runArgs);
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
const tHandle = setTimeout(() => {
|
||||
try { child.kill('SIGKILL'); } catch (_) { /* ignore */ }
|
||||
}, timeout);
|
||||
child.stdout.on('data', d => { stdout += d.toString(); });
|
||||
child.stderr.on('data', d => { stderr += d.toString(); });
|
||||
child.on('close', (code) => {
|
||||
clearTimeout(tHandle);
|
||||
if (code === 0) resolve({ ok: true, stdout: stdout.trim(), stderr: stderr.trim() });
|
||||
else resolve({ ok: false, stdout: stdout.trim(), stderr: stderr.trim(), error: `docker run exit ${code}` });
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
clearTimeout(tHandle);
|
||||
resolve({ ok: false, stdout: '', stderr: String(err?.message || err), error: 'spawn failed' });
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
if (envFileWritten) {
|
||||
try { await fs.unlink(envFilePath); } catch (_) { /* ignore */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function run(cmd, opts = {}) {
|
||||
const timeout = opts.timeout || 30000;
|
||||
try {
|
||||
|
|
@ -598,10 +691,22 @@ ${detail}`);
|
|||
const stateless = allServices.filter(s => !stateful.includes(s));
|
||||
const deployStartTs = new Date();
|
||||
if (stateless.length > 0) {
|
||||
const upRes = await runOnServer(server, `cd ${deployDir} && ${envPrefix}${kvPrefix} docker compose up -d --force-recreate --remove-orphans ${stateless.join(' ')}`, { timeout: 300000 });
|
||||
// Use transient-container recreate so kua-deploy can self-update without
|
||||
// killing the compose-up process mid-flight. Same pattern works for all
|
||||
// apps (not just kua-deploy) and replaces the old runOnServer + kua-vault-run
|
||||
// shell prefix approach.
|
||||
const composeProject = path.basename(deployDir);
|
||||
const upRes = await recreateService({
|
||||
project: composeProject,
|
||||
deployDir,
|
||||
services: stateless,
|
||||
force: true,
|
||||
vault: prod.vault || null,
|
||||
server,
|
||||
});
|
||||
if (!upRes.ok) {
|
||||
steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) };
|
||||
throw new Error('docker compose up failed for stateless services');
|
||||
steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) || upRes.error };
|
||||
throw new Error('recreateService failed for stateless services');
|
||||
}
|
||||
// POST-DEPLOY VERIFY — catches false-success (see helper comment above).
|
||||
const verify = await verifyStatelessRecreated(server, deployDir, stateless, deployStartTs);
|
||||
|
|
@ -618,10 +723,20 @@ ${detail}`);
|
|||
}
|
||||
}
|
||||
if (stateful.length > 0) {
|
||||
const upRes = await runOnServer(server, `cd ${deployDir} && ${envPrefix}${kvPrefix} docker compose up -d --remove-orphans ${stateful.join(' ')}`, { timeout: 300000 });
|
||||
// Stateful services: start if not running but don't force-recreate
|
||||
// (db/redis must keep their volume + connection state).
|
||||
const composeProject = path.basename(deployDir);
|
||||
const upRes = await recreateService({
|
||||
project: composeProject,
|
||||
deployDir,
|
||||
services: stateful,
|
||||
force: false,
|
||||
vault: prod.vault || null,
|
||||
server,
|
||||
});
|
||||
if (!upRes.ok) {
|
||||
steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) };
|
||||
throw new Error('docker compose up failed for stateful services');
|
||||
steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) || upRes.error };
|
||||
throw new Error('recreateService failed for stateful services');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -782,11 +897,25 @@ async function rollback(appName) {
|
|||
const checkoutRes = await runOnServer(server, `cd ${deployDir} && git fetch --prune ${remote} && git checkout ${tag}`, { timeout: 60000 });
|
||||
if (!checkoutRes.ok) throw new Error(`Checkout ${tag} failed: ${checkoutRes.stderr}`);
|
||||
|
||||
// Rebuild and restart
|
||||
// Rebuild + recreate via transient-container pattern (consistent with deploy()).
|
||||
// Build runs via runOnServer (local exec when server=bruno); the recreate uses
|
||||
// the transient docker:cli so kua-deploy can roll back itself reliably.
|
||||
const kvPrefix = prod.vault
|
||||
? `kua-vault run --project ${prod.vault.project} --env ${prod.vault.env} --`
|
||||
: '';
|
||||
await runOnServer(server, `cd ${deployDir} && ${composeEnvPrefix(server)}${kvPrefix} docker compose up -d --force-recreate --build`, { timeout: 600000 });
|
||||
const buildRes = await runOnServer(server, `cd ${deployDir} && ${composeEnvPrefix(server)}${kvPrefix} docker compose build`, { timeout: 600000 });
|
||||
if (!buildRes.ok) throw new Error(`rollback build failed: ${buildRes.stderr?.slice(-500)}`);
|
||||
// Recreate all services for the rollback target.
|
||||
const svcList = (await runOnServer(server, `cd ${deployDir} && docker compose config --services`)).stdout.split('\n').filter(Boolean);
|
||||
const recreateRes = await recreateService({
|
||||
project: path.basename(deployDir),
|
||||
deployDir,
|
||||
services: svcList,
|
||||
force: true,
|
||||
vault: prod.vault || null,
|
||||
server,
|
||||
});
|
||||
if (!recreateRes.ok) throw new Error(`rollback recreate failed: ${recreateRes.stderr?.slice(-500) || recreateRes.error}`);
|
||||
|
||||
// Health check
|
||||
let healthy = true;
|
||||
|
|
|
|||
Loading…
Reference in New Issue