From 9169c843810267c0d61a75875b325e7ab642ecb8 Mon Sep 17 00:00:00 2001 From: kua-deploy-split Date: Thu, 21 May 2026 18:24:13 -0400 Subject: [PATCH] feat: recreateService transient-container helper + use it in deploy() + rollback() Replaces the runOnServer("docker compose up -d --force-recreate") pattern with a one-shot transient docker:cli container that runs OUTSIDE kua-deploy lifecycle. Solves the self-recreate chicken-and-egg: when the target is kua-deploy itself, the recreate completes because the transient survives kua-deploy stopping (docker daemon does the actual work). Secrets are fetched via kua-vault export, written to a 600-perm tempfile on /app/data, passed via --env-file (docker CLI reads it from kua-deploys perspective; never on the docker run command line). Tempfile is unlinked in finally{}. Replaces: deploy() stateless recreate (force=true), deploy() stateful up (force=false), rollback() recreate (force=true with all-services svcList). Build step keeps runOnServer (local exec on bruno) since build doesnt kill kua-deploy. envPrefix/kvPrefix vars retained for the build command. --- server.js | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 138 insertions(+), 9 deletions(-) diff --git a/server.js b/server.js index 214ca83..fd1e2b9 100644 --- a/server.js +++ b/server.js @@ -3,7 +3,7 @@ import fs from 'fs/promises'; import path from 'path'; import crypto from 'crypto'; import http from 'http'; -import { exec as execCb, execFile as execFileCb } from 'child_process'; +import { exec as execCb, execFile as execFileCb, spawn } from 'child_process'; import { promisify } from 'util'; const exec = promisify(execCb); @@ -270,6 +270,99 @@ function composeEnvPrefix(server) { return tailscaleIp ? `TAILSCALE_IP=${tailscaleIp} ` : ''; } +// recreateService — spawn a one-shot transient docker:cli container that runs +// `docker compose up -d` against the host docker socket. The transient container +// is OUTSIDE the lifecycle of the service it recreates, so even when the target +// is kua-deploy itself, the recreate completes (the transient survives kua-deploy +// being stopped/started; the docker daemon does the actual work). +// +// Bind paths MUST be identical between host and transient container +// (e.g. -v /root/apps/X:/root/apps/X) so compose's path resolution matches host +// reality. Secrets are pre-fetched via kua-vault into a private --env-file rather +// than being passed on the docker run command line. +// +// Returns the same { ok, stdout, stderr, error? } shape as run()/runOnServer() +// so call sites can swap with minimal change. +async function recreateService({ + project, // compose project name (basename(deployDir) typically) + deployDir, // absolute path to the deploy dir on the docker host + services, // array of service names to recreate + force = true, // pass --force-recreate + vault = null, // { project, env } — if set, fetch secrets via kua-vault export + server = 'bruno', // for TAILSCALE_IP env var + composeFile = 'docker-compose.yml', + timeout = 300000, +} = {}) { + if (!Array.isArray(services) || services.length === 0) { + return { ok: true, stdout: '', stderr: '', error: null, skipped: true }; + } + // Stage env file on the kua-deploy data volume so the docker CLI (running + // inside kua-deploy) can read it. The transient container picks up vars via + // --env-file processed at submit time — no host-side mount needed. + const tmpName = `.env-recreate-${crypto.randomBytes(8).toString('hex')}`; + const envFilePath = `/app/data/${tmpName}`; + let envFileWritten = false; + try { + if (vault && vault.project) { + const envEnv = vault.env || 'prod'; + // kua-vault export emits KEY=VALUE lines — directly compatible with --env-file + const exportRes = await run(`kua-vault export --project ${vault.project} --env ${envEnv}`, { timeout: 30000 }); + if (!exportRes.ok) { + return { ok: false, stdout: '', stderr: `kua-vault export ${vault.project}/${envEnv} failed: ${exportRes.stderr?.slice(-300) || exportRes.error}`, error: 'vault export failed' }; + } + // Strip any non KEY=VALUE lines (e.g. status banners) and validate + const envLines = exportRes.stdout + .split('\n') + .filter(l => /^[A-Z_][A-Z0-9_]*=/.test(l)); + if (envLines.length === 0) { + return { ok: false, stdout: '', stderr: `kua-vault export returned no KEY=VALUE lines for ${vault.project}/${envEnv}`, error: 'empty vault export' }; + } + await fs.writeFile(envFilePath, envLines.join('\n') + '\n', { mode: 0o600 }); + envFileWritten = true; + } + + // Build docker run args + const runArgs = [ + 'run', '--rm', + '-v', '/var/run/docker.sock:/var/run/docker.sock', + '-v', `${deployDir}:${deployDir}`, + '-w', deployDir, + ]; + const tailscaleIp = tailscaleIpForServer(server); + if (tailscaleIp) runArgs.push('-e', `TAILSCALE_IP=${tailscaleIp}`); + if (envFileWritten) runArgs.push('--env-file', envFilePath); + runArgs.push('docker:cli'); + // Compose command (transient container will run it) + runArgs.push('docker', 'compose', '-p', project, '-f', composeFile, 'up', '-d', '--no-deps', '--remove-orphans'); + if (force) runArgs.push('--force-recreate'); + runArgs.push(...services); + + return await new Promise((resolve) => { + const child = spawn('docker', runArgs); + let stdout = ''; + let stderr = ''; + const tHandle = setTimeout(() => { + try { child.kill('SIGKILL'); } catch (_) { /* ignore */ } + }, timeout); + child.stdout.on('data', d => { stdout += d.toString(); }); + child.stderr.on('data', d => { stderr += d.toString(); }); + child.on('close', (code) => { + clearTimeout(tHandle); + if (code === 0) resolve({ ok: true, stdout: stdout.trim(), stderr: stderr.trim() }); + else resolve({ ok: false, stdout: stdout.trim(), stderr: stderr.trim(), error: `docker run exit ${code}` }); + }); + child.on('error', (err) => { + clearTimeout(tHandle); + resolve({ ok: false, stdout: '', stderr: String(err?.message || err), error: 'spawn failed' }); + }); + }); + } finally { + if (envFileWritten) { + try { await fs.unlink(envFilePath); } catch (_) { /* ignore */ } + } + } +} + async function run(cmd, opts = {}) { const timeout = opts.timeout || 30000; try { @@ -598,10 +691,22 @@ ${detail}`); const stateless = allServices.filter(s => !stateful.includes(s)); const deployStartTs = new Date(); if (stateless.length > 0) { - const upRes = await runOnServer(server, `cd ${deployDir} && ${envPrefix}${kvPrefix} docker compose up -d --force-recreate --remove-orphans ${stateless.join(' ')}`, { timeout: 300000 }); + // Use transient-container recreate so kua-deploy can self-update without + // killing the compose-up process mid-flight. Same pattern works for all + // apps (not just kua-deploy) and replaces the old runOnServer + kua-vault-run + // shell prefix approach. + const composeProject = path.basename(deployDir); + const upRes = await recreateService({ + project: composeProject, + deployDir, + services: stateless, + force: true, + vault: prod.vault || null, + server, + }); if (!upRes.ok) { - steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) }; - throw new Error('docker compose up failed for stateless services'); + steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) || upRes.error }; + throw new Error('recreateService failed for stateless services'); } // POST-DEPLOY VERIFY — catches false-success (see helper comment above). const verify = await verifyStatelessRecreated(server, deployDir, stateless, deployStartTs); @@ -618,10 +723,20 @@ ${detail}`); } } if (stateful.length > 0) { - const upRes = await runOnServer(server, `cd ${deployDir} && ${envPrefix}${kvPrefix} docker compose up -d --remove-orphans ${stateful.join(' ')}`, { timeout: 300000 }); + // Stateful services: start if not running but don't force-recreate + // (db/redis must keep their volume + connection state). + const composeProject = path.basename(deployDir); + const upRes = await recreateService({ + project: composeProject, + deployDir, + services: stateful, + force: false, + vault: prod.vault || null, + server, + }); if (!upRes.ok) { - steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) }; - throw new Error('docker compose up failed for stateful services'); + steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) || upRes.error }; + throw new Error('recreateService failed for stateful services'); } } } @@ -782,11 +897,25 @@ async function rollback(appName) { const checkoutRes = await runOnServer(server, `cd ${deployDir} && git fetch --prune ${remote} && git checkout ${tag}`, { timeout: 60000 }); if (!checkoutRes.ok) throw new Error(`Checkout ${tag} failed: ${checkoutRes.stderr}`); - // Rebuild and restart + // Rebuild + recreate via transient-container pattern (consistent with deploy()). + // Build runs via runOnServer (local exec when server=bruno); the recreate uses + // the transient docker:cli so kua-deploy can roll back itself reliably. const kvPrefix = prod.vault ? `kua-vault run --project ${prod.vault.project} --env ${prod.vault.env} --` : ''; - await runOnServer(server, `cd ${deployDir} && ${composeEnvPrefix(server)}${kvPrefix} docker compose up -d --force-recreate --build`, { timeout: 600000 }); + const buildRes = await runOnServer(server, `cd ${deployDir} && ${composeEnvPrefix(server)}${kvPrefix} docker compose build`, { timeout: 600000 }); + if (!buildRes.ok) throw new Error(`rollback build failed: ${buildRes.stderr?.slice(-500)}`); + // Recreate all services for the rollback target. + const svcList = (await runOnServer(server, `cd ${deployDir} && docker compose config --services`)).stdout.split('\n').filter(Boolean); + const recreateRes = await recreateService({ + project: path.basename(deployDir), + deployDir, + services: svcList, + force: true, + vault: prod.vault || null, + server, + }); + if (!recreateRes.ok) throw new Error(`rollback recreate failed: ${recreateRes.stderr?.slice(-500) || recreateRes.error}`); // Health check let healthy = true;