feat: recreateService transient-container helper + use it in deploy() + rollback()

Replaces the runOnServer("docker compose up -d --force-recreate") pattern with a one-shot transient docker:cli container that runs OUTSIDE kua-deploy lifecycle. Solves the self-recreate chicken-and-egg: when the target is kua-deploy itself, the recreate completes because the transient survives kua-deploy stopping (docker daemon does the actual work).

Secrets are fetched via kua-vault export, written to a 600-perm tempfile on /app/data, passed via --env-file (docker CLI reads it from kua-deploys perspective; never on the docker run command line). Tempfile is unlinked in finally{}.

Replaces: deploy() stateless recreate (force=true), deploy() stateful up (force=false), rollback() recreate (force=true with all-services svcList).

Build step keeps runOnServer (local exec on bruno) since build doesnt kill kua-deploy. envPrefix/kvPrefix vars retained for the build command.

server.js

@@ -3,7 +3,7 @@ import fs from 'fs/promises';
 import path from 'path';
 import crypto from 'crypto';
 import http from 'http';
-import { exec as execCb, execFile as execFileCb } from 'child_process';
+import { exec as execCb, execFile as execFileCb, spawn } from 'child_process';
 import { promisify } from 'util';
 
 const exec = promisify(execCb);
@@ -270,6 +270,99 @@ function composeEnvPrefix(server) {
   return tailscaleIp ? `TAILSCALE_IP=${tailscaleIp} ` : '';
 }
 
+// recreateService — spawn a one-shot transient docker:cli container that runs
+// `docker compose up -d` against the host docker socket. The transient container
+// is OUTSIDE the lifecycle of the service it recreates, so even when the target
+// is kua-deploy itself, the recreate completes (the transient survives kua-deploy
+// being stopped/started; the docker daemon does the actual work).
+//
+// Bind paths MUST be identical between host and transient container
+// (e.g. -v /root/apps/X:/root/apps/X) so compose's path resolution matches host
+// reality. Secrets are pre-fetched via kua-vault into a private --env-file rather
+// than being passed on the docker run command line.
+//
+// Returns the same { ok, stdout, stderr, error? } shape as run()/runOnServer()
+// so call sites can swap with minimal change.
+async function recreateService({
+  project,            // compose project name (basename(deployDir) typically)
+  deployDir,          // absolute path to the deploy dir on the docker host
+  services,           // array of service names to recreate
+  force = true,       // pass --force-recreate
+  vault = null,       // { project, env } — if set, fetch secrets via kua-vault export
+  server = 'bruno',   // for TAILSCALE_IP env var
+  composeFile = 'docker-compose.yml',
+  timeout = 300000,
+} = {}) {
+  if (!Array.isArray(services) || services.length === 0) {
+    return { ok: true, stdout: '', stderr: '', error: null, skipped: true };
+  }
+  // Stage env file on the kua-deploy data volume so the docker CLI (running
+  // inside kua-deploy) can read it. The transient container picks up vars via
+  // --env-file processed at submit time — no host-side mount needed.
+  const tmpName = `.env-recreate-${crypto.randomBytes(8).toString('hex')}`;
+  const envFilePath = `/app/data/${tmpName}`;
+  let envFileWritten = false;
+  try {
+    if (vault && vault.project) {
+      const envEnv = vault.env || 'prod';
+      // kua-vault export emits KEY=VALUE lines — directly compatible with --env-file
+      const exportRes = await run(`kua-vault export --project ${vault.project} --env ${envEnv}`, { timeout: 30000 });
+      if (!exportRes.ok) {
+        return { ok: false, stdout: '', stderr: `kua-vault export ${vault.project}/${envEnv} failed: ${exportRes.stderr?.slice(-300) || exportRes.error}`, error: 'vault export failed' };
+      }
+      // Strip any non KEY=VALUE lines (e.g. status banners) and validate
+      const envLines = exportRes.stdout
+        .split('\n')
+        .filter(l => /^[A-Z_][A-Z0-9_]*=/.test(l));
+      if (envLines.length === 0) {
+        return { ok: false, stdout: '', stderr: `kua-vault export returned no KEY=VALUE lines for ${vault.project}/${envEnv}`, error: 'empty vault export' };
+      }
+      await fs.writeFile(envFilePath, envLines.join('\n') + '\n', { mode: 0o600 });
+      envFileWritten = true;
+    }
+
+    // Build docker run args
+    const runArgs = [
+      'run', '--rm',
+      '-v', '/var/run/docker.sock:/var/run/docker.sock',
+      '-v', `${deployDir}:${deployDir}`,
+      '-w', deployDir,
+    ];
+    const tailscaleIp = tailscaleIpForServer(server);
+    if (tailscaleIp) runArgs.push('-e', `TAILSCALE_IP=${tailscaleIp}`);
+    if (envFileWritten) runArgs.push('--env-file', envFilePath);
+    runArgs.push('docker:cli');
+    // Compose command (transient container will run it)
+    runArgs.push('docker', 'compose', '-p', project, '-f', composeFile, 'up', '-d', '--no-deps', '--remove-orphans');
+    if (force) runArgs.push('--force-recreate');
+    runArgs.push(...services);
+
+    return await new Promise((resolve) => {
+      const child = spawn('docker', runArgs);
+      let stdout = '';
+      let stderr = '';
+      const tHandle = setTimeout(() => {
+        try { child.kill('SIGKILL'); } catch (_) { /* ignore */ }
+      }, timeout);
+      child.stdout.on('data', d => { stdout += d.toString(); });
+      child.stderr.on('data', d => { stderr += d.toString(); });
+      child.on('close', (code) => {
+        clearTimeout(tHandle);
+        if (code === 0) resolve({ ok: true, stdout: stdout.trim(), stderr: stderr.trim() });
+        else resolve({ ok: false, stdout: stdout.trim(), stderr: stderr.trim(), error: `docker run exit ${code}` });
+      });
+      child.on('error', (err) => {
+        clearTimeout(tHandle);
+        resolve({ ok: false, stdout: '', stderr: String(err?.message || err), error: 'spawn failed' });
+      });
+    });
+  } finally {
+    if (envFileWritten) {
+      try { await fs.unlink(envFilePath); } catch (_) { /* ignore */ }
+    }
+  }
+}
+
 async function run(cmd, opts = {}) {
   const timeout = opts.timeout || 30000;
   try {
@@ -598,10 +691,22 @@ ${detail}`);
       const stateless = allServices.filter(s => !stateful.includes(s));
       const deployStartTs = new Date();
       if (stateless.length > 0) {
-        const upRes = await runOnServer(server, `cd ${deployDir} && ${envPrefix}${kvPrefix} docker compose up -d --force-recreate --remove-orphans ${stateless.join(' ')}`, { timeout: 300000 });
+        // Use transient-container recreate so kua-deploy can self-update without
+        // killing the compose-up process mid-flight. Same pattern works for all
+        // apps (not just kua-deploy) and replaces the old runOnServer + kua-vault-run
+        // shell prefix approach.
+        const composeProject = path.basename(deployDir);
+        const upRes = await recreateService({
+          project: composeProject,
+          deployDir,
+          services: stateless,
+          force: true,
+          vault: prod.vault || null,
+          server,
+        });
         if (!upRes.ok) {
-          steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) };
+          steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) || upRes.error };
-          throw new Error('docker compose up failed for stateless services');
+          throw new Error('recreateService failed for stateless services');
         }
         // POST-DEPLOY VERIFY — catches false-success (see helper comment above).
         const verify = await verifyStatelessRecreated(server, deployDir, stateless, deployStartTs);
@@ -618,10 +723,20 @@ ${detail}`);
         }
       }
       if (stateful.length > 0) {
-        const upRes = await runOnServer(server, `cd ${deployDir} && ${envPrefix}${kvPrefix} docker compose up -d --remove-orphans ${stateful.join(' ')}`, { timeout: 300000 });
+        // Stateful services: start if not running but don't force-recreate
+        // (db/redis must keep their volume + connection state).
+        const composeProject = path.basename(deployDir);
+        const upRes = await recreateService({
+          project: composeProject,
+          deployDir,
+          services: stateful,
+          force: false,
+          vault: prod.vault || null,
+          server,
+        });
         if (!upRes.ok) {
-          steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) };
+          steps[steps.length - 1] = { step: 'deploy', status: 'failed', error: upRes.stderr?.slice(-500) || upRes.error };
-          throw new Error('docker compose up failed for stateful services');
+          throw new Error('recreateService failed for stateful services');
         }
       }
     }
@@ -782,11 +897,25 @@ async function rollback(appName) {
     const checkoutRes = await runOnServer(server, `cd ${deployDir} && git fetch --prune ${remote} && git checkout ${tag}`, { timeout: 60000 });
     if (!checkoutRes.ok) throw new Error(`Checkout ${tag} failed: ${checkoutRes.stderr}`);
 
-    // Rebuild and restart
+    // Rebuild + recreate via transient-container pattern (consistent with deploy()).
+    // Build runs via runOnServer (local exec when server=bruno); the recreate uses
+    // the transient docker:cli so kua-deploy can roll back itself reliably.
     const kvPrefix = prod.vault
       ? `kua-vault run --project ${prod.vault.project} --env ${prod.vault.env} --`
       : '';
-    await runOnServer(server, `cd ${deployDir} && ${composeEnvPrefix(server)}${kvPrefix} docker compose up -d --force-recreate --build`, { timeout: 600000 });
+    const buildRes = await runOnServer(server, `cd ${deployDir} && ${composeEnvPrefix(server)}${kvPrefix} docker compose build`, { timeout: 600000 });
+    if (!buildRes.ok) throw new Error(`rollback build failed: ${buildRes.stderr?.slice(-500)}`);
+    // Recreate all services for the rollback target.
+    const svcList = (await runOnServer(server, `cd ${deployDir} && docker compose config --services`)).stdout.split('\n').filter(Boolean);
+    const recreateRes = await recreateService({
+      project: path.basename(deployDir),
+      deployDir,
+      services: svcList,
+      force: true,
+      vault: prod.vault || null,
+      server,
+    });
+    if (!recreateRes.ok) throw new Error(`rollback recreate failed: ${recreateRes.stderr?.slice(-500) || recreateRes.error}`);
 
     // Health check
     let healthy = true;