import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import test from 'node:test'
import { canonicalJson } from '../lib/canonical-json.mjs'
import { BootstrapRuntime } from '../bootstrap/lib/runtime.mjs'
import { EnvironmentRunner, loadControlPlane } from '../bootstrap/lib/lifecycle.mjs'
import { createBootstrapControl, listenBootstrapControl } from '../bootstrap/lib/control.mjs'
import { LocalApiClient } from '../../control-plane/modules/updater/lib/client.mjs'

function command(script, args = []) {
  return { executable: process.execPath, args: [script, ...args], timeoutSeconds: 5 }
}

const emptyLifecycle = {
  prepare: null,
  preStart: null,
  postStart: null,
  preStop: null,
  stop: null,
  postStop: null,
}

async function environment(components) {
  const root = await mkdtemp(join(tmpdir(), 'dsh-environment-runner-'))
  await mkdir(join(root, 'artifacts'))
  const references = []
  const artifacts = []
  for (const component of components) {
    const artifactId = `component-${component.id}`
    const bytes = canonicalJson(component)
    const sha256 = createHash('sha256').update(bytes).digest('hex')
    await writeFile(join(root, 'artifacts', artifactId), bytes)
    references.push({ id: component.id, sha256 })
    artifacts.push({
      id: artifactId,
      mediaType: 'application/vnd.dsh-platform.component.v1+json',
      sha256,
      size: bytes.byteLength,
      url: `https://release.example/${artifactId}`,
    })
  }
  await writeFile(join(root, 'environment.manifest.json'), canonicalJson({
    schema: 1,
    manifestType: 'environment',
    version: 'test',
    keyringGeneration: 1,
    targetSequence: 1,
    issuedAt: '2026-08-19T00:00:00.000Z',
    artifacts,
    bootstrapApi: 1,
    components: references,
    patches: [],
    systemPlugins: [],
  }))
  return root
}

function component(id, script, type = 'oneshot', lifecycle = emptyLifecycle) {
  return {
    schema: 1,
    id,
    type,
    command: command(script),
    environment: {},
    lifecycle,
    health: null,
    logging: { stdout: true, stderr: true },
  }
}

test('runs components in manifest order and stop phases in reverse order', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-order-'))
  const log = join(temp, 'order.log')
  const append = join(temp, 'append.mjs')
  const service = join(temp, 'service.mjs')
  const check = join(temp, 'check.mjs')
  await writeFile(append, 'import { appendFileSync } from "node:fs"; appendFileSync(process.argv[2], process.argv[3] + "\\n")')
  await writeFile(service, 'import { appendFileSync } from "node:fs"; appendFileSync(process.argv[2], process.argv[3] + "\\n"); setInterval(() => {}, 1000)')
  await writeFile(check, 'import { readFileSync } from "node:fs"; process.exit(readFileSync(process.argv[2], "utf8").includes(process.argv[3]) ? 0 : 1)')
  const lifecycle = label => ({
    ...emptyLifecycle,
    prepare: command(append, [log, `${label}:prepare`]),
    preStart: command(append, [log, `${label}:preStart`]),
    postStart: command(append, [log, `${label}:postStart`]),
    preStop: command(append, [log, `${label}:preStop`]),
    postStop: command(append, [log, `${label}:postStop`]),
  })
  const first = component('first', service, 'service', lifecycle('first'))
  first.command = command(service, [log, 'first:start'])
  first.health = {
    type: 'exec',
    command: command(check, [log, 'first:start']),
    intervalSeconds: 1,
    timeoutSeconds: 5,
  }
  const second = component('second', append, 'oneshot', lifecycle('second'))
  second.command = command(append, [log, 'second:start'])
  const reports = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([first, second]),
    capture: () => {},
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await runner.start()
  await runner.stop()
  const lines = (await readFile(log, 'utf8')).trim().split('\n')
  assert.deepEqual(lines.slice(0, 8), [
    'first:prepare', 'second:prepare', 'first:preStart', 'first:start',
    'first:postStart', 'second:preStart', 'second:start', 'second:postStart',
  ])
  assert.deepEqual(lines.slice(8), ['second:preStop', 'second:postStop', 'first:preStop', 'first:postStop'])
  assert.deepEqual(reports.map(report => [report.message, report.fields.componentId]), [
    ['component.starting', 'first'],
    ['component.spawned', 'first'],
    ['component.ready', 'first'],
    ['component.starting', 'second'],
    ['component.ready', 'second'],
    ['component.stopping', 'second'],
    ['component.stopped', 'second'],
    ['component.stopping', 'first'],
    ['component.stopped', 'first'],
  ])
  assert.equal(reports.find(report => report.message === 'component.ready').fields.elapsedMs >= 0, true)
})

test('starts explicit component groups concurrently and waits before the next group', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-groups-'))
  const barrier = join(temp, 'barrier.mjs')
  const after = join(temp, 'after.mjs')
  const firstStarted = join(temp, 'first.started')
  const secondStarted = join(temp, 'second.started')
  const firstDone = join(temp, 'first.done')
  const secondDone = join(temp, 'second.done')
  await writeFile(barrier, `
    import { existsSync, writeFileSync } from 'node:fs'
    writeFileSync(process.argv[2], '')
    while (!existsSync(process.argv[3])) await new Promise(resolve => setTimeout(resolve, 10))
    writeFileSync(process.argv[4], '')
  `)
  await writeFile(after, `
    import { existsSync } from 'node:fs'
    process.exit(existsSync(process.argv[2]) && existsSync(process.argv[3]) ? 0 : 9)
  `)
  const first = component('first', barrier)
  first.command = command(barrier, [firstStarted, secondStarted, firstDone])
  const second = component('second', barrier)
  second.command = command(barrier, [secondStarted, firstStarted, secondDone])
  const final = component('final', after)
  final.command = command(after, [firstDone, secondDone])
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([first, second, final]),
    capture: () => {},
    startupGroups: [['first', 'second'], ['final']],
  })
  await runner.start()
  assert.deepEqual(runner.status().components.map(value => value.id), ['first', 'second', 'final'])
  await runner.stop()
})

test('waits for a failed startup group before rolling back every started service', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-group-failure-'))
  const service = join(temp, 'service.mjs')
  const failure = join(temp, 'failure.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  await writeFile(failure, 'process.exit(9)')
  const reports = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([
      component('service', service, 'service'),
      component('failure', failure),
    ]),
    capture: () => {},
    report: (message, fields) => { reports.push({ message, fields }) },
    startupGroups: [['service', 'failure']],
  })
  await assert.rejects(runner.start(), /failure command failed/)
  assert.deepEqual(runner.status().components, [])
  assert.equal(reports.some(report => report.message === 'component.stopped'
    && report.fields.componentId === 'service'), true)
})

test('does not recover an isolated startup failure after its group fatally fails', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-group-recovery-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  const reports = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([
      component('gateway', service, 'service'),
      component('outbound-proxy', service, 'service'),
    ]),
    capture: () => {},
    recoverableComponents: ['outbound-proxy'],
    recoveryDelaysMs: [0],
    report: (message, fields) => { reports.push({ message, fields }) },
    startupGroups: [['gateway', 'outbound-proxy']],
  })
  runner.startComponentUnlocked = candidate => Promise.reject(new Error(`${candidate.id} unavailable`))
  await assert.rejects(runner.start(), /gateway unavailable/)
  await new Promise(resolve => setImmediate(resolve))
  assert.equal(reports.some(report => report.message === 'component.recovery.started'), false)
})

test('rejects incomplete or duplicated startup groups before component preparation', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-invalid-groups-'))
  const marker = join(temp, 'prepared')
  const append = join(temp, 'append.mjs')
  await writeFile(append, `import { writeFileSync } from 'node:fs'; writeFileSync(process.argv[2], '')`)
  const first = component('first', append)
  first.lifecycle = { ...emptyLifecycle, prepare: command(append, [marker]) }
  const root = await environment([first, component('second', append)])
  const omitted = new EnvironmentRunner({
    environmentRoot: root,
    capture: () => {},
    startupGroups: [['first']],
  })
  await assert.rejects(omitted.start(), /omit components: second/)
  await assert.rejects(readFile(marker), error => error?.code === 'ENOENT')
  const duplicated = new EnvironmentRunner({
    environmentRoot: root,
    capture: () => {},
    startupGroups: [['first'], ['first', 'second']],
  })
  await assert.rejects(duplicated.start(), /component first is duplicated/)
})

test('exec health probes use only their exit status and do not emit component logs', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-health-output-'))
  const service = join(temp, 'service.mjs')
  const health = join(temp, 'health.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  await writeFile(health, 'console.log(JSON.stringify({ noisy: true }, null, 2))')
  const candidate = component('service', service, 'service')
  candidate.health = {
    type: 'exec',
    command: command(health),
    intervalSeconds: 1,
    timeoutSeconds: 5,
  }
  let captures = 0
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([candidate]),
    capture: () => { captures += 1 },
  })
  await runner.start()
  assert.equal((await runner.health()).healthy, true)
  assert.equal(captures, 1)
  await runner.stop()
})

test('startup health retries do not inherit the steady one-second interval', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-health-retry-'))
  const service = join(temp, 'service.mjs')
  const health = join(temp, 'health.mjs')
  const attempts = join(temp, 'attempts')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  await writeFile(health, `import { existsSync, writeFileSync } from 'node:fs'; if (!existsSync(process.argv[2])) { writeFileSync(process.argv[2], 'first'); process.exit(1) }`)
  const candidate = component('service', service, 'service')
  candidate.health = {
    type: 'exec',
    command: command(health, [attempts]),
    intervalSeconds: 2,
    timeoutSeconds: 5,
  }
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([candidate]),
    capture: () => {},
  })
  const started = Date.now()
  await runner.start()
  const elapsedMs = Date.now() - started
  assert.ok(elapsedMs < 1_000, `startup health retry took ${String(elapsedMs)} ms`)
  await runner.stop()
})

test('component environment replaces an inherited root user environment', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-component-environment-'))
  const output = join(temp, 'environment.json')
  const inspect = join(temp, 'inspect.mjs')
  await writeFile(inspect, `import { writeFileSync } from 'node:fs'; writeFileSync(process.argv[2], JSON.stringify({ HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME, XDG_CACHE_HOME: process.env.XDG_CACHE_HOME, XDG_DATA_HOME: process.env.XDG_DATA_HOME }))`)
  const candidate = component('dsh-runtime', inspect)
  candidate.command = command(inspect, [output])
  candidate.environment = {
    HOME: '/home/node',
    USER: 'node',
    LOGNAME: 'node',
    XDG_CONFIG_HOME: '/home/node/.config',
    XDG_CACHE_HOME: '/home/node/.cache',
    XDG_DATA_HOME: '/home/node/.local/share',
  }
  const original = {
    HOME: process.env.HOME,
    USER: process.env.USER,
    LOGNAME: process.env.LOGNAME,
    XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME,
    XDG_CACHE_HOME: process.env.XDG_CACHE_HOME,
    XDG_DATA_HOME: process.env.XDG_DATA_HOME,
  }
  process.env.HOME = '/root'
  process.env.USER = 'root'
  process.env.LOGNAME = 'root'
  process.env.XDG_CONFIG_HOME = '/root/.config'
  process.env.XDG_CACHE_HOME = '/root/.cache'
  process.env.XDG_DATA_HOME = '/root/.local/share'
  try {
    const runner = new EnvironmentRunner({
      environmentRoot: await environment([candidate]),
      capture: () => {},
    })
    await runner.start()
    assert.deepEqual(JSON.parse(await readFile(output, 'utf8')), candidate.environment)
  } finally {
    for (const [name, value] of Object.entries(original)) {
      if (value === undefined) delete process.env[name]
      else process.env[name] = value
    }
  }
})

test('binds prepared service environment and cleanup to one process launch', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-prepared-service-'))
  const output = join(temp, 'launch-token')
  const service = join(temp, 'service.mjs')
  const health = join(temp, 'health.mjs')
  await writeFile(service, `import { writeFileSync } from 'node:fs'; writeFileSync(process.argv[2], process.env.DSH_PLATFORM_LAUNCH_TOKEN); setInterval(() => {}, 1000)`)
  await writeFile(health, `import { existsSync } from 'node:fs'; process.exit(existsSync(process.argv[2]) ? 0 : 1)`)
  let releases = 0
  const candidate = component('dsh-runtime', service, 'service')
  candidate.command = command(service, [output])
  candidate.health = {
    type: 'exec', command: command(health, [output]), intervalSeconds: 1, timeoutSeconds: 5,
  }
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([candidate]),
    capture: () => {},
    prepareService: async value => ({
      environment: { DSH_PLATFORM_LAUNCH_TOKEN: `token-for-${value.id}` },
      release: () => { releases += 1 },
    }),
  })
  try {
    await runner.start()
    assert.equal(await readFile(output, 'utf8'), 'token-for-dsh-runtime')
    assert.equal(releases, 0)
  } finally {
    await runner.stop()
  }
  assert.equal(releases, 1)
})

test('releases prepared service ownership when process spawn fails', async () => {
  let releases = 0
  const candidate = component('dsh-runtime', '/path/that/does/not/exist', 'service')
  candidate.command.executable = '/path/that/does/not/exist'
  candidate.command.args = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([candidate]),
    capture: () => {},
    prepareService: async () => ({
      environment: { DSH_PLATFORM_LAUNCH_TOKEN: 'unused-token' },
      release: () => { releases += 1 },
    }),
  })
  await assert.rejects(runner.start(), /ENOENT/)
  assert.equal(releases, 1)
})

test('stops already-started services when a later component fails', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-failure-'))
  const service = join(temp, 'service.mjs')
  const failure = join(temp, 'failure.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  await writeFile(failure, 'process.exit(9)')
  const reports = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([
      component('service', service, 'service'),
      component('failure', failure),
    ]),
    capture: () => {},
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await assert.rejects(runner.start(), /failure command failed/)
  assert.deepEqual(runner.status().components, [])
  const failureReport = reports.find(report => report.message === 'component.start.failed').fields
  assert.equal(failureReport.componentId, 'failure')
  assert.equal(failureReport.elapsedMs >= 0, true)
  assert.match(failureReport.error, /failure command failed.*code=9/)
  assert.equal(failureReport.level, 'error')
})

test('reports a service exit after readiness as a fatal Bootstrap condition', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-runtime-exit-'))
  const exits = join(temp, 'exits.mjs')
  await writeFile(exits, 'setTimeout(() => process.exit(7), 20)')
  const reports = []
  const observed = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([component('service', exits, 'service')]),
    capture: () => {},
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  runner.onFatal(error => observed.push(error))
  await runner.start()
  const error = await runner.fatal
  assert.match(error.message, /service exited unexpectedly.*code=7/)
  assert.equal(error.componentId, 'service')
  assert.equal(error.exitCode, 7)
  assert.equal(observed[0], error)
  assert.equal(reports.find(report => report.message === 'component.exited').fields.level, 'error')
  await runner.stop()
})

test('recovers an opted-in service exit without stopping sibling services', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-isolated-recovery-'))
  const counter = join(temp, 'counter')
  const recoverable = join(temp, 'recoverable.mjs')
  const sibling = join(temp, 'sibling.mjs')
  await writeFile(recoverable, `import { existsSync, writeFileSync } from 'node:fs'; if (!existsSync(process.argv[2])) { writeFileSync(process.argv[2], '1'); setTimeout(() => process.exit(7), 20) } else setInterval(() => {}, 1000)`)
  await writeFile(sibling, 'setInterval(() => {}, 1000)')
  const candidate = component('outbound-proxy', recoverable, 'service')
  candidate.command = command(recoverable, [counter])
  const reports = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([candidate, component('management', sibling, 'service')]),
    capture: () => {},
    recoverableComponents: ['outbound-proxy'],
    recoveryDelaysMs: [0, 0, 0],
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await runner.start()
  const deadline = Date.now() + 2_000
  while (!reports.some(report => report.message === 'component.recovery.completed')) {
    if (Date.now() >= deadline) throw new Error('recoverable component did not restart')
    await new Promise(resolve => setTimeout(resolve, 10))
  }
  assert.deepEqual(runner.status().components.map(value => value.id), ['outbound-proxy', 'management'])
  assert.equal(reports.some(report => report.message === 'component.exited'), true)
  assert.equal(await Promise.race([
    runner.fatal.then(() => 'fatal'),
    new Promise(resolve => setTimeout(() => resolve('isolated'), 20)),
  ]), 'isolated')
  await runner.stop()
})

test('isolates a persistently failing opted-in component and starts later services', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-isolated-startup-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  const reports = []
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([
      component('outbound-proxy', service, 'service'),
      component('management', service, 'service'),
    ]),
    capture: () => {},
    recoverableComponents: ['outbound-proxy'],
    recoveryDelaysMs: [0, 0, 0],
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  const start = runner.startComponentUnlocked.bind(runner)
  runner.startComponentUnlocked = (candidate, prepare) => candidate.id === 'outbound-proxy'
    ? Promise.reject(new Error('proxy port conflict'))
    : start(candidate, prepare)
  await runner.start()
  const deadline = Date.now() + 2_000
  while (reports.filter(report => report.message === 'component.isolated').length < 2) {
    if (Date.now() >= deadline) throw new Error('proxy failure was not isolated')
    await new Promise(resolve => setTimeout(resolve, 10))
  }
  assert.deepEqual(runner.status().components.map(value => value.id), ['management'])
  assert.equal(reports.filter(report => report.message === 'component.recovery.failed').length, 3)
  await runner.stop()
})

test('contains a service exit during readiness inside the startup operation', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-startup-exit-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'process.exit(1)')
  const candidate = component('candidate', service, 'service')
  candidate.health = {
    type: 'http', host: '127.0.0.1', port: 1, path: '/', intervalSeconds: 1, timeoutSeconds: 1,
  }
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([candidate]),
    capture: () => {},
  })
  await assert.rejects(runner.start(), /exited before health check passed/)
  const outcome = await Promise.race([
    runner.fatal.then(() => 'fatal'),
    new Promise(resolve => setTimeout(() => resolve('contained'), 20)),
  ])
  assert.equal(outcome, 'contained')
})

test('suspends, resumes, and restarts one service while keeping other Environment components running', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-suspend-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([
      component('dsh-runtime', service, 'service'),
      component('platform-management', service, 'service'),
    ]),
    capture: () => {},
  })
  await runner.start()
  await runner.suspend('dsh-runtime')
  await runner.pause('dsh-runtime')
  await runner.pause('dsh-runtime')
  await assert.rejects(runner.pause('unknown-service'), /does not exist/)
  assert.deepEqual(runner.status().components.map(value => value.id), ['platform-management'])
  assert.equal((await runner.health()).healthy, false)
  await runner.resume('dsh-runtime')
  assert.deepEqual(runner.status().components.map(value => value.id), ['dsh-runtime', 'platform-management'])
  const managementPid = runner.status().components.find(value => value.id === 'platform-management').pid
  const firstDshPid = runner.status().components.find(value => value.id === 'dsh-runtime').pid
  await runner.restart('dsh-runtime')
  assert.notEqual(runner.status().components.find(value => value.id === 'dsh-runtime').pid, firstDshPid)
  assert.equal(runner.status().components.find(value => value.id === 'platform-management').pid, managementPid)
  assert.equal((await runner.health()).healthy, true)
  assert.equal(await Promise.race([
    runner.fatal.then(() => 'fatal'),
    new Promise(resolve => setTimeout(() => resolve('intentional'), 20)),
  ]), 'intentional')
  await runner.stop()
})

test('retries a service restart after its first start attempt fails', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-restart-retry-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([component('dsh-runtime', service, 'service')]),
    capture: () => {},
  })
  await runner.start()
  const startComponent = runner.startComponentUnlocked.bind(runner)
  let failStart = true
  runner.startComponentUnlocked = async (...args) => {
    if (failStart) {
      failStart = false
      throw new Error('restart failed')
    }
    return startComponent(...args)
  }
  await assert.rejects(runner.restart('dsh-runtime'), /restart failed/)
  assert.deepEqual(runner.status().components, [])
  await runner.restart('dsh-runtime')
  assert.deepEqual(runner.status().components.map(value => value.id), ['dsh-runtime'])
  await runner.stop()
})

test('restores a service after a transactional restart candidate fails', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-restart-recovery-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  const runner = new EnvironmentRunner({
    environmentRoot: await environment([component('dsh-runtime', service, 'service')]),
    capture: () => {},
  })
  await runner.start()
  const startComponent = runner.startComponentUnlocked.bind(runner)
  let candidate = true
  runner.startComponentUnlocked = async (...args) => {
    if (candidate) {
      candidate = false
      throw new Error('candidate failed')
    }
    return startComponent(...args)
  }
  const changes = []
  await assert.rejects(runner.restart('dsh-runtime', {
    recoverStart: true,
    beforeStart: async () => { changes.push('activate') },
    onStartFailure: async () => { changes.push('rollback') },
  }), /candidate failed/)
  assert.deepEqual(changes, ['activate', 'rollback'])
  assert.deepEqual(runner.status().components.map(value => value.id), ['dsh-runtime'])
  await runner.stop()
})

test('Bootstrap control socket exposes component suspension, resumption, restart, and health', async () => {
  const root = await mkdtemp(join(tmpdir(), 'dsh-bootstrap-control-'))
  const calls = []
  const runner = {
    status: () => ({ components: [] }),
    reload: async () => ({}),
    health: async () => ({ healthy: true, components: [] }),
    suspend: async id => { calls.push(['suspend', id]); return {} },
    pause: async id => { calls.push(['pause', id]); return {} },
    resume: async id => { calls.push(['resume', id]); return {} },
    restart: async (id, options = {}) => {
      calls.push(['restart', id])
      try {
        await options.beforeStart?.()
        return {}
      } catch (error) {
        await options.onStartFailure?.(error)
        if (options.recoverStart) calls.push(['restart-recovered', id])
        throw error
      }
    },
    record: async (message, fields) => { calls.push(['report', message, fields.pathname ?? null]) },
  }
  const deployments = {
    exclusive: operation => operation(),
    setOperation: async operation => { calls.push(['operation', operation]) },
    publishStatus: async options => { calls.push(['status', 'published', options ?? null]) },
    resetCurrentRuntime: async ({ pauseDsh, restartDsh }) => {
      calls.push(['runtime-reset'])
      await pauseDsh()
      await restartDsh()
      return { recordId: 'repaired' }
    },
  }
  const systemPlugins = {
    prepare: async () => {
      calls.push(['prepare-system-plugins'])
      return {
        activate: async () => { calls.push(['activate-system-plugins']) },
        commit: async () => { calls.push(['commit-system-plugins']) },
        rollback: async () => { calls.push(['rollback-system-plugins']) },
      }
    },
    list: async () => [{ id: 'platform-management', installed: true }],
    configure: async (id, action) => {
      calls.push(['configure', id, action])
      return [{ id, installed: true }]
    },
    recover: async (id, action) => {
      calls.push(['recover', id, action])
      return [{ id, installed: false }]
    },
    discard: async () => {
      calls.push(['discard-system-plugins'])
      return [{ id: 'platform-management', installed: true }]
    },
  }
  const server = createBootstrapControl(runner, { deployments, systemPlugins })
  const socket = join(root, 'run', 'bootstrap.sock')
  await listenBootstrapControl(server, socket)
  const client = new LocalApiClient(socket)
  try {
    assert.equal((await client.request('GET', '/v1/health')).healthy, true)
    assert.deepEqual((await client.request('GET', '/v1/system-plugins')).plugins, [{
      id: 'platform-management', installed: true,
    }])
    assert.deepEqual((await client.request('POST', '/v1/system-plugins/action', {
      id: 'diagnostics', action: 'disable',
    })).plugins, [{ id: 'diagnostics', installed: true }])
    assert.deepEqual((await client.request('POST', '/v1/system-plugins/recovery-action', {
      id: 'platform-management', action: 'uninstall',
    })).plugins, [{ id: 'platform-management', installed: false }])
    assert.deepEqual((await client.request('POST', '/v1/system-plugins/discard')).plugins, [{
      id: 'platform-management', installed: true,
    }])
    await client.request('POST', '/v1/components/dsh-runtime/suspend')
    await client.request('POST', '/v1/components/dsh-runtime/pause')
    await client.request('POST', '/v1/components/dsh-runtime/resume')
    await client.request('POST', '/v1/components/dsh-runtime/restart')
    assert.deepEqual(calls, [
      ['configure', 'diagnostics', 'disable'],
      ['recover', 'platform-management', 'uninstall'],
      ['discard-system-plugins'],
      ['suspend', 'dsh-runtime'],
      ['pause', 'dsh-runtime'],
      ['operation', 'starting'],
      ['resume', 'dsh-runtime'],
      ['status', 'published', null],
      ['operation', 'restarting'],
      ['prepare-system-plugins'],
      ['restart', 'dsh-runtime'],
      ['activate-system-plugins'],
      ['commit-system-plugins'],
      ['status', 'published', null],
    ])
    assert.equal((await client.request('POST', '/v1/deployments/runtime/reset')).recordId, 'repaired')
    assert.deepEqual(calls.slice(-3), [
      ['runtime-reset'],
      ['pause', 'dsh-runtime'],
      ['restart', 'dsh-runtime'],
    ])
    systemPlugins.prepare = async () => {
      calls.push(['prepare-system-plugins-failed'])
      return {
        activate: async () => {
          calls.push(['activate-system-plugins-failed'])
          throw new Error('overlay failed')
        },
        rollback: async () => { calls.push(['rollback-system-plugins']) },
      }
    }
    await assert.rejects(client.request('POST', '/v1/components/dsh-runtime/restart'), /overlay failed/)
    assert.deepEqual(calls.slice(-8), [
      ['operation', 'restarting'],
      ['prepare-system-plugins-failed'],
      ['restart', 'dsh-runtime'],
      ['activate-system-plugins-failed'],
      ['rollback-system-plugins'],
      ['restart-recovered', 'dsh-runtime'],
      ['status', 'published', { operation: 'restart-failed', recoveryMode: 'overlay failed' }],
      ['report', 'bootstrap.request.failed', '/v1/components/dsh-runtime/restart'],
    ])

    runner.resume = async id => {
      calls.push(['resume-failed', id])
      throw new Error('resume remained unavailable')
    }
    runner.status = () => ({ components: [], recoveryMode: 'resume remained unavailable' })
    await assert.rejects(
      client.request('POST', '/v1/components/dsh-runtime/resume'),
      /resume remained unavailable/,
    )
    assert.deepEqual(calls.slice(-4), [
      ['operation', 'starting'],
      ['resume-failed', 'dsh-runtime'],
      ['status', 'published', { recoveryMode: 'resume remained unavailable' }],
      ['report', 'bootstrap.request.failed', '/v1/components/dsh-runtime/resume'],
    ])
  } finally {
    await new Promise(resolve => server.close(resolve))
  }
})

test('loads the checked-in Control Plane independently from an Environment manifest', async () => {
  const controlPlane = await loadControlPlane(join(
    dirname(dirname(fileURLToPath(import.meta.url))),
    '..',
    'control-plane',
  ))
  assert.equal(controlPlane.manifest.version, null)
  assert.deepEqual(controlPlane.components.map(component => component.id), [
    'gateway', 'access-manager', 'outbound-proxy', 'platform-recovery', 'platform-management',
  ])
})

test('keeps the Control Plane running while Environment operations replace DSH', async () => {
  const calls = []
  const runner = (name, status) => ({
    fatal: new Promise(() => {}),
    start: async () => { calls.push(`${name}:start`) },
    stop: async () => { calls.push(`${name}:stop`) },
    reload: async () => { calls.push(`${name}:reload`); return status },
    suspend: async id => { calls.push(`${name}:suspend:${id}`); return status },
    pause: async id => { calls.push(`${name}:pause:${id}`); return status },
    resume: async id => { calls.push(`${name}:resume:${id}`); return status },
    restart: async id => { calls.push(`${name}:restart:${id}`); return status },
    health: async () => ({ healthy: true, components: status.components }),
    status: () => status,
  })
  const controlPlane = runner('control', {
    environmentVersion: null,
    components: [{ id: 'gateway' }],
  })
  const environmentRunner = runner('environment', {
    environmentVersion: '1.0.0',
    components: [{ id: 'dsh-runtime' }],
  })
  const runtime = new BootstrapRuntime({
    controlPlane,
    environment: environmentRunner,
    onEnvironmentVerified: async () => { calls.push('environment:verified') },
    onDshAvailable: async () => { calls.push('dsh:available') },
  })

  await runtime.start()
  await runtime.suspend('dsh-runtime')
  assert.equal(runtime.status().dshLifecycle.state, 'stopped')
  await runtime.reload()
  assert.equal(runtime.status().dshLifecycle.state, 'running')
  await runtime.pause('dsh-runtime')
  await runtime.resume('dsh-runtime')
  assert.equal(runtime.status().dshLifecycle.state, 'running')
  await runtime.restart('dsh-runtime')
  await runtime.reload()
  assert.equal((await runtime.health()).healthy, true)
  assert.deepEqual(runtime.status().controlPlane, [{ id: 'gateway' }])
  await runtime.stop()

  assert.deepEqual(calls, [
    'control:start',
    'environment:start',
    'environment:verified',
    'environment:suspend:dsh-runtime',
    'environment:reload',
    'environment:verified',
    'dsh:available',
    'environment:pause:dsh-runtime',
    'environment:resume:dsh-runtime',
    'environment:verified',
    'dsh:available',
    'environment:restart:dsh-runtime',
    'environment:verified',
    'dsh:available',
    'environment:reload',
    'environment:verified',
    'dsh:available',
    'environment:stop',
    'control:stop',
  ])
})

test('starts DSH alongside the Control Plane but gates readiness on access classification', async () => {
  const calls = []
  let resolveEnvironmentStarted
  const environmentStarted = new Promise(resolve => { resolveEnvironmentStarted = resolve })
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}),
      start: async () => { calls.push('control:start'); await environmentStarted },
      stop: async () => { calls.push('control:stop') },
      status: () => ({ components: [{ id: 'access-manager' }] }),
    },
    environment: {
      fatal: new Promise(() => {}),
      start: async () => { calls.push('environment:start'); resolveEnvironmentStarted() },
      stop: async () => { calls.push('environment:stop') },
      status: () => ({ environmentVersion: '1.0.0', components: [{ id: 'dsh-runtime' }] }),
    },
    beforeReady: async () => { calls.push('access:classify') },
  })

  await runtime.start()
  assert.deepEqual(calls, ['control:start', 'environment:start', 'access:classify'])
  await runtime.stop()

  const blocked = []
  let resolveBlockedEnvironmentStarted
  const blockedEnvironmentStarted = new Promise(resolve => { resolveBlockedEnvironmentStarted = resolve })
  const failure = new Error('access classification failed')
  const blockedRuntime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}),
      start: async () => { blocked.push('control:start'); await blockedEnvironmentStarted },
      stop: async () => { blocked.push('control:stop') },
      status: () => ({ components: [{ id: 'access-manager' }] }),
    },
    environment: {
      fatal: new Promise(() => {}),
      start: async () => { blocked.push('environment:start'); resolveBlockedEnvironmentStarted() },
      stop: async () => { blocked.push('environment:stop') },
      status: () => ({ environmentVersion: '1.0.0', components: [] }),
    },
    beforeReady: async () => { blocked.push('access:classify'); throw failure },
  })

  await assert.rejects(blockedRuntime.start(), error => error === failure)
  assert.deepEqual(blocked, [
    'control:start', 'environment:start', 'access:classify', 'environment:stop', 'control:stop',
  ])
})

test('isolates a non-DSH Environment exit and keeps the Control Plane available', async () => {
  const calls = []
  let emitEnvironmentFatal
  const controlPlane = {
    fatal: new Promise(() => {}),
    start: async () => { calls.push('control:start') },
    stop: async () => { calls.push('control:stop') },
    status: () => ({ components: [{ id: 'gateway' }, { id: 'platform-management' }] }),
  }
  const environmentRunner = {
    fatal: new Promise(() => {}),
    onFatal: listener => {
      emitEnvironmentFatal = listener
      return () => { emitEnvironmentFatal = undefined }
    },
    start: async () => { calls.push('environment:start') },
    stop: async () => { calls.push('environment:stop') },
    restart: async id => { calls.push(`environment:restart:${id}`); return { components: [{ id }] } },
    status: () => ({ environmentVersion: '1.0.0', components: [] }),
  }
  const runtime = new BootstrapRuntime({
    controlPlane,
    environment: environmentRunner,
    onEnvironmentFatal: async error => { calls.push(`recovery:${error.message}`) },
  })
  await runtime.start()
  const failure = Object.assign(new Error('worker exited unexpectedly'), { componentId: 'worker' })
  emitEnvironmentFatal(failure)
  await runtime.recovery

  assert.equal(runtime.status().recoveryMode, 'worker exited unexpectedly')
  assert.deepEqual(runtime.status().controlPlane.map(component => component.id), ['gateway', 'platform-management'])
  assert.deepEqual(calls, [
    'control:start',
    'environment:start',
    'recovery:worker exited unexpectedly',
    'environment:stop',
  ])
  assert.equal(await Promise.race([
    runtime.fatal.then(() => 'fatal'),
    new Promise(resolve => setTimeout(() => resolve('available'), 20)),
  ]), 'available')

  await runtime.restart('dsh-runtime')
  assert.equal(runtime.status().recoveryMode, null)
  await runtime.stop()
})

test('recovers an unexpected DSH exit without stopping the Control Plane', async () => {
  const calls = []
  let emitEnvironmentFatal
  const environmentRunner = {
    fatal: new Promise(() => {}),
    onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
    start: async () => { calls.push('environment:start') },
    stop: async () => { calls.push('environment:stop') },
    restart: async id => { calls.push(`environment:restart:${id}`); return { components: [{ id }] } },
    status: () => ({ environmentVersion: '1.0.3', components: [{ id: 'dsh-runtime' }] }),
  }
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}),
      start: async () => { calls.push('control:start') },
      stop: async () => { calls.push('control:stop') },
      status: () => ({ components: [{ id: 'gateway' }] }),
    },
    environment: environmentRunner,
    validateDeployment: async () => { calls.push('validate') },
    prepareDeployment: async () => { calls.push('prepare') },
    onEnvironmentVerified: async () => { calls.push('environment:verified') },
    onDshRecovered: async () => { calls.push('recovery:published') },
    onDshAvailable: async () => { calls.push('dsh:available') },
    recoveryDelaysMs: [0, 2_000, 5_000],
    sleep: async milliseconds => { calls.push(`delay:${String(milliseconds)}`) },
  })
  await runtime.start()
  calls.length = 0
  emitEnvironmentFatal(Object.assign(new Error('dsh-runtime exited unexpectedly'), { componentId: 'dsh-runtime' }))
  await runtime.recovery

  assert.deepEqual(calls, [
    'delay:0', 'validate', 'prepare', 'environment:restart:dsh-runtime',
    'environment:verified', 'recovery:published', 'dsh:available',
  ])
  assert.deepEqual(runtime.status().dshLifecycle, {
    state: 'running', action: null, taskId: null, attempt: 0, maxAttempts: 3,
    error: null, updatedAt: runtime.status().dshLifecycle.updatedAt,
  })
  assert.equal(runtime.status().recoveryMode, null)
  await runtime.stop()
})

test('requires exactly three bounded DSH recovery delays', () => {
  const runner = {
    fatal: new Promise(() => {}), onFatal: () => () => {},
    start: async () => {}, stop: async () => {}, status: () => ({ components: [] }),
  }
  assert.throws(() => new BootstrapRuntime({
    controlPlane: runner, environment: runner, recoveryDelaysMs: [0, 2_000],
  }), /three non-negative/)
  assert.throws(() => new BootstrapRuntime({
    controlPlane: runner, environment: runner, recoveryDelaysMs: [0, -1, 5_000],
  }), /three non-negative/)
})

test('replaces a SIGKILLed DSH process while preserving the Control Plane', async () => {
  const temp = await mkdtemp(join(tmpdir(), 'dsh-runtime-sigkill-recovery-'))
  const service = join(temp, 'service.mjs')
  await writeFile(service, 'setInterval(() => {}, 1000)')
  const environmentRunner = new EnvironmentRunner({
    environmentRoot: await environment([component('dsh-runtime', service, 'service')]),
    capture: () => {},
  })
  let controlStarts = 0
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}),
      start: async () => { controlStarts += 1 },
      stop: async () => {},
      status: () => ({ components: [{ id: 'gateway', pid: 100 }] }),
    },
    environment: environmentRunner,
    recoveryDelaysMs: [0, 0, 0],
  })
  await runtime.start()
  runtime.markStartupComplete()
  const firstPid = runtime.status().components.find(value => value.id === 'dsh-runtime').pid
  process.kill(firstPid, 'SIGKILL')

  let secondPid = firstPid
  for (let attempt = 0; attempt < 100; attempt += 1) {
    secondPid = runtime.status().components.find(value => value.id === 'dsh-runtime')?.pid
    if (runtime.status().dshLifecycle.state === 'running' && secondPid !== undefined && secondPid !== firstPid) break
    await new Promise(resolve => setTimeout(resolve, 10))
  }
  assert.notEqual(secondPid, firstPid)
  assert.equal(runtime.status().dshLifecycle.state, 'running')
  assert.equal(runtime.status().recoveryMode, null)
  assert.equal(controlStarts, 1)
  await runtime.stop()
})

test('does not restart DSH again when recovered-status publication fails', async () => {
  let emitEnvironmentFatal
  let restarts = 0
  let publications = 0
  const reports = []
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}), start: async () => {}, stop: async () => {},
      status: () => ({ components: [{ id: 'gateway' }] }),
    },
    environment: {
      fatal: new Promise(() => {}),
      onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
      start: async () => {}, stop: async () => {},
      restart: async () => { restarts += 1 },
      status: () => ({ environmentVersion: '1.0.3', components: [{ id: 'dsh-runtime' }] }),
    },
    onDshRecovered: async () => { throw new Error('status unavailable') },
    onDshAvailable: async () => { publications += 1 },
    recoveryDelaysMs: [0, 0, 0],
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await runtime.start()
  emitEnvironmentFatal(Object.assign(new Error('dsh-runtime exited'), { componentId: 'dsh-runtime' }))
  await runtime.recovery
  assert.equal(restarts, 1)
  assert.equal(publications, 0)
  assert.equal(runtime.status().dshLifecycle.state, 'running')
  assert.equal(reports.some(entry => entry.message === 'dsh.recovery-status.failed'), true)
  await runtime.stop()
})

test('limits unexpected DSH recovery to three attempts before recovery mode', async () => {
  const delays = []
  const reports = []
  let emitEnvironmentFatal
  let restarts = 0
  let stopped = 0
  let isolated = 0
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}), start: async () => {}, stop: async () => {},
      status: () => ({ components: [{ id: 'gateway' }] }),
    },
    environment: {
      fatal: new Promise(() => {}),
      onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
      start: async () => {},
      stop: async () => { stopped += 1 },
      restart: async () => { restarts += 1; throw new Error(`restart ${String(restarts)} failed`) },
      status: () => ({ environmentVersion: '1.0.3', components: [] }),
    },
    recoveryDelaysMs: [0, 2_000, 5_000],
    sleep: async milliseconds => { delays.push(milliseconds) },
    onEnvironmentFatal: async () => { isolated += 1 },
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await runtime.start()
  emitEnvironmentFatal(Object.assign(new Error('DSH repeatedly crashed'), { componentId: 'dsh-runtime' }))
  await runtime.recovery

  assert.deepEqual(delays, [0, 2_000, 5_000])
  assert.equal(restarts, 3)
  assert.equal(isolated, 1)
  assert.equal(stopped, 1)
  assert.equal(runtime.status().dshLifecycle.state, 'failed')
  assert.equal(runtime.status().dshLifecycle.attempt, 3)
  assert.equal(runtime.status().recoveryMode, 'DSH repeatedly crashed')
  assert.deepEqual(reports.filter(entry => entry.message === 'dsh.recovery.attempt.failed')
    .map(entry => entry.fields.attempt), [1, 2, 3])
  await runtime.stop()
})

test('defers unexpected DSH recovery while a Deployment transaction owns lifecycle', async () => {
  let emitEnvironmentFatal
  let restarts = 0
  let stops = 0
  const reports = []
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}), start: async () => {}, stop: async () => {},
      status: () => ({ components: [{ id: 'gateway' }] }),
    },
    environment: {
      fatal: new Promise(() => {}),
      onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
      start: async () => {},
      stop: async () => { stops += 1 },
      restart: async () => { restarts += 1 },
      status: () => ({ environmentVersion: '1.0.3', components: [] }),
    },
    ownsDshLifecycle: async () => true,
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await runtime.start()
  emitEnvironmentFatal(Object.assign(new Error('candidate exited'), { componentId: 'dsh-runtime' }))
  await runtime.recovery

  assert.equal(restarts, 0)
  assert.equal(stops, 0)
  assert.equal(runtime.status().dshLifecycle.state, 'recovering')
  assert.equal(runtime.status().dshLifecycle.attempt, 0)
  assert.equal(reports.some(entry => entry.message === 'dsh.recovery.deferred'), true)
  await runtime.stop()
})

test('enters recovery mode when an explicit DSH resume or restart fails', async () => {
  const environment = {
    fatal: new Promise(() => {}),
    onFatal: () => () => {},
    start: async () => {},
    stop: async () => {},
    resume: async () => { throw new Error('DSH resume failed') },
    restart: async () => { throw new Error('DSH restart failed') },
    status: () => ({ environmentVersion: '1.0.0', components: [] }),
  }
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}),
      start: async () => {},
      stop: async () => {},
      status: () => ({ components: [{ id: 'platform-management' }] }),
    },
    environment,
  })
  await runtime.start()
  await assert.rejects(runtime.resume('dsh-runtime'), /DSH resume failed/)
  assert.equal(runtime.status().recoveryMode, 'DSH resume failed')
  await assert.rejects(runtime.restart('dsh-runtime'), /DSH restart failed/)
  assert.equal(runtime.status().recoveryMode, 'DSH restart failed')
  await runtime.stop()
})

test('reports secondary failures while isolating an unexpected Environment exit', async () => {
  let emitEnvironmentFatal
  const reports = []
  const runtime = new BootstrapRuntime({
    controlPlane: {
      fatal: new Promise(() => {}),
      start: async () => {},
      stop: async () => {},
      status: () => ({ components: [] }),
    },
    environment: {
      fatal: new Promise(() => {}),
      onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
      start: async () => {},
      stop: async () => { throw new Error('environment cleanup failed') },
      status: () => ({ environmentVersion: 'env-1', components: [] }),
    },
    onEnvironmentFatal: async () => { throw new Error('status publication failed') },
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  await runtime.start()
  emitEnvironmentFatal(new Error('dsh-runtime crashed'))
  await runtime.recovery
  assert.equal(reports.some(entry => entry.message === 'environment.recovery-report.failed'
    && entry.fields.error.message === 'status publication failed'), true)
  assert.equal(reports.some(entry => entry.message === 'environment.recovery-stop.failed'
    && entry.fields.error.message === 'environment cleanup failed'), true)
})

test('validates mandatory Patches before every operation that starts DSH', async () => {
  const calls = []
  const service = name => ({
    fatal: new Promise(() => {}),
    start: async () => { calls.push(`${name}:start`) },
    stop: async () => { calls.push(`${name}:stop`) },
    reload: async () => { calls.push(`${name}:reload`) },
    suspend: async id => { calls.push(`${name}:suspend:${id}`) },
    resume: async id => { calls.push(`${name}:resume:${id}`) },
    restart: async id => { calls.push(`${name}:restart:${id}`) },
    health: async () => ({ healthy: true, components: [] }),
    status: () => ({ components: [] }),
  })
  let validations = 0
  const runtime = new BootstrapRuntime({
    controlPlane: service('control'),
    environment: service('environment'),
    validateDeployment: async () => {
      validations += 1
      calls.push('validate')
      if (validations === 1) throw new Error('Patch verification failed')
    },
  })
  await runtime.start({ onEnvironmentFailure: async () => true })
  await runtime.reload()
  await runtime.resume('dsh-runtime')
  await runtime.restart('dsh-runtime')
  await runtime.resume('gateway')
  assert.deepEqual(calls, [
    'control:start',
    'validate',
    'validate',
    'environment:start',
    'validate',
    'environment:reload',
    'validate',
    'environment:resume:dsh-runtime',
    'validate',
    'environment:restart:dsh-runtime',
    'environment:resume:gateway',
  ])
})

test('retries a previous Deployment without restarting the Control Plane', async () => {
  const calls = []
  let environmentStarts = 0
  const controlPlane = {
    fatal: new Promise(() => {}),
    start: async () => { calls.push('control:start') },
    stop: async () => { calls.push('control:stop') },
    status: () => ({ components: [] }),
  }
  const environmentRunner = {
    fatal: new Promise(() => {}),
    start: async () => {
      environmentStarts += 1
      calls.push(`environment:start:${String(environmentStarts)}`)
      if (environmentStarts === 1) throw new Error('candidate failed')
    },
    stop: async () => { calls.push('environment:stop') },
    status: () => ({ environmentVersion: 'test', components: [] }),
  }
  const runtime = new BootstrapRuntime({ controlPlane, environment: environmentRunner })
  await runtime.start({
    onEnvironmentFailure: async error => {
      assert.match(error.message, /candidate failed/)
      calls.push('deployment:restore')
      return true
    },
  })
  assert.deepEqual(calls, [
    'control:start',
    'environment:start:1',
    'deployment:restore',
    'environment:start:2',
  ])
})

test('keeps DSH stopped when mandatory Patch verification fails for current and previous', async () => {
  let validations = 0
  let environmentStarts = 0
  const controlPlane = {
    fatal: new Promise(() => {}),
    start: async () => {},
    stop: async () => {},
    status: () => ({ components: [{ id: 'gateway' }] }),
  }
  const environmentRunner = {
    fatal: new Promise(() => {}),
    start: async () => { environmentStarts += 1 },
    stop: async () => {},
    status: () => ({ environmentVersion: null, components: [] }),
  }
  const runtime = new BootstrapRuntime({
    controlPlane,
    environment: environmentRunner,
    validateDeployment: async () => {
      validations += 1
      throw new Error(`Patch verification failed for ${validations === 1 ? 'current' : 'previous'}`)
    },
  })
  await runtime.start({
    allowRecovery: true,
    onEnvironmentFailure: async error => {
      assert.match(error.message, /current/)
      return true
    },
  })
  assert.equal(validations, 2)
  assert.equal(environmentStarts, 0)
  assert.equal(runtime.status().recoveryMode, 'Deployment candidate and fallback both failed')
  assert.deepEqual(runtime.status().controlPlane, [{ id: 'gateway' }])
})

test('keeps the Control Plane alive in recovery mode when no Deployment starts', async () => {
  const calls = []
  const controlPlane = {
    fatal: new Promise(() => {}),
    start: async () => { calls.push('control:start') },
    stop: async () => { calls.push('control:stop') },
    status: () => ({ components: [{ id: 'gateway' }] }),
  }
  const environmentRunner = {
    fatal: new Promise(() => {}),
    start: async () => { throw new Error('Runtime missing') },
    stop: async () => {},
    status: () => ({ environmentVersion: null, components: [] }),
  }
  const runtime = new BootstrapRuntime({ controlPlane, environment: environmentRunner })
  await runtime.start({ allowRecovery: true, onEnvironmentFailure: async () => false })
  assert.equal(runtime.status().recoveryMode, 'Runtime missing')
  assert.deepEqual(runtime.status().controlPlane, [{ id: 'gateway' }])
  assert.deepEqual(calls, ['control:start'])
})

test('enters recovery mode when a failed candidate fallback cannot resolve', async () => {
  const calls = []
  const controlPlane = {
    fatal: new Promise(() => {}),
    start: async () => { calls.push('control:start') },
    stop: async () => { calls.push('control:stop') },
    status: () => ({ components: [{ id: 'gateway' }] }),
  }
  const environmentRunner = {
    fatal: new Promise(() => {}),
    start: async () => { throw new Error('candidate unhealthy') },
    stop: async () => {},
    status: () => ({ environmentVersion: null, components: [] }),
  }
  const runtime = new BootstrapRuntime({ controlPlane, environment: environmentRunner })
  await runtime.start({
    allowRecovery: true,
    onEnvironmentFailure: async () => { throw new Error('previous image is unavailable') },
  })
  assert.equal(runtime.status().recoveryMode, 'Deployment failed and its fallback could not be resolved')
  assert.deepEqual(calls, ['control:start'])
})