1319 lines
53 KiB
JavaScript
1319 lines
53 KiB
JavaScript
import assert from 'node:assert/strict'
|
|
import { createHash } from 'node:crypto'
|
|
import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
import { tmpdir } from 'node:os'
|
|
import { dirname, join } from 'node:path'
|
|
import { fileURLToPath } from 'node:url'
|
|
import test from 'node:test'
|
|
import { canonicalJson } from '../lib/canonical-json.mjs'
|
|
import { BootstrapRuntime } from '../bootstrap/lib/runtime.mjs'
|
|
import { EnvironmentRunner, loadControlPlane } from '../bootstrap/lib/lifecycle.mjs'
|
|
import { createBootstrapControl, listenBootstrapControl } from '../bootstrap/lib/control.mjs'
|
|
import { LocalApiClient } from '../../control-plane/modules/updater/lib/client.mjs'
|
|
|
|
function command(script, args = []) {
|
|
return { executable: process.execPath, args: [script, ...args], timeoutSeconds: 5 }
|
|
}
|
|
|
|
const emptyLifecycle = {
|
|
prepare: null,
|
|
preStart: null,
|
|
postStart: null,
|
|
preStop: null,
|
|
stop: null,
|
|
postStop: null,
|
|
}
|
|
|
|
async function environment(components) {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-environment-runner-'))
|
|
await mkdir(join(root, 'artifacts'))
|
|
const references = []
|
|
const artifacts = []
|
|
for (const component of components) {
|
|
const artifactId = `component-${component.id}`
|
|
const bytes = canonicalJson(component)
|
|
const sha256 = createHash('sha256').update(bytes).digest('hex')
|
|
await writeFile(join(root, 'artifacts', artifactId), bytes)
|
|
references.push({ id: component.id, sha256 })
|
|
artifacts.push({
|
|
id: artifactId,
|
|
mediaType: 'application/vnd.dsh-platform.component.v1+json',
|
|
sha256,
|
|
size: bytes.byteLength,
|
|
url: `https://release.example/${artifactId}`,
|
|
})
|
|
}
|
|
await writeFile(join(root, 'environment.manifest.json'), canonicalJson({
|
|
schema: 1,
|
|
manifestType: 'environment',
|
|
version: 'test',
|
|
keyringGeneration: 1,
|
|
targetSequence: 1,
|
|
issuedAt: '2026-08-19T00:00:00.000Z',
|
|
artifacts,
|
|
bootstrapApi: 1,
|
|
components: references,
|
|
patches: [],
|
|
systemPlugins: [],
|
|
}))
|
|
return root
|
|
}
|
|
|
|
function component(id, script, type = 'oneshot', lifecycle = emptyLifecycle) {
|
|
return {
|
|
schema: 1,
|
|
id,
|
|
type,
|
|
command: command(script),
|
|
environment: {},
|
|
lifecycle,
|
|
health: null,
|
|
logging: { stdout: true, stderr: true },
|
|
}
|
|
}
|
|
|
|
test('runs components in manifest order and stop phases in reverse order', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-order-'))
|
|
const log = join(temp, 'order.log')
|
|
const append = join(temp, 'append.mjs')
|
|
const service = join(temp, 'service.mjs')
|
|
const check = join(temp, 'check.mjs')
|
|
await writeFile(append, 'import { appendFileSync } from "node:fs"; appendFileSync(process.argv[2], process.argv[3] + "\\n")')
|
|
await writeFile(service, 'import { appendFileSync } from "node:fs"; appendFileSync(process.argv[2], process.argv[3] + "\\n"); setInterval(() => {}, 1000)')
|
|
await writeFile(check, 'import { readFileSync } from "node:fs"; process.exit(readFileSync(process.argv[2], "utf8").includes(process.argv[3]) ? 0 : 1)')
|
|
const lifecycle = label => ({
|
|
...emptyLifecycle,
|
|
prepare: command(append, [log, `${label}:prepare`]),
|
|
preStart: command(append, [log, `${label}:preStart`]),
|
|
postStart: command(append, [log, `${label}:postStart`]),
|
|
preStop: command(append, [log, `${label}:preStop`]),
|
|
postStop: command(append, [log, `${label}:postStop`]),
|
|
})
|
|
const first = component('first', service, 'service', lifecycle('first'))
|
|
first.command = command(service, [log, 'first:start'])
|
|
first.health = {
|
|
type: 'exec',
|
|
command: command(check, [log, 'first:start']),
|
|
intervalSeconds: 1,
|
|
timeoutSeconds: 5,
|
|
}
|
|
const second = component('second', append, 'oneshot', lifecycle('second'))
|
|
second.command = command(append, [log, 'second:start'])
|
|
const reports = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([first, second]),
|
|
capture: () => {},
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await runner.start()
|
|
await runner.stop()
|
|
const lines = (await readFile(log, 'utf8')).trim().split('\n')
|
|
assert.deepEqual(lines.slice(0, 8), [
|
|
'first:prepare', 'second:prepare', 'first:preStart', 'first:start',
|
|
'first:postStart', 'second:preStart', 'second:start', 'second:postStart',
|
|
])
|
|
assert.deepEqual(lines.slice(8), ['second:preStop', 'second:postStop', 'first:preStop', 'first:postStop'])
|
|
assert.deepEqual(reports.map(report => [report.message, report.fields.componentId]), [
|
|
['component.starting', 'first'],
|
|
['component.spawned', 'first'],
|
|
['component.ready', 'first'],
|
|
['component.starting', 'second'],
|
|
['component.ready', 'second'],
|
|
['component.stopping', 'second'],
|
|
['component.stopped', 'second'],
|
|
['component.stopping', 'first'],
|
|
['component.stopped', 'first'],
|
|
])
|
|
assert.equal(reports.find(report => report.message === 'component.ready').fields.elapsedMs >= 0, true)
|
|
})
|
|
|
|
test('starts explicit component groups concurrently and waits before the next group', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-groups-'))
|
|
const barrier = join(temp, 'barrier.mjs')
|
|
const after = join(temp, 'after.mjs')
|
|
const firstStarted = join(temp, 'first.started')
|
|
const secondStarted = join(temp, 'second.started')
|
|
const firstDone = join(temp, 'first.done')
|
|
const secondDone = join(temp, 'second.done')
|
|
await writeFile(barrier, `
|
|
import { existsSync, writeFileSync } from 'node:fs'
|
|
writeFileSync(process.argv[2], '')
|
|
while (!existsSync(process.argv[3])) await new Promise(resolve => setTimeout(resolve, 10))
|
|
writeFileSync(process.argv[4], '')
|
|
`)
|
|
await writeFile(after, `
|
|
import { existsSync } from 'node:fs'
|
|
process.exit(existsSync(process.argv[2]) && existsSync(process.argv[3]) ? 0 : 9)
|
|
`)
|
|
const first = component('first', barrier)
|
|
first.command = command(barrier, [firstStarted, secondStarted, firstDone])
|
|
const second = component('second', barrier)
|
|
second.command = command(barrier, [secondStarted, firstStarted, secondDone])
|
|
const final = component('final', after)
|
|
final.command = command(after, [firstDone, secondDone])
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([first, second, final]),
|
|
capture: () => {},
|
|
startupGroups: [['first', 'second'], ['final']],
|
|
})
|
|
await runner.start()
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['first', 'second', 'final'])
|
|
await runner.stop()
|
|
})
|
|
|
|
test('waits for a failed startup group before rolling back every started service', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-group-failure-'))
|
|
const service = join(temp, 'service.mjs')
|
|
const failure = join(temp, 'failure.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
await writeFile(failure, 'process.exit(9)')
|
|
const reports = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([
|
|
component('service', service, 'service'),
|
|
component('failure', failure),
|
|
]),
|
|
capture: () => {},
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
startupGroups: [['service', 'failure']],
|
|
})
|
|
await assert.rejects(runner.start(), /failure command failed/)
|
|
assert.deepEqual(runner.status().components, [])
|
|
assert.equal(reports.some(report => report.message === 'component.stopped'
|
|
&& report.fields.componentId === 'service'), true)
|
|
})
|
|
|
|
test('does not recover an isolated startup failure after its group fatally fails', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-group-recovery-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
const reports = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([
|
|
component('gateway', service, 'service'),
|
|
component('outbound-proxy', service, 'service'),
|
|
]),
|
|
capture: () => {},
|
|
recoverableComponents: ['outbound-proxy'],
|
|
recoveryDelaysMs: [0],
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
startupGroups: [['gateway', 'outbound-proxy']],
|
|
})
|
|
runner.startComponentUnlocked = candidate => Promise.reject(new Error(`${candidate.id} unavailable`))
|
|
await assert.rejects(runner.start(), /gateway unavailable/)
|
|
await new Promise(resolve => setImmediate(resolve))
|
|
assert.equal(reports.some(report => report.message === 'component.recovery.started'), false)
|
|
})
|
|
|
|
test('rejects incomplete or duplicated startup groups before component preparation', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-invalid-groups-'))
|
|
const marker = join(temp, 'prepared')
|
|
const append = join(temp, 'append.mjs')
|
|
await writeFile(append, `import { writeFileSync } from 'node:fs'; writeFileSync(process.argv[2], '')`)
|
|
const first = component('first', append)
|
|
first.lifecycle = { ...emptyLifecycle, prepare: command(append, [marker]) }
|
|
const root = await environment([first, component('second', append)])
|
|
const omitted = new EnvironmentRunner({
|
|
environmentRoot: root,
|
|
capture: () => {},
|
|
startupGroups: [['first']],
|
|
})
|
|
await assert.rejects(omitted.start(), /omit components: second/)
|
|
await assert.rejects(readFile(marker), error => error?.code === 'ENOENT')
|
|
const duplicated = new EnvironmentRunner({
|
|
environmentRoot: root,
|
|
capture: () => {},
|
|
startupGroups: [['first'], ['first', 'second']],
|
|
})
|
|
await assert.rejects(duplicated.start(), /component first is duplicated/)
|
|
})
|
|
|
|
test('exec health probes use only their exit status and do not emit component logs', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-health-output-'))
|
|
const service = join(temp, 'service.mjs')
|
|
const health = join(temp, 'health.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
await writeFile(health, 'console.log(JSON.stringify({ noisy: true }, null, 2))')
|
|
const candidate = component('service', service, 'service')
|
|
candidate.health = {
|
|
type: 'exec',
|
|
command: command(health),
|
|
intervalSeconds: 1,
|
|
timeoutSeconds: 5,
|
|
}
|
|
let captures = 0
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate]),
|
|
capture: () => { captures += 1 },
|
|
})
|
|
await runner.start()
|
|
assert.equal((await runner.health()).healthy, true)
|
|
assert.equal(captures, 1)
|
|
await runner.stop()
|
|
})
|
|
|
|
test('startup health retries do not inherit the steady one-second interval', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-health-retry-'))
|
|
const service = join(temp, 'service.mjs')
|
|
const health = join(temp, 'health.mjs')
|
|
const attempts = join(temp, 'attempts')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
await writeFile(health, `import { existsSync, writeFileSync } from 'node:fs'; if (!existsSync(process.argv[2])) { writeFileSync(process.argv[2], 'first'); process.exit(1) }`)
|
|
const candidate = component('service', service, 'service')
|
|
candidate.health = {
|
|
type: 'exec',
|
|
command: command(health, [attempts]),
|
|
intervalSeconds: 2,
|
|
timeoutSeconds: 5,
|
|
}
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate]),
|
|
capture: () => {},
|
|
})
|
|
const started = Date.now()
|
|
await runner.start()
|
|
const elapsedMs = Date.now() - started
|
|
assert.ok(elapsedMs < 1_000, `startup health retry took ${String(elapsedMs)} ms`)
|
|
await runner.stop()
|
|
})
|
|
|
|
test('component environment replaces an inherited root user environment', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-component-environment-'))
|
|
const output = join(temp, 'environment.json')
|
|
const inspect = join(temp, 'inspect.mjs')
|
|
await writeFile(inspect, `import { writeFileSync } from 'node:fs'; writeFileSync(process.argv[2], JSON.stringify({ HOME: process.env.HOME, USER: process.env.USER, LOGNAME: process.env.LOGNAME, XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME, XDG_CACHE_HOME: process.env.XDG_CACHE_HOME, XDG_DATA_HOME: process.env.XDG_DATA_HOME }))`)
|
|
const candidate = component('dsh-runtime', inspect)
|
|
candidate.command = command(inspect, [output])
|
|
candidate.environment = {
|
|
HOME: '/home/node',
|
|
USER: 'node',
|
|
LOGNAME: 'node',
|
|
XDG_CONFIG_HOME: '/home/node/.config',
|
|
XDG_CACHE_HOME: '/home/node/.cache',
|
|
XDG_DATA_HOME: '/home/node/.local/share',
|
|
}
|
|
const original = {
|
|
HOME: process.env.HOME,
|
|
USER: process.env.USER,
|
|
LOGNAME: process.env.LOGNAME,
|
|
XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME,
|
|
XDG_CACHE_HOME: process.env.XDG_CACHE_HOME,
|
|
XDG_DATA_HOME: process.env.XDG_DATA_HOME,
|
|
}
|
|
process.env.HOME = '/root'
|
|
process.env.USER = 'root'
|
|
process.env.LOGNAME = 'root'
|
|
process.env.XDG_CONFIG_HOME = '/root/.config'
|
|
process.env.XDG_CACHE_HOME = '/root/.cache'
|
|
process.env.XDG_DATA_HOME = '/root/.local/share'
|
|
try {
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate]),
|
|
capture: () => {},
|
|
})
|
|
await runner.start()
|
|
assert.deepEqual(JSON.parse(await readFile(output, 'utf8')), candidate.environment)
|
|
} finally {
|
|
for (const [name, value] of Object.entries(original)) {
|
|
if (value === undefined) delete process.env[name]
|
|
else process.env[name] = value
|
|
}
|
|
}
|
|
})
|
|
|
|
test('binds prepared service environment and cleanup to one process launch', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-prepared-service-'))
|
|
const output = join(temp, 'launch-token')
|
|
const service = join(temp, 'service.mjs')
|
|
const health = join(temp, 'health.mjs')
|
|
await writeFile(service, `import { writeFileSync } from 'node:fs'; writeFileSync(process.argv[2], process.env.DSH_PLATFORM_LAUNCH_TOKEN); setInterval(() => {}, 1000)`)
|
|
await writeFile(health, `import { existsSync } from 'node:fs'; process.exit(existsSync(process.argv[2]) ? 0 : 1)`)
|
|
let releases = 0
|
|
const candidate = component('dsh-runtime', service, 'service')
|
|
candidate.command = command(service, [output])
|
|
candidate.health = {
|
|
type: 'exec', command: command(health, [output]), intervalSeconds: 1, timeoutSeconds: 5,
|
|
}
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate]),
|
|
capture: () => {},
|
|
prepareService: async value => ({
|
|
environment: { DSH_PLATFORM_LAUNCH_TOKEN: `token-for-${value.id}` },
|
|
release: () => { releases += 1 },
|
|
}),
|
|
})
|
|
try {
|
|
await runner.start()
|
|
assert.equal(await readFile(output, 'utf8'), 'token-for-dsh-runtime')
|
|
assert.equal(releases, 0)
|
|
} finally {
|
|
await runner.stop()
|
|
}
|
|
assert.equal(releases, 1)
|
|
})
|
|
|
|
test('releases prepared service ownership when process spawn fails', async () => {
|
|
let releases = 0
|
|
const candidate = component('dsh-runtime', '/path/that/does/not/exist', 'service')
|
|
candidate.command.executable = '/path/that/does/not/exist'
|
|
candidate.command.args = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate]),
|
|
capture: () => {},
|
|
prepareService: async () => ({
|
|
environment: { DSH_PLATFORM_LAUNCH_TOKEN: 'unused-token' },
|
|
release: () => { releases += 1 },
|
|
}),
|
|
})
|
|
await assert.rejects(runner.start(), /ENOENT/)
|
|
assert.equal(releases, 1)
|
|
})
|
|
|
|
test('stops already-started services when a later component fails', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-failure-'))
|
|
const service = join(temp, 'service.mjs')
|
|
const failure = join(temp, 'failure.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
await writeFile(failure, 'process.exit(9)')
|
|
const reports = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([
|
|
component('service', service, 'service'),
|
|
component('failure', failure),
|
|
]),
|
|
capture: () => {},
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await assert.rejects(runner.start(), /failure command failed/)
|
|
assert.deepEqual(runner.status().components, [])
|
|
const failureReport = reports.find(report => report.message === 'component.start.failed').fields
|
|
assert.equal(failureReport.componentId, 'failure')
|
|
assert.equal(failureReport.elapsedMs >= 0, true)
|
|
assert.match(failureReport.error, /failure command failed.*code=9/)
|
|
assert.equal(failureReport.level, 'error')
|
|
})
|
|
|
|
test('reports a service exit after readiness as a fatal Bootstrap condition', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-runtime-exit-'))
|
|
const exits = join(temp, 'exits.mjs')
|
|
await writeFile(exits, 'setTimeout(() => process.exit(7), 20)')
|
|
const reports = []
|
|
const observed = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([component('service', exits, 'service')]),
|
|
capture: () => {},
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
runner.onFatal(error => observed.push(error))
|
|
await runner.start()
|
|
const error = await runner.fatal
|
|
assert.match(error.message, /service exited unexpectedly.*code=7/)
|
|
assert.equal(error.componentId, 'service')
|
|
assert.equal(error.exitCode, 7)
|
|
assert.equal(observed[0], error)
|
|
assert.equal(reports.find(report => report.message === 'component.exited').fields.level, 'error')
|
|
await runner.stop()
|
|
})
|
|
|
|
test('recovers an opted-in service exit without stopping sibling services', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-isolated-recovery-'))
|
|
const counter = join(temp, 'counter')
|
|
const recoverable = join(temp, 'recoverable.mjs')
|
|
const sibling = join(temp, 'sibling.mjs')
|
|
await writeFile(recoverable, `import { existsSync, writeFileSync } from 'node:fs'; if (!existsSync(process.argv[2])) { writeFileSync(process.argv[2], '1'); setTimeout(() => process.exit(7), 20) } else setInterval(() => {}, 1000)`)
|
|
await writeFile(sibling, 'setInterval(() => {}, 1000)')
|
|
const candidate = component('outbound-proxy', recoverable, 'service')
|
|
candidate.command = command(recoverable, [counter])
|
|
const reports = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate, component('management', sibling, 'service')]),
|
|
capture: () => {},
|
|
recoverableComponents: ['outbound-proxy'],
|
|
recoveryDelaysMs: [0, 0, 0],
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await runner.start()
|
|
const deadline = Date.now() + 2_000
|
|
while (!reports.some(report => report.message === 'component.recovery.completed')) {
|
|
if (Date.now() >= deadline) throw new Error('recoverable component did not restart')
|
|
await new Promise(resolve => setTimeout(resolve, 10))
|
|
}
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['outbound-proxy', 'management'])
|
|
assert.equal(reports.some(report => report.message === 'component.exited'), true)
|
|
assert.equal(await Promise.race([
|
|
runner.fatal.then(() => 'fatal'),
|
|
new Promise(resolve => setTimeout(() => resolve('isolated'), 20)),
|
|
]), 'isolated')
|
|
await runner.stop()
|
|
})
|
|
|
|
test('isolates a persistently failing opted-in component and starts later services', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-isolated-startup-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
const reports = []
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([
|
|
component('outbound-proxy', service, 'service'),
|
|
component('management', service, 'service'),
|
|
]),
|
|
capture: () => {},
|
|
recoverableComponents: ['outbound-proxy'],
|
|
recoveryDelaysMs: [0, 0, 0],
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
const start = runner.startComponentUnlocked.bind(runner)
|
|
runner.startComponentUnlocked = (candidate, prepare) => candidate.id === 'outbound-proxy'
|
|
? Promise.reject(new Error('proxy port conflict'))
|
|
: start(candidate, prepare)
|
|
await runner.start()
|
|
const deadline = Date.now() + 2_000
|
|
while (reports.filter(report => report.message === 'component.isolated').length < 2) {
|
|
if (Date.now() >= deadline) throw new Error('proxy failure was not isolated')
|
|
await new Promise(resolve => setTimeout(resolve, 10))
|
|
}
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['management'])
|
|
assert.equal(reports.filter(report => report.message === 'component.recovery.failed').length, 3)
|
|
await runner.stop()
|
|
})
|
|
|
|
test('contains a service exit during readiness inside the startup operation', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-startup-exit-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'process.exit(1)')
|
|
const candidate = component('candidate', service, 'service')
|
|
candidate.health = {
|
|
type: 'http', host: '127.0.0.1', port: 1, path: '/', intervalSeconds: 1, timeoutSeconds: 1,
|
|
}
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([candidate]),
|
|
capture: () => {},
|
|
})
|
|
await assert.rejects(runner.start(), /exited before health check passed/)
|
|
const outcome = await Promise.race([
|
|
runner.fatal.then(() => 'fatal'),
|
|
new Promise(resolve => setTimeout(() => resolve('contained'), 20)),
|
|
])
|
|
assert.equal(outcome, 'contained')
|
|
})
|
|
|
|
test('suspends, resumes, and restarts one service while keeping other Environment components running', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-suspend-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([
|
|
component('dsh-runtime', service, 'service'),
|
|
component('platform-management', service, 'service'),
|
|
]),
|
|
capture: () => {},
|
|
})
|
|
await runner.start()
|
|
await runner.suspend('dsh-runtime')
|
|
await runner.pause('dsh-runtime')
|
|
await runner.pause('dsh-runtime')
|
|
await assert.rejects(runner.pause('unknown-service'), /does not exist/)
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['platform-management'])
|
|
assert.equal((await runner.health()).healthy, false)
|
|
await runner.resume('dsh-runtime')
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['dsh-runtime', 'platform-management'])
|
|
const managementPid = runner.status().components.find(value => value.id === 'platform-management').pid
|
|
const firstDshPid = runner.status().components.find(value => value.id === 'dsh-runtime').pid
|
|
await runner.restart('dsh-runtime')
|
|
assert.notEqual(runner.status().components.find(value => value.id === 'dsh-runtime').pid, firstDshPid)
|
|
assert.equal(runner.status().components.find(value => value.id === 'platform-management').pid, managementPid)
|
|
assert.equal((await runner.health()).healthy, true)
|
|
assert.equal(await Promise.race([
|
|
runner.fatal.then(() => 'fatal'),
|
|
new Promise(resolve => setTimeout(() => resolve('intentional'), 20)),
|
|
]), 'intentional')
|
|
await runner.stop()
|
|
})
|
|
|
|
test('retries a service restart after its first start attempt fails', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-restart-retry-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([component('dsh-runtime', service, 'service')]),
|
|
capture: () => {},
|
|
})
|
|
await runner.start()
|
|
const startComponent = runner.startComponentUnlocked.bind(runner)
|
|
let failStart = true
|
|
runner.startComponentUnlocked = async (...args) => {
|
|
if (failStart) {
|
|
failStart = false
|
|
throw new Error('restart failed')
|
|
}
|
|
return startComponent(...args)
|
|
}
|
|
await assert.rejects(runner.restart('dsh-runtime'), /restart failed/)
|
|
assert.deepEqual(runner.status().components, [])
|
|
await runner.restart('dsh-runtime')
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['dsh-runtime'])
|
|
await runner.stop()
|
|
})
|
|
|
|
test('restores a service after a transactional restart candidate fails', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-lifecycle-restart-recovery-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
const runner = new EnvironmentRunner({
|
|
environmentRoot: await environment([component('dsh-runtime', service, 'service')]),
|
|
capture: () => {},
|
|
})
|
|
await runner.start()
|
|
const startComponent = runner.startComponentUnlocked.bind(runner)
|
|
let candidate = true
|
|
runner.startComponentUnlocked = async (...args) => {
|
|
if (candidate) {
|
|
candidate = false
|
|
throw new Error('candidate failed')
|
|
}
|
|
return startComponent(...args)
|
|
}
|
|
const changes = []
|
|
await assert.rejects(runner.restart('dsh-runtime', {
|
|
recoverStart: true,
|
|
beforeStart: async () => { changes.push('activate') },
|
|
onStartFailure: async () => { changes.push('rollback') },
|
|
}), /candidate failed/)
|
|
assert.deepEqual(changes, ['activate', 'rollback'])
|
|
assert.deepEqual(runner.status().components.map(value => value.id), ['dsh-runtime'])
|
|
await runner.stop()
|
|
})
|
|
|
|
test('Bootstrap control socket exposes component suspension, resumption, restart, and health', async () => {
|
|
const root = await mkdtemp(join(tmpdir(), 'dsh-bootstrap-control-'))
|
|
const calls = []
|
|
const runner = {
|
|
status: () => ({ components: [] }),
|
|
reload: async () => ({}),
|
|
health: async () => ({ healthy: true, components: [] }),
|
|
suspend: async id => { calls.push(['suspend', id]); return {} },
|
|
pause: async id => { calls.push(['pause', id]); return {} },
|
|
resume: async id => { calls.push(['resume', id]); return {} },
|
|
restart: async (id, options = {}) => {
|
|
calls.push(['restart', id])
|
|
try {
|
|
await options.beforeStart?.()
|
|
return {}
|
|
} catch (error) {
|
|
await options.onStartFailure?.(error)
|
|
if (options.recoverStart) calls.push(['restart-recovered', id])
|
|
throw error
|
|
}
|
|
},
|
|
record: async (message, fields) => { calls.push(['report', message, fields.pathname ?? null]) },
|
|
}
|
|
const deployments = {
|
|
exclusive: operation => operation(),
|
|
setOperation: async operation => { calls.push(['operation', operation]) },
|
|
publishStatus: async options => { calls.push(['status', 'published', options ?? null]) },
|
|
resetCurrentRuntime: async ({ pauseDsh, restartDsh }) => {
|
|
calls.push(['runtime-reset'])
|
|
await pauseDsh()
|
|
await restartDsh()
|
|
return { recordId: 'repaired' }
|
|
},
|
|
}
|
|
const systemPlugins = {
|
|
prepare: async () => {
|
|
calls.push(['prepare-system-plugins'])
|
|
return {
|
|
activate: async () => { calls.push(['activate-system-plugins']) },
|
|
commit: async () => { calls.push(['commit-system-plugins']) },
|
|
rollback: async () => { calls.push(['rollback-system-plugins']) },
|
|
}
|
|
},
|
|
list: async () => [{ id: 'platform-management', installed: true }],
|
|
configure: async (id, action) => {
|
|
calls.push(['configure', id, action])
|
|
return [{ id, installed: true }]
|
|
},
|
|
recover: async (id, action) => {
|
|
calls.push(['recover', id, action])
|
|
return [{ id, installed: false }]
|
|
},
|
|
discard: async () => {
|
|
calls.push(['discard-system-plugins'])
|
|
return [{ id: 'platform-management', installed: true }]
|
|
},
|
|
}
|
|
const server = createBootstrapControl(runner, { deployments, systemPlugins })
|
|
const socket = join(root, 'run', 'bootstrap.sock')
|
|
await listenBootstrapControl(server, socket)
|
|
const client = new LocalApiClient(socket)
|
|
try {
|
|
assert.equal((await client.request('GET', '/v1/health')).healthy, true)
|
|
assert.deepEqual((await client.request('GET', '/v1/system-plugins')).plugins, [{
|
|
id: 'platform-management', installed: true,
|
|
}])
|
|
assert.deepEqual((await client.request('POST', '/v1/system-plugins/action', {
|
|
id: 'diagnostics', action: 'disable',
|
|
})).plugins, [{ id: 'diagnostics', installed: true }])
|
|
assert.deepEqual((await client.request('POST', '/v1/system-plugins/recovery-action', {
|
|
id: 'platform-management', action: 'uninstall',
|
|
})).plugins, [{ id: 'platform-management', installed: false }])
|
|
assert.deepEqual((await client.request('POST', '/v1/system-plugins/discard')).plugins, [{
|
|
id: 'platform-management', installed: true,
|
|
}])
|
|
await client.request('POST', '/v1/components/dsh-runtime/suspend')
|
|
await client.request('POST', '/v1/components/dsh-runtime/pause')
|
|
await client.request('POST', '/v1/components/dsh-runtime/resume')
|
|
await client.request('POST', '/v1/components/dsh-runtime/restart')
|
|
assert.deepEqual(calls, [
|
|
['configure', 'diagnostics', 'disable'],
|
|
['recover', 'platform-management', 'uninstall'],
|
|
['discard-system-plugins'],
|
|
['suspend', 'dsh-runtime'],
|
|
['pause', 'dsh-runtime'],
|
|
['operation', 'starting'],
|
|
['resume', 'dsh-runtime'],
|
|
['status', 'published', null],
|
|
['operation', 'restarting'],
|
|
['prepare-system-plugins'],
|
|
['restart', 'dsh-runtime'],
|
|
['activate-system-plugins'],
|
|
['commit-system-plugins'],
|
|
['status', 'published', null],
|
|
])
|
|
assert.equal((await client.request('POST', '/v1/deployments/runtime/reset')).recordId, 'repaired')
|
|
assert.deepEqual(calls.slice(-3), [
|
|
['runtime-reset'],
|
|
['pause', 'dsh-runtime'],
|
|
['restart', 'dsh-runtime'],
|
|
])
|
|
systemPlugins.prepare = async () => {
|
|
calls.push(['prepare-system-plugins-failed'])
|
|
return {
|
|
activate: async () => {
|
|
calls.push(['activate-system-plugins-failed'])
|
|
throw new Error('overlay failed')
|
|
},
|
|
rollback: async () => { calls.push(['rollback-system-plugins']) },
|
|
}
|
|
}
|
|
await assert.rejects(client.request('POST', '/v1/components/dsh-runtime/restart'), /overlay failed/)
|
|
assert.deepEqual(calls.slice(-8), [
|
|
['operation', 'restarting'],
|
|
['prepare-system-plugins-failed'],
|
|
['restart', 'dsh-runtime'],
|
|
['activate-system-plugins-failed'],
|
|
['rollback-system-plugins'],
|
|
['restart-recovered', 'dsh-runtime'],
|
|
['status', 'published', { operation: 'restart-failed', recoveryMode: 'overlay failed' }],
|
|
['report', 'bootstrap.request.failed', '/v1/components/dsh-runtime/restart'],
|
|
])
|
|
|
|
runner.resume = async id => {
|
|
calls.push(['resume-failed', id])
|
|
throw new Error('resume remained unavailable')
|
|
}
|
|
runner.status = () => ({ components: [], recoveryMode: 'resume remained unavailable' })
|
|
await assert.rejects(
|
|
client.request('POST', '/v1/components/dsh-runtime/resume'),
|
|
/resume remained unavailable/,
|
|
)
|
|
assert.deepEqual(calls.slice(-4), [
|
|
['operation', 'starting'],
|
|
['resume-failed', 'dsh-runtime'],
|
|
['status', 'published', { recoveryMode: 'resume remained unavailable' }],
|
|
['report', 'bootstrap.request.failed', '/v1/components/dsh-runtime/resume'],
|
|
])
|
|
} finally {
|
|
await new Promise(resolve => server.close(resolve))
|
|
}
|
|
})
|
|
|
|
test('loads the checked-in Control Plane independently from an Environment manifest', async () => {
|
|
const controlPlane = await loadControlPlane(join(
|
|
dirname(dirname(fileURLToPath(import.meta.url))),
|
|
'..',
|
|
'control-plane',
|
|
))
|
|
assert.equal(controlPlane.manifest.version, null)
|
|
assert.deepEqual(controlPlane.components.map(component => component.id), [
|
|
'gateway', 'access-manager', 'outbound-proxy', 'platform-recovery', 'platform-management',
|
|
])
|
|
})
|
|
|
|
test('keeps the Control Plane running while Environment operations replace DSH', async () => {
|
|
const calls = []
|
|
const runner = (name, status) => ({
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push(`${name}:start`) },
|
|
stop: async () => { calls.push(`${name}:stop`) },
|
|
reload: async () => { calls.push(`${name}:reload`); return status },
|
|
suspend: async id => { calls.push(`${name}:suspend:${id}`); return status },
|
|
pause: async id => { calls.push(`${name}:pause:${id}`); return status },
|
|
resume: async id => { calls.push(`${name}:resume:${id}`); return status },
|
|
restart: async id => { calls.push(`${name}:restart:${id}`); return status },
|
|
health: async () => ({ healthy: true, components: status.components }),
|
|
status: () => status,
|
|
})
|
|
const controlPlane = runner('control', {
|
|
environmentVersion: null,
|
|
components: [{ id: 'gateway' }],
|
|
})
|
|
const environmentRunner = runner('environment', {
|
|
environmentVersion: '1.0.0',
|
|
components: [{ id: 'dsh-runtime' }],
|
|
})
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane,
|
|
environment: environmentRunner,
|
|
onEnvironmentVerified: async () => { calls.push('environment:verified') },
|
|
onDshAvailable: async () => { calls.push('dsh:available') },
|
|
})
|
|
|
|
await runtime.start()
|
|
await runtime.suspend('dsh-runtime')
|
|
assert.equal(runtime.status().dshLifecycle.state, 'stopped')
|
|
await runtime.reload()
|
|
assert.equal(runtime.status().dshLifecycle.state, 'running')
|
|
await runtime.pause('dsh-runtime')
|
|
await runtime.resume('dsh-runtime')
|
|
assert.equal(runtime.status().dshLifecycle.state, 'running')
|
|
await runtime.restart('dsh-runtime')
|
|
await runtime.reload()
|
|
assert.equal((await runtime.health()).healthy, true)
|
|
assert.deepEqual(runtime.status().controlPlane, [{ id: 'gateway' }])
|
|
await runtime.stop()
|
|
|
|
assert.deepEqual(calls, [
|
|
'control:start',
|
|
'environment:start',
|
|
'environment:verified',
|
|
'environment:suspend:dsh-runtime',
|
|
'environment:reload',
|
|
'environment:verified',
|
|
'dsh:available',
|
|
'environment:pause:dsh-runtime',
|
|
'environment:resume:dsh-runtime',
|
|
'environment:verified',
|
|
'dsh:available',
|
|
'environment:restart:dsh-runtime',
|
|
'environment:verified',
|
|
'dsh:available',
|
|
'environment:reload',
|
|
'environment:verified',
|
|
'dsh:available',
|
|
'environment:stop',
|
|
'control:stop',
|
|
])
|
|
})
|
|
|
|
test('starts DSH alongside the Control Plane but gates readiness on access classification', async () => {
|
|
const calls = []
|
|
let resolveEnvironmentStarted
|
|
const environmentStarted = new Promise(resolve => { resolveEnvironmentStarted = resolve })
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('control:start'); await environmentStarted },
|
|
stop: async () => { calls.push('control:stop') },
|
|
status: () => ({ components: [{ id: 'access-manager' }] }),
|
|
},
|
|
environment: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('environment:start'); resolveEnvironmentStarted() },
|
|
stop: async () => { calls.push('environment:stop') },
|
|
status: () => ({ environmentVersion: '1.0.0', components: [{ id: 'dsh-runtime' }] }),
|
|
},
|
|
beforeReady: async () => { calls.push('access:classify') },
|
|
})
|
|
|
|
await runtime.start()
|
|
assert.deepEqual(calls, ['control:start', 'environment:start', 'access:classify'])
|
|
await runtime.stop()
|
|
|
|
const blocked = []
|
|
let resolveBlockedEnvironmentStarted
|
|
const blockedEnvironmentStarted = new Promise(resolve => { resolveBlockedEnvironmentStarted = resolve })
|
|
const failure = new Error('access classification failed')
|
|
const blockedRuntime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { blocked.push('control:start'); await blockedEnvironmentStarted },
|
|
stop: async () => { blocked.push('control:stop') },
|
|
status: () => ({ components: [{ id: 'access-manager' }] }),
|
|
},
|
|
environment: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { blocked.push('environment:start'); resolveBlockedEnvironmentStarted() },
|
|
stop: async () => { blocked.push('environment:stop') },
|
|
status: () => ({ environmentVersion: '1.0.0', components: [] }),
|
|
},
|
|
beforeReady: async () => { blocked.push('access:classify'); throw failure },
|
|
})
|
|
|
|
await assert.rejects(blockedRuntime.start(), error => error === failure)
|
|
assert.deepEqual(blocked, [
|
|
'control:start', 'environment:start', 'access:classify', 'environment:stop', 'control:stop',
|
|
])
|
|
})
|
|
|
|
test('isolates a non-DSH Environment exit and keeps the Control Plane available', async () => {
|
|
const calls = []
|
|
let emitEnvironmentFatal
|
|
const controlPlane = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('control:start') },
|
|
stop: async () => { calls.push('control:stop') },
|
|
status: () => ({ components: [{ id: 'gateway' }, { id: 'platform-management' }] }),
|
|
}
|
|
const environmentRunner = {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: listener => {
|
|
emitEnvironmentFatal = listener
|
|
return () => { emitEnvironmentFatal = undefined }
|
|
},
|
|
start: async () => { calls.push('environment:start') },
|
|
stop: async () => { calls.push('environment:stop') },
|
|
restart: async id => { calls.push(`environment:restart:${id}`); return { components: [{ id }] } },
|
|
status: () => ({ environmentVersion: '1.0.0', components: [] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane,
|
|
environment: environmentRunner,
|
|
onEnvironmentFatal: async error => { calls.push(`recovery:${error.message}`) },
|
|
})
|
|
await runtime.start()
|
|
const failure = Object.assign(new Error('worker exited unexpectedly'), { componentId: 'worker' })
|
|
emitEnvironmentFatal(failure)
|
|
await runtime.recovery
|
|
|
|
assert.equal(runtime.status().recoveryMode, 'worker exited unexpectedly')
|
|
assert.deepEqual(runtime.status().controlPlane.map(component => component.id), ['gateway', 'platform-management'])
|
|
assert.deepEqual(calls, [
|
|
'control:start',
|
|
'environment:start',
|
|
'recovery:worker exited unexpectedly',
|
|
'environment:stop',
|
|
])
|
|
assert.equal(await Promise.race([
|
|
runtime.fatal.then(() => 'fatal'),
|
|
new Promise(resolve => setTimeout(() => resolve('available'), 20)),
|
|
]), 'available')
|
|
|
|
await runtime.restart('dsh-runtime')
|
|
assert.equal(runtime.status().recoveryMode, null)
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('recovers an unexpected DSH exit without stopping the Control Plane', async () => {
|
|
const calls = []
|
|
let emitEnvironmentFatal
|
|
const environmentRunner = {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
|
|
start: async () => { calls.push('environment:start') },
|
|
stop: async () => { calls.push('environment:stop') },
|
|
restart: async id => { calls.push(`environment:restart:${id}`); return { components: [{ id }] } },
|
|
status: () => ({ environmentVersion: '1.0.3', components: [{ id: 'dsh-runtime' }] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('control:start') },
|
|
stop: async () => { calls.push('control:stop') },
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
},
|
|
environment: environmentRunner,
|
|
validateDeployment: async () => { calls.push('validate') },
|
|
prepareDeployment: async () => { calls.push('prepare') },
|
|
onEnvironmentVerified: async () => { calls.push('environment:verified') },
|
|
onDshRecovered: async () => { calls.push('recovery:published') },
|
|
onDshAvailable: async () => { calls.push('dsh:available') },
|
|
recoveryDelaysMs: [0, 2_000, 5_000],
|
|
sleep: async milliseconds => { calls.push(`delay:${String(milliseconds)}`) },
|
|
})
|
|
await runtime.start()
|
|
calls.length = 0
|
|
emitEnvironmentFatal(Object.assign(new Error('dsh-runtime exited unexpectedly'), { componentId: 'dsh-runtime' }))
|
|
await runtime.recovery
|
|
|
|
assert.deepEqual(calls, [
|
|
'delay:0', 'validate', 'prepare', 'environment:restart:dsh-runtime',
|
|
'environment:verified', 'recovery:published', 'dsh:available',
|
|
])
|
|
assert.deepEqual(runtime.status().dshLifecycle, {
|
|
state: 'running', action: null, taskId: null, attempt: 0, maxAttempts: 3,
|
|
error: null, updatedAt: runtime.status().dshLifecycle.updatedAt,
|
|
})
|
|
assert.equal(runtime.status().recoveryMode, null)
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('requires exactly three bounded DSH recovery delays', () => {
|
|
const runner = {
|
|
fatal: new Promise(() => {}), onFatal: () => () => {},
|
|
start: async () => {}, stop: async () => {}, status: () => ({ components: [] }),
|
|
}
|
|
assert.throws(() => new BootstrapRuntime({
|
|
controlPlane: runner, environment: runner, recoveryDelaysMs: [0, 2_000],
|
|
}), /three non-negative/)
|
|
assert.throws(() => new BootstrapRuntime({
|
|
controlPlane: runner, environment: runner, recoveryDelaysMs: [0, -1, 5_000],
|
|
}), /three non-negative/)
|
|
})
|
|
|
|
test('replaces a SIGKILLed DSH process while preserving the Control Plane', async () => {
|
|
const temp = await mkdtemp(join(tmpdir(), 'dsh-runtime-sigkill-recovery-'))
|
|
const service = join(temp, 'service.mjs')
|
|
await writeFile(service, 'setInterval(() => {}, 1000)')
|
|
const environmentRunner = new EnvironmentRunner({
|
|
environmentRoot: await environment([component('dsh-runtime', service, 'service')]),
|
|
capture: () => {},
|
|
})
|
|
let controlStarts = 0
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { controlStarts += 1 },
|
|
stop: async () => {},
|
|
status: () => ({ components: [{ id: 'gateway', pid: 100 }] }),
|
|
},
|
|
environment: environmentRunner,
|
|
recoveryDelaysMs: [0, 0, 0],
|
|
})
|
|
await runtime.start()
|
|
runtime.markStartupComplete()
|
|
const firstPid = runtime.status().components.find(value => value.id === 'dsh-runtime').pid
|
|
process.kill(firstPid, 'SIGKILL')
|
|
|
|
let secondPid = firstPid
|
|
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
secondPid = runtime.status().components.find(value => value.id === 'dsh-runtime')?.pid
|
|
if (runtime.status().dshLifecycle.state === 'running' && secondPid !== undefined && secondPid !== firstPid) break
|
|
await new Promise(resolve => setTimeout(resolve, 10))
|
|
}
|
|
assert.notEqual(secondPid, firstPid)
|
|
assert.equal(runtime.status().dshLifecycle.state, 'running')
|
|
assert.equal(runtime.status().recoveryMode, null)
|
|
assert.equal(controlStarts, 1)
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('does not restart DSH again when recovered-status publication fails', async () => {
|
|
let emitEnvironmentFatal
|
|
let restarts = 0
|
|
let publications = 0
|
|
const reports = []
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}), start: async () => {}, stop: async () => {},
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
},
|
|
environment: {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
|
|
start: async () => {}, stop: async () => {},
|
|
restart: async () => { restarts += 1 },
|
|
status: () => ({ environmentVersion: '1.0.3', components: [{ id: 'dsh-runtime' }] }),
|
|
},
|
|
onDshRecovered: async () => { throw new Error('status unavailable') },
|
|
onDshAvailable: async () => { publications += 1 },
|
|
recoveryDelaysMs: [0, 0, 0],
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await runtime.start()
|
|
emitEnvironmentFatal(Object.assign(new Error('dsh-runtime exited'), { componentId: 'dsh-runtime' }))
|
|
await runtime.recovery
|
|
assert.equal(restarts, 1)
|
|
assert.equal(publications, 0)
|
|
assert.equal(runtime.status().dshLifecycle.state, 'running')
|
|
assert.equal(reports.some(entry => entry.message === 'dsh.recovery-status.failed'), true)
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('limits unexpected DSH recovery to three attempts before recovery mode', async () => {
|
|
const delays = []
|
|
const reports = []
|
|
let emitEnvironmentFatal
|
|
let restarts = 0
|
|
let stopped = 0
|
|
let isolated = 0
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}), start: async () => {}, stop: async () => {},
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
},
|
|
environment: {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
|
|
start: async () => {},
|
|
stop: async () => { stopped += 1 },
|
|
restart: async () => { restarts += 1; throw new Error(`restart ${String(restarts)} failed`) },
|
|
status: () => ({ environmentVersion: '1.0.3', components: [] }),
|
|
},
|
|
recoveryDelaysMs: [0, 2_000, 5_000],
|
|
sleep: async milliseconds => { delays.push(milliseconds) },
|
|
onEnvironmentFatal: async () => { isolated += 1 },
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await runtime.start()
|
|
emitEnvironmentFatal(Object.assign(new Error('DSH repeatedly crashed'), { componentId: 'dsh-runtime' }))
|
|
await runtime.recovery
|
|
|
|
assert.deepEqual(delays, [0, 2_000, 5_000])
|
|
assert.equal(restarts, 3)
|
|
assert.equal(isolated, 1)
|
|
assert.equal(stopped, 1)
|
|
assert.equal(runtime.status().dshLifecycle.state, 'failed')
|
|
assert.equal(runtime.status().dshLifecycle.attempt, 3)
|
|
assert.equal(runtime.status().recoveryMode, 'DSH repeatedly crashed')
|
|
assert.deepEqual(reports.filter(entry => entry.message === 'dsh.recovery.attempt.failed')
|
|
.map(entry => entry.fields.attempt), [1, 2, 3])
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('defers unexpected DSH recovery while a Deployment transaction owns lifecycle', async () => {
|
|
let emitEnvironmentFatal
|
|
let restarts = 0
|
|
let stops = 0
|
|
const reports = []
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}), start: async () => {}, stop: async () => {},
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
},
|
|
environment: {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
|
|
start: async () => {},
|
|
stop: async () => { stops += 1 },
|
|
restart: async () => { restarts += 1 },
|
|
status: () => ({ environmentVersion: '1.0.3', components: [] }),
|
|
},
|
|
ownsDshLifecycle: async () => true,
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await runtime.start()
|
|
emitEnvironmentFatal(Object.assign(new Error('candidate exited'), { componentId: 'dsh-runtime' }))
|
|
await runtime.recovery
|
|
|
|
assert.equal(restarts, 0)
|
|
assert.equal(stops, 0)
|
|
assert.equal(runtime.status().dshLifecycle.state, 'recovering')
|
|
assert.equal(runtime.status().dshLifecycle.attempt, 0)
|
|
assert.equal(reports.some(entry => entry.message === 'dsh.recovery.deferred'), true)
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('enters recovery mode when an explicit DSH resume or restart fails', async () => {
|
|
const environment = {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: () => () => {},
|
|
start: async () => {},
|
|
stop: async () => {},
|
|
resume: async () => { throw new Error('DSH resume failed') },
|
|
restart: async () => { throw new Error('DSH restart failed') },
|
|
status: () => ({ environmentVersion: '1.0.0', components: [] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => {},
|
|
stop: async () => {},
|
|
status: () => ({ components: [{ id: 'platform-management' }] }),
|
|
},
|
|
environment,
|
|
})
|
|
await runtime.start()
|
|
await assert.rejects(runtime.resume('dsh-runtime'), /DSH resume failed/)
|
|
assert.equal(runtime.status().recoveryMode, 'DSH resume failed')
|
|
await assert.rejects(runtime.restart('dsh-runtime'), /DSH restart failed/)
|
|
assert.equal(runtime.status().recoveryMode, 'DSH restart failed')
|
|
await runtime.stop()
|
|
})
|
|
|
|
test('reports secondary failures while isolating an unexpected Environment exit', async () => {
|
|
let emitEnvironmentFatal
|
|
const reports = []
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => {},
|
|
stop: async () => {},
|
|
status: () => ({ components: [] }),
|
|
},
|
|
environment: {
|
|
fatal: new Promise(() => {}),
|
|
onFatal: listener => { emitEnvironmentFatal = listener; return () => {} },
|
|
start: async () => {},
|
|
stop: async () => { throw new Error('environment cleanup failed') },
|
|
status: () => ({ environmentVersion: 'env-1', components: [] }),
|
|
},
|
|
onEnvironmentFatal: async () => { throw new Error('status publication failed') },
|
|
report: (message, fields) => { reports.push({ message, fields }) },
|
|
})
|
|
await runtime.start()
|
|
emitEnvironmentFatal(new Error('dsh-runtime crashed'))
|
|
await runtime.recovery
|
|
assert.equal(reports.some(entry => entry.message === 'environment.recovery-report.failed'
|
|
&& entry.fields.error.message === 'status publication failed'), true)
|
|
assert.equal(reports.some(entry => entry.message === 'environment.recovery-stop.failed'
|
|
&& entry.fields.error.message === 'environment cleanup failed'), true)
|
|
})
|
|
|
|
test('validates mandatory Patches before every operation that starts DSH', async () => {
|
|
const calls = []
|
|
const service = name => ({
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push(`${name}:start`) },
|
|
stop: async () => { calls.push(`${name}:stop`) },
|
|
reload: async () => { calls.push(`${name}:reload`) },
|
|
suspend: async id => { calls.push(`${name}:suspend:${id}`) },
|
|
resume: async id => { calls.push(`${name}:resume:${id}`) },
|
|
restart: async id => { calls.push(`${name}:restart:${id}`) },
|
|
health: async () => ({ healthy: true, components: [] }),
|
|
status: () => ({ components: [] }),
|
|
})
|
|
let validations = 0
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane: service('control'),
|
|
environment: service('environment'),
|
|
validateDeployment: async () => {
|
|
validations += 1
|
|
calls.push('validate')
|
|
if (validations === 1) throw new Error('Patch verification failed')
|
|
},
|
|
})
|
|
await runtime.start({ onEnvironmentFailure: async () => true })
|
|
await runtime.reload()
|
|
await runtime.resume('dsh-runtime')
|
|
await runtime.restart('dsh-runtime')
|
|
await runtime.resume('gateway')
|
|
assert.deepEqual(calls, [
|
|
'control:start',
|
|
'validate',
|
|
'validate',
|
|
'environment:start',
|
|
'validate',
|
|
'environment:reload',
|
|
'validate',
|
|
'environment:resume:dsh-runtime',
|
|
'validate',
|
|
'environment:restart:dsh-runtime',
|
|
'environment:resume:gateway',
|
|
])
|
|
})
|
|
|
|
test('retries a previous Deployment without restarting the Control Plane', async () => {
|
|
const calls = []
|
|
let environmentStarts = 0
|
|
const controlPlane = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('control:start') },
|
|
stop: async () => { calls.push('control:stop') },
|
|
status: () => ({ components: [] }),
|
|
}
|
|
const environmentRunner = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => {
|
|
environmentStarts += 1
|
|
calls.push(`environment:start:${String(environmentStarts)}`)
|
|
if (environmentStarts === 1) throw new Error('candidate failed')
|
|
},
|
|
stop: async () => { calls.push('environment:stop') },
|
|
status: () => ({ environmentVersion: 'test', components: [] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({ controlPlane, environment: environmentRunner })
|
|
await runtime.start({
|
|
onEnvironmentFailure: async error => {
|
|
assert.match(error.message, /candidate failed/)
|
|
calls.push('deployment:restore')
|
|
return true
|
|
},
|
|
})
|
|
assert.deepEqual(calls, [
|
|
'control:start',
|
|
'environment:start:1',
|
|
'deployment:restore',
|
|
'environment:start:2',
|
|
])
|
|
})
|
|
|
|
test('keeps DSH stopped when mandatory Patch verification fails for current and previous', async () => {
|
|
let validations = 0
|
|
let environmentStarts = 0
|
|
const controlPlane = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => {},
|
|
stop: async () => {},
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
}
|
|
const environmentRunner = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { environmentStarts += 1 },
|
|
stop: async () => {},
|
|
status: () => ({ environmentVersion: null, components: [] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({
|
|
controlPlane,
|
|
environment: environmentRunner,
|
|
validateDeployment: async () => {
|
|
validations += 1
|
|
throw new Error(`Patch verification failed for ${validations === 1 ? 'current' : 'previous'}`)
|
|
},
|
|
})
|
|
await runtime.start({
|
|
allowRecovery: true,
|
|
onEnvironmentFailure: async error => {
|
|
assert.match(error.message, /current/)
|
|
return true
|
|
},
|
|
})
|
|
assert.equal(validations, 2)
|
|
assert.equal(environmentStarts, 0)
|
|
assert.equal(runtime.status().recoveryMode, 'Deployment candidate and fallback both failed')
|
|
assert.deepEqual(runtime.status().controlPlane, [{ id: 'gateway' }])
|
|
})
|
|
|
|
test('keeps the Control Plane alive in recovery mode when no Deployment starts', async () => {
|
|
const calls = []
|
|
const controlPlane = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('control:start') },
|
|
stop: async () => { calls.push('control:stop') },
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
}
|
|
const environmentRunner = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { throw new Error('Runtime missing') },
|
|
stop: async () => {},
|
|
status: () => ({ environmentVersion: null, components: [] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({ controlPlane, environment: environmentRunner })
|
|
await runtime.start({ allowRecovery: true, onEnvironmentFailure: async () => false })
|
|
assert.equal(runtime.status().recoveryMode, 'Runtime missing')
|
|
assert.deepEqual(runtime.status().controlPlane, [{ id: 'gateway' }])
|
|
assert.deepEqual(calls, ['control:start'])
|
|
})
|
|
|
|
test('enters recovery mode when a failed candidate fallback cannot resolve', async () => {
|
|
const calls = []
|
|
const controlPlane = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { calls.push('control:start') },
|
|
stop: async () => { calls.push('control:stop') },
|
|
status: () => ({ components: [{ id: 'gateway' }] }),
|
|
}
|
|
const environmentRunner = {
|
|
fatal: new Promise(() => {}),
|
|
start: async () => { throw new Error('candidate unhealthy') },
|
|
stop: async () => {},
|
|
status: () => ({ environmentVersion: null, components: [] }),
|
|
}
|
|
const runtime = new BootstrapRuntime({ controlPlane, environment: environmentRunner })
|
|
await runtime.start({
|
|
allowRecovery: true,
|
|
onEnvironmentFailure: async () => { throw new Error('previous image is unavailable') },
|
|
})
|
|
assert.equal(runtime.status().recoveryMode, 'Deployment failed and its fallback could not be resolved')
|
|
assert.deepEqual(calls, ['control:start'])
|
|
})
|