dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,170 @@
|
||||
import { createServer } from 'node:http'
|
||||
import { chmod, mkdir, rm } from 'node:fs/promises'
|
||||
import { dirname } from 'node:path'
|
||||
import { PlatformGarbageCollector } from './gc.mjs'
|
||||
|
||||
async function jsonBody(request) {
|
||||
const chunks = []
|
||||
let size = 0
|
||||
for await (const chunk of request) {
|
||||
size += chunk.byteLength
|
||||
if (size > 256 * 1024) throw new Error('Bootstrap request body is too large')
|
||||
chunks.push(chunk)
|
||||
}
|
||||
return size === 0 ? {} : JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
||||
}
|
||||
|
||||
function send(response, status, value) {
|
||||
response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' })
|
||||
response.end(`${JSON.stringify(value)}\n`)
|
||||
}
|
||||
|
||||
export function createBootstrapControl(runner, { deployments, trust, systemPlugins, systemSkills, snapshots } = {}) {
|
||||
return createServer(async (request, response) => {
|
||||
let pathname = 'invalid-url'
|
||||
try {
|
||||
pathname = new URL(request.url ?? '/', 'http://bootstrap.internal').pathname
|
||||
if (request.method === 'GET' && pathname === '/v1/status') send(response, 200, {
|
||||
...runner.status(),
|
||||
bootstrapVersion: process.env.DSH_BOOTSTRAP_VERSION ?? '1.0.0',
|
||||
})
|
||||
else if (request.method === 'POST' && pathname === '/v1/reload') send(response, 200, await runner.reload())
|
||||
else if (request.method === 'GET' && pathname === '/v1/health') send(response, 200, await runner.health())
|
||||
else if (request.method === 'GET' && pathname === '/v1/system-plugins' && systemPlugins !== undefined) {
|
||||
send(response, 200, { plugins: await systemPlugins.list() })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/system-plugins/action' && systemPlugins !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
send(response, 200, { plugins: await systemPlugins.configure(body.id, body.action) })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/system-plugins/recovery-action' && systemPlugins !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
send(response, 200, { plugins: await systemPlugins.recover(body.id, body.action) })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/system-plugins/discard' && systemPlugins !== undefined) {
|
||||
send(response, 200, { plugins: await systemPlugins.discard() })
|
||||
} else if (request.method === 'GET' && pathname === '/v1/system-skills' && systemSkills !== undefined) {
|
||||
send(response, 200, { skills: await systemSkills.list() })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/system-skills/action' && systemSkills !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
send(response, 200, { skills: await systemSkills.configure(body.skillId, body.action) })
|
||||
}
|
||||
else if (request.method === 'GET' && pathname === '/v1/deployments/current' && deployments !== undefined) {
|
||||
const state = await deployments.state()
|
||||
send(response, 200, { record: state.current === null ? null : await deployments.record(state.current) })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/deployments/activate' && deployments !== undefined && trust !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
const slots = await deployments.activateManaged(body.record, {
|
||||
healthCheck: () => runner.reload(),
|
||||
activateReceipts: tokens => trust.activate(tokens),
|
||||
})
|
||||
send(response, 200, { slots })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/deployments/rollback' && deployments !== undefined && trust !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
const state = await deployments.state()
|
||||
if (state.previous === null) throw new Error('no previous Deployment exists')
|
||||
if (body.recordId !== undefined && body.recordId !== state.previous) throw new Error('requested Deployment is not previous')
|
||||
const slots = await deployments.rollback({
|
||||
healthCheck: () => runner.reload(),
|
||||
activateReceipts: tokens => trust.activate(tokens),
|
||||
})
|
||||
send(response, 200, { slots })
|
||||
} else if (request.method === 'GET' && pathname === '/v1/deployments/rollback-plan' && deployments !== undefined) {
|
||||
send(response, 200, await deployments.rollbackPlan())
|
||||
} else if (request.method === 'POST' && pathname === '/v1/deployments/candidate' && deployments !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
const record = await deployments.stageCandidate(body.record, () => runner.reload())
|
||||
send(response, 200, { recordId: record.id })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/deployments/candidate/commit' && deployments !== undefined && trust !== undefined) {
|
||||
const body = await jsonBody(request)
|
||||
const slots = await deployments.commitCandidate(body.recordId, tokens => trust.activate(tokens))
|
||||
send(response, 200, { slots })
|
||||
} else if (request.method === 'POST' && pathname === '/v1/deployments/candidate/cancel' && deployments !== undefined) {
|
||||
send(response, 200, await deployments.cancelCandidate())
|
||||
} else if (request.method === 'POST' && pathname === '/v1/platform/gc' && deployments !== undefined) {
|
||||
send(response, 200, await new PlatformGarbageCollector({ paths: deployments.paths, deployments, snapshots }).collect())
|
||||
} else if (request.method === 'POST' && pathname === '/v1/deployments/runtime/reset' && deployments !== undefined) {
|
||||
send(response, 200, await deployments.resetCurrentRuntime({
|
||||
pauseDsh: () => runner.pause('dsh-runtime'),
|
||||
restartDsh: () => runner.restart('dsh-runtime'),
|
||||
}))
|
||||
}
|
||||
else {
|
||||
const operation = /^\/v1\/components\/([a-z0-9][a-z0-9._-]{0,127})\/(pause|suspend|resume|restart)$/.exec(pathname)
|
||||
if (request.method === 'POST' && operation !== null) {
|
||||
const [componentId, action] = operation.slice(1)
|
||||
if (componentId === 'dsh-runtime' && action === 'resume' && deployments !== undefined) {
|
||||
const status = await deployments.exclusive(async () => {
|
||||
await deployments.setOperation('starting')
|
||||
try {
|
||||
const value = await runner.resume(componentId)
|
||||
await deployments.publishStatus()
|
||||
return value
|
||||
} catch (error) {
|
||||
await deployments.publishStatus({
|
||||
recoveryMode: runner.status().recoveryMode
|
||||
?? (error instanceof Error ? error.message : String(error)),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
})
|
||||
send(response, 200, status)
|
||||
} else if (componentId === 'dsh-runtime' && action === 'restart' && deployments !== undefined) {
|
||||
const status = await deployments.exclusive(async () => {
|
||||
await deployments.setOperation('restarting')
|
||||
let transaction
|
||||
try {
|
||||
transaction = await systemPlugins?.prepare?.()
|
||||
const value = await runner.restart(componentId, {
|
||||
skipPreparation: true,
|
||||
recoverStart: true,
|
||||
beforeStart: () => transaction?.activate?.(),
|
||||
onStartFailure: () => transaction?.rollback?.(),
|
||||
})
|
||||
if (transaction?.commit !== undefined) {
|
||||
try {
|
||||
await transaction.commit()
|
||||
} catch (error) {
|
||||
await runner.record?.('system-plugin.view.commit.failed', { error, level: 'warning' })
|
||||
}
|
||||
}
|
||||
await deployments.publishStatus()
|
||||
return value
|
||||
} catch (error) {
|
||||
try {
|
||||
await deployments.publishStatus({
|
||||
operation: 'restart-failed',
|
||||
recoveryMode: runner.status().recoveryMode
|
||||
?? (error instanceof Error ? error.message : String(error)),
|
||||
})
|
||||
} catch (statusError) {
|
||||
await runner.record?.('deployment.status.failed', {
|
||||
error: statusError,
|
||||
level: 'warning',
|
||||
originalError: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
})
|
||||
send(response, 200, status)
|
||||
} else send(response, 200, await runner[action](componentId))
|
||||
} else send(response, 404, { error: 'not found' })
|
||||
}
|
||||
} catch (error) {
|
||||
await runner.record?.('bootstrap.request.failed', {
|
||||
error,
|
||||
method: request.method ?? null,
|
||||
pathname,
|
||||
})
|
||||
send(response, 500, { error: error instanceof Error ? error.message : 'Bootstrap operation failed' })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export async function listenBootstrapControl(server, socketPath) {
|
||||
await mkdir(dirname(socketPath), { recursive: true })
|
||||
await rm(socketPath, { force: true })
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject)
|
||||
server.listen(socketPath, resolve)
|
||||
})
|
||||
await chmod(socketPath, 0o600)
|
||||
}
|
||||
@@ -0,0 +1,808 @@
|
||||
import { lstat, mkdir, readFile, rename, rm, symlink } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { canonicalJson } from '../../lib/canonical-json.mjs'
|
||||
import {
|
||||
deriveRecordId,
|
||||
parseDeploymentRecord,
|
||||
parseSlots,
|
||||
recordsFromImageInventory,
|
||||
} from '../../lib/deployment-contracts.mjs'
|
||||
import { durableCreate, durableReplace } from '../../lib/atomic.mjs'
|
||||
import { readDeploymentStatus, writeDeploymentStatus } from '../../lib/deployment-status.mjs'
|
||||
import { replaceDeploymentView, replaceSystemPluginView } from '../../lib/paths.mjs'
|
||||
import { hashTree, hashTreeContent } from '../../lib/tree-hash.mjs'
|
||||
import { compareDshVersions } from '../../lib/supported-target.mjs'
|
||||
import { TrustError } from '../../lib/validation.mjs'
|
||||
import { buildRuntime, cloneTree, loadEnvironmentPatchSet } from '../../../control-plane/modules/patch-manager/index.mjs'
|
||||
|
||||
const KINDS = Object.freeze(['environment', 'pristine', 'runtime', 'system-plugins'])
|
||||
|
||||
export class DeploymentResolutionError extends Error {
|
||||
constructor(message, cause) {
|
||||
super(message, { cause })
|
||||
this.name = 'DeploymentResolutionError'
|
||||
}
|
||||
}
|
||||
|
||||
function sameDeployment(left, right) {
|
||||
return left.dshVersion === right.dshVersion
|
||||
&& left.environmentVersion === right.environmentVersion
|
||||
&& KINDS.every(kind => {
|
||||
const field = kind === 'system-plugins' ? 'systemPlugins' : kind
|
||||
return left[field].sha256 === right[field].sha256
|
||||
})
|
||||
}
|
||||
|
||||
function validateCandidate(from, candidate, experimental) {
|
||||
if (experimental) {
|
||||
if (candidate.authority !== 'experimental') throw new TrustError('probation candidate must use Experimental authority')
|
||||
if (candidate.targetSequence !== from.targetSequence) throw new TrustError('Experimental candidate must retain the Stable targetSequence')
|
||||
if (compareDshVersions(candidate.dshVersion, from.dshVersion) <= 0) {
|
||||
throw new TrustError('Experimental candidate DSH must advance the current version')
|
||||
}
|
||||
if (
|
||||
candidate.environment.sha256 !== from.environment.sha256
|
||||
|| candidate.systemPlugins.sha256 !== from.systemPlugins.sha256
|
||||
) throw new TrustError('Experimental candidate must retain the current Environment and System Plugins')
|
||||
return
|
||||
}
|
||||
if (candidate.authority !== 'stable') throw new TrustError('managed release activation requires Stable authority')
|
||||
if (candidate.targetSequence < from.targetSequence) throw new TrustError('Stable candidate targetSequence would roll back')
|
||||
if (from.authority === 'stable' && candidate.targetSequence === from.targetSequence && !sameDeployment(from, candidate)) {
|
||||
throw new TrustError('Stable candidate conflicts with current content at the same targetSequence')
|
||||
}
|
||||
if (from.authority === 'experimental' && compareDshVersions(candidate.dshVersion, from.dshVersion) < 0) {
|
||||
throw new TrustError('Stable candidate has not caught up with Experimental DSH')
|
||||
}
|
||||
}
|
||||
|
||||
async function readOptional(path) {
|
||||
try { return await readFile(path) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
async function replaceRuntimeDirectory(staging, destination) {
|
||||
const backup = `${destination}.${randomUUID()}.reset-backup`
|
||||
const hadDestination = await exists(destination)
|
||||
if (hadDestination) await rename(destination, backup)
|
||||
try {
|
||||
await rename(staging, destination)
|
||||
} catch (error) {
|
||||
if (hadDestination) await rename(backup, destination).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
return Object.freeze({
|
||||
commit: () => hadDestination ? rm(backup, { recursive: true, force: true }) : Promise.resolve(),
|
||||
rollback: async () => {
|
||||
await rm(destination, { recursive: true, force: true })
|
||||
if (hadDestination) await rename(backup, destination)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export class DeploymentManager {
|
||||
constructor({ paths, seedRoot, inventory }) {
|
||||
this.paths = paths
|
||||
this.seedRoot = resolve(seedRoot)
|
||||
this.inventory = inventory
|
||||
this.imageRecord = recordsFromImageInventory(inventory).deployment
|
||||
this.stateRoot = paths.deploymentStateRoot
|
||||
this.recordsRoot = join(this.stateRoot, 'records')
|
||||
this.slotsPath = join(this.stateRoot, 'slots.json')
|
||||
this.activationPath = join(this.stateRoot, 'activation.json')
|
||||
this.queue = Promise.resolve()
|
||||
}
|
||||
|
||||
exclusive(operation) {
|
||||
const result = this.queue.then(operation, operation)
|
||||
this.queue = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
recordPath(id) {
|
||||
return join(this.recordsRoot, `${id}.json`)
|
||||
}
|
||||
|
||||
async writeRecord(value) {
|
||||
const record = parseDeploymentRecord(value)
|
||||
const bytes = canonicalJson(record)
|
||||
const path = this.recordPath(record.id)
|
||||
const existing = await readOptional(path)
|
||||
if (existing === undefined) await durableCreate(path, bytes)
|
||||
else if (!existing.equals(bytes)) throw new TrustError('Deployment Record ID identifies conflicting content')
|
||||
return record
|
||||
}
|
||||
|
||||
async repairImageRecord(value) {
|
||||
const record = parseDeploymentRecord(value)
|
||||
const bytes = canonicalJson(record)
|
||||
if (!bytes.equals(canonicalJson(this.imageRecord))) {
|
||||
throw new TrustError('image recovery Record differs from the current image inventory')
|
||||
}
|
||||
await durableReplace(this.recordPath(record.id), bytes)
|
||||
return record
|
||||
}
|
||||
|
||||
async record(id) {
|
||||
const bytes = await readOptional(this.recordPath(id))
|
||||
if (bytes === undefined) throw new TrustError(`Deployment Record ${id} does not exist`)
|
||||
return parseDeploymentRecord(bytes)
|
||||
}
|
||||
|
||||
async state() {
|
||||
const bytes = await readOptional(this.slotsPath)
|
||||
if (bytes === undefined) return Object.freeze({ generation: 0, current: null, previous: null })
|
||||
return parseSlots(bytes, 'deployment-record', 'Deployment slots')
|
||||
}
|
||||
|
||||
availableInCurrentImage(record) {
|
||||
return KINDS.every(kind => {
|
||||
const reference = record[kind === 'system-plugins' ? 'systemPlugins' : kind]
|
||||
return reference.storage !== 'image' || reference.imageBuildId === this.inventory.imageBuildId
|
||||
})
|
||||
}
|
||||
|
||||
imageRecordFromAnotherImage(record) {
|
||||
return KINDS.every(kind => {
|
||||
const reference = record[kind === 'system-plugins' ? 'systemPlugins' : kind]
|
||||
return reference.storage === 'image' && reference.imageBuildId !== this.inventory.imageBuildId
|
||||
})
|
||||
}
|
||||
|
||||
async rollbackPlan() {
|
||||
const state = await this.state()
|
||||
const current = state.current === null ? null : await this.record(state.current)
|
||||
let previous = state.previous === null ? null : await this.record(state.previous)
|
||||
if (previous !== null && !this.availableInCurrentImage(previous)) previous = null
|
||||
return Object.freeze({ current, previous })
|
||||
}
|
||||
|
||||
async commit(current, previous) {
|
||||
const state = await this.state()
|
||||
if (current === previous) previous = null
|
||||
const slots = parseSlots({
|
||||
schema: 1,
|
||||
generation: state.generation + 1,
|
||||
current,
|
||||
previous,
|
||||
}, 'deployment-record', 'Deployment slots')
|
||||
await durableReplace(this.slotsPath, canonicalJson(slots))
|
||||
return slots
|
||||
}
|
||||
|
||||
imageAsset(reference) {
|
||||
if (reference.imageBuildId !== this.inventory.imageBuildId) {
|
||||
throw new TrustError('Deployment Image Reference belongs to a different image')
|
||||
}
|
||||
const expected = reference.kind === 'system-plugins'
|
||||
? this.inventory.deployment.systemPlugins
|
||||
: this.inventory.deployment[reference.kind]
|
||||
if (expected === undefined || reference.id !== expected.id || reference.sha256 !== expected.sha256) {
|
||||
throw new TrustError(`Deployment Image Reference for ${reference.kind} is absent from inventory`)
|
||||
}
|
||||
const roots = {
|
||||
environment: 'environments',
|
||||
pristine: 'pristine',
|
||||
runtime: 'runtimes',
|
||||
'system-plugins': 'system-plugins',
|
||||
}
|
||||
return join(this.seedRoot, roots[reference.kind], reference.id)
|
||||
}
|
||||
|
||||
storeAsset(reference) {
|
||||
const roots = {
|
||||
environment: this.paths.environmentsRoot,
|
||||
pristine: this.paths.pristineRoot,
|
||||
runtime: this.paths.runtimesRoot,
|
||||
'system-plugins': this.paths.systemPluginsRoot,
|
||||
}
|
||||
return join(roots[reference.kind], reference.id)
|
||||
}
|
||||
|
||||
async resolveReference(reference) {
|
||||
const path = reference.storage === 'image' ? this.imageAsset(reference) : this.storeAsset(reference)
|
||||
const details = await lstat(path)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new TrustError(`resolved ${reference.kind} must be an immutable directory`)
|
||||
}
|
||||
// Image bytes are bound to the signed inventory during image assembly. Rehashing a large
|
||||
// Runtime on every boot would defeat zero-copy startup; managed objects are checked here.
|
||||
if (reference.storage === 'store' && await hashTree(path) !== reference.sha256) {
|
||||
throw new TrustError(`resolved ${reference.kind} content hash differs from its Record`)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
async resolveRecord(recordId) {
|
||||
const record = await this.record(recordId)
|
||||
const entries = await Promise.all(KINDS.map(async kind => [kind, await this.resolveReference(record[kind === 'system-plugins' ? 'systemPlugins' : kind])]))
|
||||
return Object.freeze({ record, paths: Object.freeze(Object.fromEntries(entries)) })
|
||||
}
|
||||
|
||||
async sameImageDeployment(current, image) {
|
||||
if (sameDeployment(current, image)) return true
|
||||
if (
|
||||
current.dshVersion !== image.dshVersion
|
||||
|| current.environmentVersion !== image.environmentVersion
|
||||
|| current.pristine.sha256 !== image.pristine.sha256
|
||||
|| current.runtime.sha256 !== image.runtime.sha256
|
||||
|| current.systemPlugins.sha256 !== image.systemPlugins.sha256
|
||||
) return false
|
||||
const [currentEnvironment, imageEnvironment] = await Promise.all([
|
||||
this.resolveReference(current.environment),
|
||||
this.resolveReference(image.environment),
|
||||
])
|
||||
const [currentContent, imageContent] = await Promise.all([
|
||||
hashTreeContent(currentEnvironment),
|
||||
hashTreeContent(imageEnvironment),
|
||||
])
|
||||
return currentContent === imageContent
|
||||
}
|
||||
|
||||
async selected() {
|
||||
const recordId = this.selectedRecordId ?? (await this.state()).current
|
||||
return recordId === null ? null : this.resolveRecord(recordId)
|
||||
}
|
||||
|
||||
async prepareView(recordId) {
|
||||
const resolved = await this.resolveRecord(recordId)
|
||||
const destination = join(this.paths.deploymentViewsRoot, recordId)
|
||||
const temporary = join(this.paths.deploymentViewsRoot, `.${recordId}.${randomUUID()}.tmp`)
|
||||
await mkdir(temporary, { recursive: false })
|
||||
try {
|
||||
for (const kind of KINDS) await symlink(resolved.paths[kind], join(temporary, kind), 'dir')
|
||||
try {
|
||||
await rename(temporary, destination)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
|
||||
}
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true })
|
||||
}
|
||||
return Object.freeze({ ...resolved, view: destination })
|
||||
}
|
||||
|
||||
async select(recordId) {
|
||||
const candidate = await this.prepareView(recordId)
|
||||
await replaceDeploymentView(this.paths, candidate.view)
|
||||
await replaceSystemPluginView(this.paths)
|
||||
this.selectedRecordId = recordId
|
||||
return candidate
|
||||
}
|
||||
|
||||
async initialize(record) {
|
||||
const plan = await this.prepareImage(record)
|
||||
return this.acceptImage(plan)
|
||||
}
|
||||
|
||||
async prepareImage(record) {
|
||||
return this.exclusive(async () => {
|
||||
const image = await this.writeRecord(record)
|
||||
const state = await this.state()
|
||||
if (state.current === null) {
|
||||
await this.select(image.id)
|
||||
return Object.freeze({
|
||||
baseGeneration: 0,
|
||||
target: image.id,
|
||||
fallback: null,
|
||||
action: 'initialize',
|
||||
imageBehindCurrent: false,
|
||||
requiresExperimentalRebuild: false,
|
||||
})
|
||||
}
|
||||
|
||||
const current = await this.record(state.current)
|
||||
let target = current.id
|
||||
let action = 'retain'
|
||||
let imageBehindCurrent = false
|
||||
let requiresExperimentalRebuild = false
|
||||
|
||||
if (
|
||||
image.authority === 'development'
|
||||
&& image.targetSequence === 0
|
||||
&& current.id !== image.id
|
||||
&& compareDshVersions(image.dshVersion, current.dshVersion) >= 0
|
||||
) {
|
||||
target = image.id
|
||||
action = 'development-refresh'
|
||||
} else if (
|
||||
image.targetSequence < current.targetSequence
|
||||
&& current.receiptTokens.length === 0
|
||||
&& this.imageRecordFromAnotherImage(current)
|
||||
) {
|
||||
target = image.id
|
||||
action = 'image-rollback'
|
||||
} else if (current.authority === 'experimental') {
|
||||
const dshCaughtUp = compareDshVersions(image.dshVersion, current.dshVersion) >= 0
|
||||
if (image.targetSequence >= current.targetSequence && dshCaughtUp) {
|
||||
target = image.id
|
||||
action = 'stable-caught-up'
|
||||
} else {
|
||||
imageBehindCurrent = image.targetSequence < current.targetSequence
|
||||
requiresExperimentalRebuild = image.targetSequence > current.targetSequence && !dshCaughtUp
|
||||
}
|
||||
} else if (image.targetSequence > current.targetSequence) {
|
||||
target = image.id
|
||||
action = 'image-forward'
|
||||
} else if (image.targetSequence < current.targetSequence) {
|
||||
imageBehindCurrent = true
|
||||
} else if (!await this.sameImageDeployment(current, image)) {
|
||||
throw new TrustError('Deployment image conflicts with current content at the same targetSequence')
|
||||
} else if (current.id !== image.id) {
|
||||
target = image.id
|
||||
action = 'prefer-image'
|
||||
}
|
||||
|
||||
let candidateError = null
|
||||
try {
|
||||
await this.select(target)
|
||||
} catch (error) {
|
||||
if (target === current.id) {
|
||||
throw new DeploymentResolutionError('current Deployment assets could not be resolved', error)
|
||||
}
|
||||
try {
|
||||
await this.select(current.id)
|
||||
} catch (fallbackError) {
|
||||
throw new DeploymentResolutionError(
|
||||
'image candidate and current Deployment assets could not be resolved',
|
||||
new AggregateError([error, fallbackError]),
|
||||
)
|
||||
}
|
||||
target = current.id
|
||||
action = 'image-rejected'
|
||||
candidateError = error instanceof Error ? error.message : 'image candidate could not be resolved'
|
||||
}
|
||||
return Object.freeze({
|
||||
baseGeneration: state.generation,
|
||||
target,
|
||||
fallback: target === current.id || action === 'development-refresh' ? null : current.id,
|
||||
action,
|
||||
imageBehindCurrent,
|
||||
requiresExperimentalRebuild,
|
||||
candidateError,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async acceptImage(plan) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.generation !== plan.baseGeneration) throw new TrustError('Deployment slots changed during image startup')
|
||||
if (state.current === plan.target) return state
|
||||
return this.commit(plan.target, plan.action === 'development-refresh' ? null : state.current)
|
||||
})
|
||||
}
|
||||
|
||||
async rejectImage(plan) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.generation !== plan.baseGeneration) throw new TrustError('Deployment slots changed during image startup')
|
||||
if (plan.fallback === null) return false
|
||||
await this.select(plan.fallback)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
async preparePreviousRecovery(failedRecordId) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
const failed = failedRecordId ?? state.current
|
||||
if (failed === null || state.current !== failed) {
|
||||
throw new TrustError('failed Deployment is no longer current')
|
||||
}
|
||||
if (state.previous === null || state.previous === failed) {
|
||||
throw new TrustError('no previous Deployment exists')
|
||||
}
|
||||
await this.record(failed)
|
||||
await this.select(state.previous)
|
||||
return Object.freeze({
|
||||
baseGeneration: state.generation,
|
||||
failed,
|
||||
target: state.previous,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async acceptPreviousRecovery(plan, activateReceipts) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.generation !== plan.baseGeneration || state.current !== plan.failed || state.previous !== plan.target) {
|
||||
throw new TrustError('Deployment slots changed during automatic recovery')
|
||||
}
|
||||
const previous = await this.record(plan.target)
|
||||
await activateReceipts(previous.receiptTokens)
|
||||
await this.writeActivation({ phase: 'receipts-active', from: plan.failed, to: plan.target })
|
||||
const committed = await this.commit(plan.target, plan.failed)
|
||||
await this.clearActivation()
|
||||
return committed
|
||||
})
|
||||
}
|
||||
|
||||
async activate(recordId, healthCheck) {
|
||||
return this.exclusive(async () => {
|
||||
await this.record(recordId)
|
||||
const state = await this.state()
|
||||
if (state.current === recordId) {
|
||||
await this.select(recordId)
|
||||
return state
|
||||
}
|
||||
await this.select(recordId)
|
||||
try {
|
||||
await healthCheck()
|
||||
} catch (error) {
|
||||
if (state.current !== null) {
|
||||
await this.select(state.current)
|
||||
await healthCheck().catch(() => {})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
return this.commit(recordId, state.current)
|
||||
})
|
||||
}
|
||||
|
||||
async rollback({ healthCheck, activateReceipts }) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.previous === null) throw new TrustError('no previous Deployment exists')
|
||||
const from = await this.record(state.current)
|
||||
const previous = await this.record(state.previous)
|
||||
await this.writeActivation({ phase: 'prepared', from: from.id, to: previous.id })
|
||||
await this.publishStatus({ operation: 'recovering' })
|
||||
let receiptsActive = false
|
||||
try {
|
||||
await this.select(previous.id)
|
||||
await healthCheck()
|
||||
await this.writeActivation({ phase: 'healthy', from: from.id, to: previous.id })
|
||||
await activateReceipts(previous.receiptTokens)
|
||||
receiptsActive = true
|
||||
await this.writeActivation({ phase: 'receipts-active', from: from.id, to: previous.id })
|
||||
const committed = await this.commit(previous.id, from.id)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return committed
|
||||
} catch (error) {
|
||||
if (receiptsActive) await activateReceipts(from.receiptTokens).catch(() => {})
|
||||
await this.select(from.id).catch(() => {})
|
||||
await healthCheck().catch(() => {})
|
||||
await this.clearActivation().catch(() => {})
|
||||
await this.publishStatus().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async materializeRecord(recordId) {
|
||||
const record = await this.record(recordId)
|
||||
const fields = {
|
||||
environment: 'environment',
|
||||
pristine: 'pristine',
|
||||
runtime: 'runtime',
|
||||
systemPlugins: 'system-plugins',
|
||||
}
|
||||
const references = {}
|
||||
for (const [field, kind] of Object.entries(fields)) {
|
||||
const reference = record[field]
|
||||
if (reference.storage === 'store') {
|
||||
references[field] = reference
|
||||
continue
|
||||
}
|
||||
const source = await this.resolveReference(reference)
|
||||
const id = `image-${kind}-${reference.sha256}`
|
||||
const destination = this.storeAsset({ kind, id })
|
||||
if (!await exists(destination)) {
|
||||
const temporary = `${destination}.${randomUUID()}.tmp`
|
||||
try {
|
||||
await cloneTree(source, temporary)
|
||||
if (await hashTree(temporary) !== reference.sha256) throw new TrustError(`materialized ${kind} differs from Image Reference`)
|
||||
await rename(temporary, destination)
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true })
|
||||
}
|
||||
} else if (await hashTree(destination) !== reference.sha256) {
|
||||
throw new TrustError(`materialized ${kind} conflicts with Managed Store`)
|
||||
}
|
||||
references[field] = { storage: 'store', kind, id, sha256: reference.sha256 }
|
||||
}
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: record.authority,
|
||||
targetSequence: record.targetSequence,
|
||||
dshVersion: record.dshVersion,
|
||||
environmentVersion: record.environmentVersion,
|
||||
...references,
|
||||
receiptTokens: [...record.receiptTokens],
|
||||
snapshotId: record.snapshotId,
|
||||
}
|
||||
return this.writeRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
}
|
||||
|
||||
async materializeCurrent() {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.current === null) throw new TrustError('no current Deployment exists')
|
||||
const current = await this.record(state.current)
|
||||
const references = [current.environment, current.pristine, current.runtime, current.systemPlugins]
|
||||
if (references.every(reference => reference.storage === 'store')) return current
|
||||
const materialized = await this.materializeRecord(current.id)
|
||||
await this.select(materialized.id)
|
||||
await this.commit(materialized.id, state.previous)
|
||||
return materialized
|
||||
})
|
||||
}
|
||||
|
||||
async resetCurrentRuntime({ pauseDsh, restartDsh }) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.current === null) throw new TrustError('no current Deployment exists')
|
||||
const current = await this.record(state.current)
|
||||
const [pristineRoot, environmentRoot] = await Promise.all([
|
||||
this.resolveReference(current.pristine),
|
||||
this.resolveReference(current.environment),
|
||||
])
|
||||
const patchSet = await loadEnvironmentPatchSet(environmentRoot)
|
||||
const stagingId = `runtime-reset-${randomUUID()}`
|
||||
const versionsRoot = this.paths.runtimesRoot
|
||||
const staging = join(versionsRoot, stagingId)
|
||||
const destinationId = current.runtime.storage === 'store'
|
||||
? current.runtime.id
|
||||
: `reset-${current.runtime.sha256}`
|
||||
const destination = join(this.paths.runtimesRoot, destinationId)
|
||||
let replacement
|
||||
let paused = false
|
||||
await this.publishStatus({ operation: 'runtime-reset-building' })
|
||||
try {
|
||||
await buildRuntime({
|
||||
pristineRoot,
|
||||
versionsRoot,
|
||||
runtimeId: stagingId,
|
||||
patchPaths: patchSet.paths,
|
||||
})
|
||||
await this.publishStatus({ operation: 'runtime-reset-verifying' })
|
||||
const sha256 = await hashTree(staging)
|
||||
if (sha256 !== current.runtime.sha256) {
|
||||
throw new TrustError('rebuilt Runtime differs from the current Deployment Record')
|
||||
}
|
||||
|
||||
await this.publishStatus({ operation: 'runtime-reset-switching' })
|
||||
await pauseDsh()
|
||||
paused = true
|
||||
replacement = await replaceRuntimeDirectory(staging, destination)
|
||||
const runtime = Object.freeze({
|
||||
storage: 'store', kind: 'runtime', id: destinationId, sha256,
|
||||
})
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: current.authority,
|
||||
targetSequence: current.targetSequence,
|
||||
dshVersion: current.dshVersion,
|
||||
environmentVersion: current.environmentVersion,
|
||||
environment: current.environment,
|
||||
pristine: current.pristine,
|
||||
runtime,
|
||||
systemPlugins: current.systemPlugins,
|
||||
receiptTokens: [...current.receiptTokens],
|
||||
snapshotId: current.snapshotId,
|
||||
}
|
||||
const repaired = current.runtime.storage === 'store'
|
||||
? current
|
||||
: await this.writeRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
await this.select(repaired.id)
|
||||
await this.publishStatus({ operation: 'runtime-reset-starting' })
|
||||
await restartDsh()
|
||||
const slots = repaired.id === current.id
|
||||
? state
|
||||
: await this.commit(repaired.id, state.previous)
|
||||
await replacement.commit()
|
||||
await this.publishStatus({ recoveryMode: null })
|
||||
return Object.freeze({ recordId: repaired.id, slots })
|
||||
} catch (error) {
|
||||
if (replacement !== undefined) await replacement.rollback().catch(() => {})
|
||||
if (paused) {
|
||||
await this.publishStatus({ operation: 'runtime-reset-recovering' }).catch(() => {})
|
||||
await this.select(current.id).catch(() => {})
|
||||
await restartDsh().catch(() => {})
|
||||
}
|
||||
await this.publishStatus({ operation: 'runtime-reset-failed' }).catch(() => {})
|
||||
throw error
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async activation() {
|
||||
const bytes = await readOptional(this.activationPath)
|
||||
if (bytes === undefined) return undefined
|
||||
const value = JSON.parse(bytes.toString('utf8'))
|
||||
if (
|
||||
value?.schema !== 1
|
||||
|| !['prepared', 'healthy', 'probation', 'receipts-active'].includes(value.phase)
|
||||
|| typeof value.from !== 'string'
|
||||
|| typeof value.to !== 'string'
|
||||
) throw new TrustError('Deployment activation journal is invalid')
|
||||
return Object.freeze(value)
|
||||
}
|
||||
|
||||
async writeActivation(value) {
|
||||
const journal = Object.freeze({ schema: 1, ...value, updatedAt: new Date().toISOString() })
|
||||
await durableReplace(this.activationPath, canonicalJson(journal))
|
||||
return journal
|
||||
}
|
||||
|
||||
async clearActivation() {
|
||||
await rm(this.activationPath, { force: true })
|
||||
}
|
||||
|
||||
async recoverActivation(trust) {
|
||||
const journal = await this.activation()
|
||||
if (journal === undefined) return null
|
||||
const [from, to, active] = await Promise.all([
|
||||
this.record(journal.from),
|
||||
this.record(journal.to),
|
||||
trust.activeReceipts(),
|
||||
])
|
||||
const activeTokens = new Set(active.receipts.map(receipt => receipt.token))
|
||||
const candidateReceiptsActive = to.receiptTokens.every(token => activeTokens.has(token))
|
||||
if (journal.phase === 'receipts-active' || (journal.phase === 'healthy' && candidateReceiptsActive)) {
|
||||
await this.select(to.id)
|
||||
const state = await this.state()
|
||||
if (state.current !== to.id) await this.commit(to.id, from.id)
|
||||
await this.clearActivation()
|
||||
return Object.freeze({ action: 'committed', recordId: to.id })
|
||||
}
|
||||
await trust.activate(from.receiptTokens)
|
||||
await this.select(from.id)
|
||||
const state = await this.state()
|
||||
if (state.current !== from.id) await this.commit(from.id, state.current)
|
||||
await this.clearActivation()
|
||||
return Object.freeze({ action: 'rolled-back', recordId: from.id })
|
||||
}
|
||||
|
||||
async activateManaged(value, { healthCheck, activateReceipts }) {
|
||||
const candidate = await this.writeRecord(value)
|
||||
await this.materializeCurrent()
|
||||
const state = await this.state()
|
||||
const from = await this.record(state.current)
|
||||
if (candidate.id === from.id) return state
|
||||
validateCandidate(from, candidate, false)
|
||||
await this.writeActivation({ phase: 'prepared', from: from.id, to: candidate.id })
|
||||
await this.publishStatus({ operation: 'switching' })
|
||||
let receiptsActive = false
|
||||
try {
|
||||
await this.select(candidate.id)
|
||||
await healthCheck()
|
||||
await this.writeActivation({ phase: 'healthy', from: from.id, to: candidate.id })
|
||||
await activateReceipts(candidate.receiptTokens)
|
||||
receiptsActive = true
|
||||
await this.writeActivation({ phase: 'receipts-active', from: from.id, to: candidate.id })
|
||||
const committed = await this.commit(candidate.id, from.id)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return committed
|
||||
} catch (error) {
|
||||
await this.publishStatus({ operation: 'recovering' }).catch(() => {})
|
||||
if (receiptsActive) await activateReceipts(from.receiptTokens).catch(() => {})
|
||||
await this.select(from.id).catch(() => {})
|
||||
await healthCheck().catch(() => {})
|
||||
await this.clearActivation().catch(() => {})
|
||||
await this.publishStatus().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async stageCandidate(value, healthCheck) {
|
||||
const candidate = await this.writeRecord(value)
|
||||
await this.materializeCurrent()
|
||||
const state = await this.state()
|
||||
const from = await this.record(state.current)
|
||||
validateCandidate(from, candidate, true)
|
||||
await this.writeActivation({ phase: 'prepared', from: from.id, to: candidate.id })
|
||||
try {
|
||||
await this.select(candidate.id)
|
||||
await healthCheck()
|
||||
await this.writeActivation({ phase: 'probation', from: from.id, to: candidate.id })
|
||||
return candidate
|
||||
} catch (error) {
|
||||
await this.select(from.id).catch(() => {})
|
||||
await this.clearActivation().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async commitCandidate(recordId, activateReceipts) {
|
||||
const journal = await this.activation()
|
||||
if (journal?.phase !== 'probation' || journal.to !== recordId) {
|
||||
throw new TrustError('Deployment candidate is not in probation')
|
||||
}
|
||||
const candidate = await this.record(recordId)
|
||||
await activateReceipts(candidate.receiptTokens)
|
||||
await this.writeActivation({ phase: 'receipts-active', from: journal.from, to: journal.to })
|
||||
const committed = await this.commit(candidate.id, journal.from)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return committed
|
||||
}
|
||||
|
||||
async cancelCandidate() {
|
||||
const journal = await this.activation()
|
||||
if (journal === undefined) return Object.freeze({ cancelled: false, slots: await this.state() })
|
||||
const from = await this.record(journal.from)
|
||||
await this.select(from.id)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return Object.freeze({ cancelled: true, slots: await this.state() })
|
||||
}
|
||||
|
||||
async recoverImageBaseline(value, { healthCheck, activateReceipts }) {
|
||||
const previousStatus = await readDeploymentStatus(this.paths.deploymentStatusPath)
|
||||
await this.publishStatus({ operation: 'recovering', recoveryMode: previousStatus.recoveryMode })
|
||||
const image = await this.repairImageRecord(value)
|
||||
await this.prepareView(image.id)
|
||||
const state = await this.state()
|
||||
await this.select(image.id)
|
||||
try {
|
||||
await healthCheck()
|
||||
await activateReceipts(image.receiptTokens)
|
||||
const committed = state.current === image.id ? state : await this.commit(image.id, state.current)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus({ recoveryMode: null })
|
||||
return committed
|
||||
} catch (error) {
|
||||
if (state.current !== null) await this.select(state.current).catch(() => {})
|
||||
await this.publishStatus({
|
||||
recoveryMode: error instanceof Error ? error.message : 'image baseline recovery failed',
|
||||
}).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
describe(record) {
|
||||
const references = [record.environment, record.pristine, record.runtime, record.systemPlugins]
|
||||
return Object.freeze({
|
||||
recordId: record.id,
|
||||
source: references.every(reference => reference.storage === 'image') ? 'image' : 'managed',
|
||||
authority: record.authority,
|
||||
targetSequence: record.targetSequence,
|
||||
dsh: record.dshVersion,
|
||||
environment: record.environmentVersion,
|
||||
})
|
||||
}
|
||||
|
||||
async publishStatus({ plan = null, recoveryMode = null, currentId, operation = null } = {}) {
|
||||
const state = await this.state().catch(() => ({ current: null }))
|
||||
const selectedId = currentId === undefined ? state.current : currentId
|
||||
const current = selectedId === null
|
||||
? null
|
||||
: await this.record(selectedId).then(record => this.describe(record), () => null)
|
||||
const value = Object.freeze({
|
||||
platformLayout: 1,
|
||||
imageBaseline: Object.freeze({
|
||||
imageBuildId: this.inventory.imageBuildId,
|
||||
targetSequence: this.inventory.targetSequence,
|
||||
dsh: this.inventory.deployment.dshVersion,
|
||||
environment: this.inventory.deployment.environmentVersion,
|
||||
}),
|
||||
current,
|
||||
imageBehindCurrent: plan?.imageBehindCurrent
|
||||
?? (current !== null && current.targetSequence > this.inventory.targetSequence),
|
||||
recoveryMode,
|
||||
operation,
|
||||
})
|
||||
return writeDeploymentStatus(this.paths.deploymentStatusPath, value)
|
||||
}
|
||||
|
||||
async setOperation(operation) {
|
||||
const status = await readDeploymentStatus(this.paths.deploymentStatusPath)
|
||||
return writeDeploymentStatus(this.paths.deploymentStatusPath, Object.freeze({
|
||||
...status,
|
||||
operation,
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import { randomBytes, randomUUID, timingSafeEqual } from 'node:crypto'
|
||||
import { chmod, mkdir, rm } from 'node:fs/promises'
|
||||
import { createServer } from 'node:http'
|
||||
import { dirname } from 'node:path'
|
||||
|
||||
const MAX_BODY_BYTES = 4 * 1024
|
||||
|
||||
function exactObject(value, keys, name) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`${name} must be an object`)
|
||||
}
|
||||
const actual = Object.keys(value).sort()
|
||||
const expected = [...keys].sort()
|
||||
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) {
|
||||
throw new Error(`${name} fields are invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
async function jsonBody(request) {
|
||||
const chunks = []
|
||||
let size = 0
|
||||
for await (const chunk of request) {
|
||||
size += chunk.byteLength
|
||||
if (size > MAX_BODY_BYTES) throw new Error('lifecycle request body is too large')
|
||||
chunks.push(chunk)
|
||||
}
|
||||
if (size === 0) throw new Error('lifecycle request body is required')
|
||||
return JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
||||
}
|
||||
|
||||
function send(response, status, value) {
|
||||
response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' })
|
||||
response.end(`${JSON.stringify(value)}\n`)
|
||||
}
|
||||
|
||||
function matchesSecret(actual, expected) {
|
||||
if (typeof actual !== 'string' || typeof expected !== 'string') return false
|
||||
const left = Buffer.from(actual)
|
||||
const right = Buffer.from(expected)
|
||||
return left.byteLength === right.byteLength && timingSafeEqual(left, right)
|
||||
}
|
||||
|
||||
export class DshLifecycleBroker {
|
||||
constructor({ shouldTerminate = async () => false, report = async () => {} } = {}) {
|
||||
this.shouldTerminate = shouldTerminate
|
||||
this.report = report
|
||||
this.launch = null
|
||||
this.session = null
|
||||
this.published = false
|
||||
this.shuttingDown = false
|
||||
}
|
||||
|
||||
record(message, fields = {}) {
|
||||
return Promise.resolve().then(() => this.report(message, fields)).catch(() => {})
|
||||
}
|
||||
|
||||
prepareLaunch(componentId) {
|
||||
if (componentId !== 'dsh-runtime') return Object.freeze({ environment: Object.freeze({}), release: () => {} })
|
||||
const launch = Object.freeze({ id: randomUUID(), token: randomBytes(32).toString('base64url') })
|
||||
this.launch = launch
|
||||
this.session = null
|
||||
this.published = false
|
||||
void this.record('dsh.launch.authorized')
|
||||
return Object.freeze({
|
||||
environment: Object.freeze({ DSH_PLATFORM_LAUNCH_TOKEN: launch.token }),
|
||||
release: () => this.release(launch.id),
|
||||
})
|
||||
}
|
||||
|
||||
release(launchId) {
|
||||
if (this.launch?.id !== launchId) return
|
||||
this.launch = null
|
||||
this.session = null
|
||||
void this.record('dsh.launch.released')
|
||||
}
|
||||
|
||||
claim(launchToken) {
|
||||
if (this.launch === null || this.session !== null || !matchesSecret(launchToken, this.launch.token)) {
|
||||
throw Object.assign(new Error('DSH launch token is invalid or already consumed'), { statusCode: 409 })
|
||||
}
|
||||
const sessionId = randomUUID()
|
||||
this.session = Object.freeze({ id: sessionId, launchId: this.launch.id, ready: false, verified: false })
|
||||
this.launch = Object.freeze({ ...this.launch, token: null })
|
||||
void this.record('dsh.launch.claimed')
|
||||
return Object.freeze({ sessionId })
|
||||
}
|
||||
|
||||
ready(sessionId, readyUrl = null) {
|
||||
if (this.session === null || !matchesSecret(sessionId, this.session.id)) {
|
||||
throw Object.assign(new Error('DSH lifecycle session is invalid'), { statusCode: 409 })
|
||||
}
|
||||
if (readyUrl !== null) {
|
||||
let parsed
|
||||
try {
|
||||
parsed = new URL(readyUrl)
|
||||
} catch {
|
||||
throw Object.assign(new Error('DSH Web readiness URL is invalid'), { statusCode: 400 })
|
||||
}
|
||||
if (parsed.protocol !== 'http:' || parsed.hostname !== '127.0.0.1'
|
||||
|| parsed.pathname !== '/' || parsed.username !== '' || parsed.password !== ''
|
||||
|| parsed.hash !== '' || parsed.searchParams.getAll('token').length !== 1) {
|
||||
throw Object.assign(new Error('DSH Web readiness URL is invalid'), { statusCode: 400 })
|
||||
}
|
||||
}
|
||||
if (!this.session.ready) {
|
||||
this.session = Object.freeze({ ...this.session, ready: true, readyUrl })
|
||||
void this.record('dsh.launch.ready')
|
||||
}
|
||||
return Object.freeze({ ready: true })
|
||||
}
|
||||
|
||||
verify() {
|
||||
if (this.session?.ready !== true) throw new Error('DSH lifecycle is not ready for verification')
|
||||
if (!this.session.verified) {
|
||||
this.session = Object.freeze({ ...this.session, verified: true })
|
||||
return this.record('dsh.launch.verified')
|
||||
}
|
||||
}
|
||||
|
||||
publish() {
|
||||
if (this.session?.verified !== true) throw new Error('DSH lifecycle is not verified for publication')
|
||||
if (!this.published) {
|
||||
this.published = true
|
||||
return this.record('platform.public-ready')
|
||||
}
|
||||
}
|
||||
|
||||
readiness() {
|
||||
const ready = this.session?.ready === true
|
||||
const verified = this.session?.verified === true
|
||||
return Object.freeze({
|
||||
generation: this.session?.id ?? null,
|
||||
publicReady: ready && verified && this.published,
|
||||
ready,
|
||||
readyUrl: ready ? this.session.readyUrl ?? null : null,
|
||||
verified,
|
||||
})
|
||||
}
|
||||
|
||||
async signal(sessionId, signal) {
|
||||
if (signal !== 'SIGTERM') throw Object.assign(new Error('lifecycle signal is invalid'), { statusCode: 400 })
|
||||
if (this.session === null || !matchesSecret(sessionId, this.session.id)) {
|
||||
throw Object.assign(new Error('DSH lifecycle session is invalid'), { statusCode: 409 })
|
||||
}
|
||||
const disposition = this.session.ready !== true || this.shuttingDown || await this.shouldTerminate()
|
||||
? 'terminate'
|
||||
: 'request-restart'
|
||||
await this.record('dsh.signal.disposition', { disposition, signal })
|
||||
return Object.freeze({ disposition })
|
||||
}
|
||||
|
||||
beginShutdown() {
|
||||
this.published = false
|
||||
this.shuttingDown = true
|
||||
}
|
||||
}
|
||||
|
||||
export function createDshLifecycleServer(broker) {
|
||||
return createServer(async (request, response) => {
|
||||
let pathname = 'invalid-url'
|
||||
try {
|
||||
pathname = new URL(request.url ?? '/', 'http://dsh-lifecycle.internal').pathname
|
||||
if (request.method !== 'POST') {
|
||||
send(response, 404, { error: 'not found' })
|
||||
return
|
||||
}
|
||||
if (pathname === '/v1/runtime/claim') {
|
||||
const body = exactObject(await jsonBody(request), ['launchToken'], 'claim request')
|
||||
send(response, 200, broker.claim(body.launchToken))
|
||||
} else if (pathname === '/v1/runtime/ready') {
|
||||
const value = await jsonBody(request)
|
||||
const body = Object.hasOwn(value, 'readyUrl')
|
||||
? exactObject(value, ['readyUrl', 'sessionId'], 'ready request')
|
||||
: exactObject(value, ['sessionId'], 'ready request')
|
||||
send(response, 200, broker.ready(body.sessionId, body.readyUrl ?? null))
|
||||
} else if (pathname === '/v1/runtime/readiness') {
|
||||
exactObject(await jsonBody(request), [], 'readiness request')
|
||||
send(response, 200, broker.readiness())
|
||||
} else if (pathname === '/v1/runtime/signal') {
|
||||
const body = exactObject(await jsonBody(request), ['sessionId', 'signal'], 'signal request')
|
||||
send(response, 200, await broker.signal(body.sessionId, body.signal))
|
||||
} else send(response, 404, { error: 'not found' })
|
||||
} catch (error) {
|
||||
await broker.record('dsh.lifecycle-request.failed', {
|
||||
error,
|
||||
method: request.method ?? null,
|
||||
pathname,
|
||||
level: error?.statusCode === 409 ? 'warning' : 'error',
|
||||
})
|
||||
send(response, Number.isInteger(error?.statusCode) ? error.statusCode : 400, {
|
||||
error: error instanceof Error ? error.message : 'invalid lifecycle request',
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export async function listenDshLifecycle(server, socketPath) {
|
||||
await mkdir(dirname(socketPath), { recursive: true })
|
||||
await rm(socketPath, { force: true })
|
||||
await new Promise((resolve, reject) => {
|
||||
server.once('error', reject)
|
||||
server.listen(socketPath, resolve)
|
||||
})
|
||||
await chmod(socketPath, 0o600)
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import { lstat, readFile, readdir, rm } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { parseDeploymentRecord } from '../../lib/deployment-contracts.mjs'
|
||||
|
||||
async function jsonOptional(path) {
|
||||
try { return JSON.parse(await readFile(path, 'utf8')) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function files(root, suffix = '') {
|
||||
try { return (await readdir(root)).filter(name => !name.startsWith('.') && name.endsWith(suffix)).sort() } catch (error) {
|
||||
if (error?.code === 'ENOENT') return []
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function removeUnmarked(root, retained, reserved = new Set()) {
|
||||
const removed = []
|
||||
for (const name of await files(root)) {
|
||||
if (reserved.has(name) || retained.has(name)) continue
|
||||
const path = join(root, name)
|
||||
const details = await lstat(path)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) continue
|
||||
await rm(path, { recursive: true, force: true })
|
||||
removed.push(name)
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
async function deploymentRecords(root) {
|
||||
const records = new Map()
|
||||
for (const name of await files(root, '.json')) {
|
||||
const record = parseDeploymentRecord(await readFile(join(root, name)))
|
||||
records.set(record.id, record)
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
function recordIds(value, output) {
|
||||
if (typeof value === 'string' && value.startsWith('deployment-record-')) output.add(value)
|
||||
else if (Array.isArray(value)) for (const entry of value) recordIds(entry, output)
|
||||
else if (value !== null && typeof value === 'object') for (const entry of Object.values(value)) recordIds(entry, output)
|
||||
}
|
||||
|
||||
function heldRecords(channel, records, roots) {
|
||||
const held = [...(channel?.holds ?? []), ...(channel?.experimentalBlocked === null || channel?.experimentalBlocked === undefined
|
||||
? []
|
||||
: [channel.experimentalBlocked])]
|
||||
for (const hold of held) {
|
||||
for (const record of records.values()) {
|
||||
if (
|
||||
record.dshVersion === hold.dshVersion
|
||||
&& (hold.environmentVersion === null || hold.environmentVersion === undefined || record.environmentVersion === hold.environmentVersion)
|
||||
) roots.add(record.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class PlatformGarbageCollector {
|
||||
constructor({ paths, deployments, snapshots }) {
|
||||
this.paths = paths
|
||||
this.deployments = deployments
|
||||
this.snapshots = snapshots
|
||||
}
|
||||
|
||||
async removeSnapshots(retained) {
|
||||
if (this.snapshots?.remove === undefined) throw new Error('privileged snapshot cleanup is unavailable')
|
||||
const removed = []
|
||||
for (const name of await files(join(this.paths.snapshotsRoot, 'versions'))) {
|
||||
if (retained.has(name)) continue
|
||||
const details = await lstat(join(this.paths.snapshotsRoot, 'versions', name))
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) continue
|
||||
if (await this.snapshots.remove(name)) removed.push(name)
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
async roots(records) {
|
||||
const roots = new Set()
|
||||
const slots = await this.deployments.state()
|
||||
if (slots.current !== null) roots.add(slots.current)
|
||||
if (slots.previous !== null) roots.add(slots.previous)
|
||||
recordIds(await jsonOptional(this.deployments.activationPath), roots)
|
||||
const transaction = await jsonOptional(join(this.paths.updaterStateRoot, 'transaction.json'))
|
||||
recordIds(transaction, roots)
|
||||
heldRecords(await jsonOptional(join(this.paths.updaterStateRoot, 'channel.json')), records, roots)
|
||||
return { roots, transaction }
|
||||
}
|
||||
|
||||
async collectDeployments() {
|
||||
const records = await deploymentRecords(this.deployments.recordsRoot)
|
||||
const { roots, transaction } = await this.roots(records)
|
||||
const retained = {
|
||||
environments: new Set(),
|
||||
pristine: new Set(),
|
||||
runtimes: new Set(),
|
||||
systemPlugins: new Set(),
|
||||
snapshots: new Set(),
|
||||
}
|
||||
for (const id of roots) {
|
||||
const record = records.get(id)
|
||||
if (record === undefined) continue
|
||||
for (const [field, output] of [
|
||||
['environment', retained.environments],
|
||||
['pristine', retained.pristine],
|
||||
['runtime', retained.runtimes],
|
||||
['systemPlugins', retained.systemPlugins],
|
||||
]) if (record[field].storage === 'store') output.add(record[field].id)
|
||||
if (record.snapshotId !== null) retained.snapshots.add(record.snapshotId)
|
||||
}
|
||||
if (typeof transaction?.snapshotId === 'string') retained.snapshots.add(transaction.snapshotId)
|
||||
|
||||
const reserved = new Set(['current', 'previous', 'versions'])
|
||||
const removed = {
|
||||
environments: await removeUnmarked(this.paths.environmentsRoot, retained.environments, reserved),
|
||||
pristine: await removeUnmarked(this.paths.pristineRoot, retained.pristine, reserved),
|
||||
runtimes: await removeUnmarked(this.paths.runtimesRoot, retained.runtimes, reserved),
|
||||
systemPlugins: await removeUnmarked(this.paths.systemPluginsRoot, retained.systemPlugins, reserved),
|
||||
snapshots: await this.removeSnapshots(retained.snapshots),
|
||||
records: [],
|
||||
}
|
||||
for (const [id] of records) {
|
||||
if (!roots.has(id)) {
|
||||
await rm(join(this.deployments.recordsRoot, `${id}.json`), { force: true })
|
||||
removed.records.push(id)
|
||||
}
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
async cleanCache() {
|
||||
const removed = []
|
||||
for (const name of await files(this.paths.downloadsRoot)) {
|
||||
await rm(join(this.paths.downloadsRoot, name), { recursive: true, force: true })
|
||||
removed.push(name)
|
||||
}
|
||||
return removed
|
||||
}
|
||||
|
||||
async collect() {
|
||||
return Object.freeze({
|
||||
deployments: await this.collectDeployments(),
|
||||
cache: await this.cleanCache(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,572 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { once } from 'node:events'
|
||||
import { request } from 'node:http'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { artifactForReference, parseComponentManifest, parseEnvironmentManifest } from '../../lib/contracts.mjs'
|
||||
import { exactKeys, parseJsonDocument, plainObject, TrustError } from '../../lib/validation.mjs'
|
||||
|
||||
const STARTUP_HEALTH_RETRY_MS = 100
|
||||
|
||||
function delay(milliseconds) {
|
||||
return new Promise(resolveDelay => {
|
||||
const timer = setTimeout(resolveDelay, milliseconds)
|
||||
timer.unref?.()
|
||||
})
|
||||
}
|
||||
|
||||
function terminate(child, graceMs = 5_000) {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
|
||||
return new Promise(resolveTerminate => {
|
||||
const timer = setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL')
|
||||
}, graceMs)
|
||||
child.once('exit', () => {
|
||||
clearTimeout(timer)
|
||||
resolveTerminate()
|
||||
})
|
||||
child.kill('SIGTERM')
|
||||
})
|
||||
}
|
||||
|
||||
function runCommand(spec, options, captureOutput = true) {
|
||||
return new Promise((resolveCommand, reject) => {
|
||||
const child = options.spawnImpl(spec.executable, spec.args, {
|
||||
env: options.environment,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
})
|
||||
if (captureOutput) options.capture(child, options.componentId, options.logging)
|
||||
else {
|
||||
child.stdout?.resume()
|
||||
child.stderr?.resume()
|
||||
}
|
||||
const timer = setTimeout(() => child.kill('SIGKILL'), spec.timeoutSeconds * 1000)
|
||||
child.once('error', reject)
|
||||
child.once('exit', (code, signal) => {
|
||||
clearTimeout(timer)
|
||||
if (code === 0) resolveCommand()
|
||||
else reject(new Error(`${options.componentId} command failed (code=${String(code)}, signal=${String(signal)})`))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function probeHttp(health) {
|
||||
return new Promise(resolveProbe => {
|
||||
const req = request({ hostname: health.host, port: health.port, path: health.path }, response => {
|
||||
response.resume()
|
||||
resolveProbe((response.statusCode ?? 500) < 500)
|
||||
})
|
||||
req.setTimeout(Math.min(health.intervalSeconds * 1000, 1_000), () => req.destroy())
|
||||
req.once('error', () => resolveProbe(false))
|
||||
req.end()
|
||||
})
|
||||
}
|
||||
|
||||
async function waitForHealth(component, running, options) {
|
||||
if (component.health === null) return
|
||||
const deadline = Date.now() + component.health.timeoutSeconds * 1000
|
||||
while (Date.now() < deadline) {
|
||||
if (running?.child.exitCode !== null) throw new Error(`${component.id} exited before health check passed`)
|
||||
const healthy = component.health.type === 'http'
|
||||
? await probeHttp(component.health)
|
||||
: await runCommand(component.health.command, options, false).then(() => true, () => false)
|
||||
if (healthy) return
|
||||
await delay(Math.min(component.health.intervalSeconds * 1000, STARTUP_HEALTH_RETRY_MS))
|
||||
}
|
||||
if (running?.child !== undefined && running.child.exitCode !== null) {
|
||||
throw new Error(`${component.id} exited before health check passed`)
|
||||
}
|
||||
throw new Error(`${component.id} health check timed out`)
|
||||
}
|
||||
|
||||
function defaultCapture(child, componentId) {
|
||||
for (const [stream, source] of [[child.stdout, 'stdout'], [child.stderr, 'stderr']]) {
|
||||
stream?.on('data', chunk => process.stdout.write(`[${componentId}:${source}] ${chunk.toString()}`))
|
||||
}
|
||||
}
|
||||
|
||||
export class ComponentExitError extends Error {
|
||||
constructor(componentId, code, signal) {
|
||||
super(`${componentId} exited unexpectedly (code=${String(code)}, signal=${String(signal)})`)
|
||||
this.name = 'ComponentExitError'
|
||||
this.componentId = componentId
|
||||
this.exitCode = code
|
||||
this.signal = signal
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadEnvironment(root) {
|
||||
const environmentRoot = resolve(root)
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(environmentRoot, 'environment.manifest.json')))
|
||||
const components = []
|
||||
for (const reference of manifest.components) {
|
||||
const artifact = artifactForReference(manifest, reference)
|
||||
const component = parseComponentManifest(await readFile(join(environmentRoot, 'artifacts', artifact.id)))
|
||||
if (component.id !== reference.id) {
|
||||
throw new Error(`component ${reference.id} differs from the Environment manifest`)
|
||||
}
|
||||
components.push(component)
|
||||
}
|
||||
return Object.freeze({ root: environmentRoot, manifest, components: Object.freeze(components) })
|
||||
}
|
||||
|
||||
export async function loadControlPlane(root) {
|
||||
const controlPlaneRoot = resolve(root)
|
||||
const definition = parseJsonDocument(await readFile(join(controlPlaneRoot, 'definition.json')), 'control plane')
|
||||
exactKeys(definition, ['components', 'schema'], 'control plane')
|
||||
if (definition.schema !== 1) throw new TrustError('control plane schema must be 1')
|
||||
if (!Array.isArray(definition.components) || definition.components.length === 0) {
|
||||
throw new TrustError('control plane components must be a non-empty array')
|
||||
}
|
||||
const components = []
|
||||
const ids = new Set()
|
||||
for (const [index, value] of definition.components.entries()) {
|
||||
const reference = plainObject(value, `control plane components[${String(index)}]`)
|
||||
exactKeys(reference, ['id', 'source'], `control plane components[${String(index)}]`)
|
||||
if (typeof reference.id !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,127}$/.test(reference.id)) {
|
||||
throw new TrustError('control plane component ID is invalid')
|
||||
}
|
||||
if (ids.has(reference.id)) throw new TrustError('control plane component IDs must be unique')
|
||||
if (typeof reference.source !== 'string' || reference.source.startsWith('/')) {
|
||||
throw new TrustError('control plane component source is invalid')
|
||||
}
|
||||
const source = resolve(controlPlaneRoot, reference.source)
|
||||
if (source === controlPlaneRoot || !source.startsWith(`${controlPlaneRoot}/`)) {
|
||||
throw new TrustError('control plane component source escapes its root')
|
||||
}
|
||||
const component = parseComponentManifest(await readFile(source))
|
||||
if (component.id !== reference.id) throw new TrustError(`control plane component ${reference.id} differs from its manifest`)
|
||||
ids.add(reference.id)
|
||||
components.push(component)
|
||||
}
|
||||
return Object.freeze({
|
||||
root: controlPlaneRoot,
|
||||
manifest: Object.freeze({ version: null }),
|
||||
components: Object.freeze(components),
|
||||
})
|
||||
}
|
||||
|
||||
export class EnvironmentRunner {
|
||||
constructor({
|
||||
environmentRoot,
|
||||
spawnImpl = spawn,
|
||||
capture = defaultCapture,
|
||||
loader = loadEnvironment,
|
||||
prepareService = async () => ({ environment: {}, release: () => {} }),
|
||||
recoverableComponents = [],
|
||||
recoveryDelaysMs = [0, 2_000, 5_000],
|
||||
report = () => {},
|
||||
startupGroups = null,
|
||||
}) {
|
||||
if (!Array.isArray(recoverableComponents)
|
||||
|| recoverableComponents.some(value => typeof value !== 'string' || value.length === 0)) {
|
||||
throw new Error('Recoverable component IDs must be non-empty strings')
|
||||
}
|
||||
if (!Array.isArray(recoveryDelaysMs) || recoveryDelaysMs.length === 0
|
||||
|| recoveryDelaysMs.some(value => !Number.isFinite(value) || value < 0)) {
|
||||
throw new Error('Component recovery delays must be non-negative durations')
|
||||
}
|
||||
if (startupGroups !== null && (!Array.isArray(startupGroups) || startupGroups.length === 0
|
||||
|| startupGroups.some(group => !Array.isArray(group) || group.length === 0
|
||||
|| group.some(value => typeof value !== 'string' || value.length === 0)))) {
|
||||
throw new Error('Startup groups must contain non-empty component ID lists')
|
||||
}
|
||||
this.environmentRoot = environmentRoot
|
||||
this.spawnImpl = spawnImpl
|
||||
this.capture = capture
|
||||
this.loader = loader
|
||||
this.prepareService = prepareService
|
||||
this.recoverableComponents = new Set(recoverableComponents)
|
||||
this.recoveryDelaysMs = Object.freeze([...recoveryDelaysMs])
|
||||
this.report = report
|
||||
this.startupGroups = startupGroups === null ? null : Object.freeze(
|
||||
startupGroups.map(group => Object.freeze([...group])),
|
||||
)
|
||||
this.running = []
|
||||
this.environment = undefined
|
||||
this.operation = Promise.resolve()
|
||||
this.stopping = false
|
||||
this.componentRecoveries = new Map()
|
||||
this.fatalListeners = new Set()
|
||||
this.fatal = new Promise(resolveFatal => { this.resolveFatal = resolveFatal })
|
||||
}
|
||||
|
||||
onFatal(listener) {
|
||||
this.fatalListeners.add(listener)
|
||||
return () => this.fatalListeners.delete(listener)
|
||||
}
|
||||
|
||||
emitFatal(error) {
|
||||
this.resolveFatal(error)
|
||||
for (const listener of this.fatalListeners) {
|
||||
try { listener(error) } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
emitLifecycle(message, fields = {}) {
|
||||
void Promise.resolve().then(() => this.report(message, fields)).catch(() => {})
|
||||
}
|
||||
|
||||
serialized(operation) {
|
||||
const result = this.operation.then(operation, operation)
|
||||
this.operation = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
recoverable(component) {
|
||||
return this.recoverableComponents.has(component.id) && component.type === 'service'
|
||||
}
|
||||
|
||||
scheduleRecovery(component, originalError) {
|
||||
if (this.stopping || !this.recoverable(component) || this.componentRecoveries.has(component.id)) return
|
||||
const recover = () => this.recoverComponentUnlocked(component, originalError)
|
||||
const pending = this.serialized(recover).finally(() => {
|
||||
if (this.componentRecoveries.get(component.id) === pending) this.componentRecoveries.delete(component.id)
|
||||
})
|
||||
this.componentRecoveries.set(component.id, pending)
|
||||
void pending.catch(() => {})
|
||||
}
|
||||
|
||||
async recoverComponentUnlocked(component, originalError) {
|
||||
const stale = this.running.find(value => value.component.id === component.id)
|
||||
if (stale !== undefined) {
|
||||
try { await this.stopComponentUnlocked(stale) } catch {}
|
||||
}
|
||||
for (let index = 0; index < this.recoveryDelaysMs.length; index += 1) {
|
||||
if (this.stopping) return
|
||||
const delayMs = this.recoveryDelaysMs[index]
|
||||
if (delayMs > 0) await delay(delayMs)
|
||||
if (this.stopping) return
|
||||
const attempt = index + 1
|
||||
this.emitLifecycle('component.recovery.started', {
|
||||
componentId: component.id,
|
||||
attempt,
|
||||
maxAttempts: this.recoveryDelaysMs.length,
|
||||
})
|
||||
try {
|
||||
await this.startComponentUnlocked(component, true)
|
||||
this.emitLifecycle('component.recovery.completed', {
|
||||
componentId: component.id,
|
||||
attempt,
|
||||
maxAttempts: this.recoveryDelaysMs.length,
|
||||
})
|
||||
return
|
||||
} catch (error) {
|
||||
this.emitLifecycle('component.recovery.failed', {
|
||||
componentId: component.id,
|
||||
attempt,
|
||||
maxAttempts: this.recoveryDelaysMs.length,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
level: 'warning',
|
||||
})
|
||||
}
|
||||
}
|
||||
this.emitLifecycle('component.isolated', {
|
||||
componentId: component.id,
|
||||
error: originalError instanceof Error ? originalError.message : String(originalError),
|
||||
level: 'error',
|
||||
})
|
||||
}
|
||||
|
||||
commandOptions(component) {
|
||||
return {
|
||||
spawnImpl: this.spawnImpl,
|
||||
capture: this.capture,
|
||||
componentId: component.id,
|
||||
logging: component.logging,
|
||||
environment: { ...process.env, ...component.environment },
|
||||
}
|
||||
}
|
||||
|
||||
async phase(component, name) {
|
||||
const spec = component.lifecycle[name]
|
||||
if (spec !== null) await runCommand(spec, this.commandOptions(component))
|
||||
}
|
||||
|
||||
startupSequence() {
|
||||
if (this.startupGroups === null) return this.environment.components.map(component => [component])
|
||||
const components = new Map(this.environment.components.map(component => [component.id, component]))
|
||||
const seen = new Set()
|
||||
const groups = this.startupGroups.map(group => group.map(componentId => {
|
||||
const component = components.get(componentId)
|
||||
if (component === undefined) throw new Error(`Startup group component ${componentId} does not exist`)
|
||||
if (seen.has(componentId)) throw new Error(`Startup group component ${componentId} is duplicated`)
|
||||
seen.add(componentId)
|
||||
return component
|
||||
}))
|
||||
const missing = this.environment.components.filter(component => !seen.has(component.id))
|
||||
if (missing.length > 0) {
|
||||
throw new Error(`Startup groups omit components: ${missing.map(component => component.id).join(', ')}`)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
start() {
|
||||
return this.serialized(() => this.startUnlocked())
|
||||
}
|
||||
|
||||
async startUnlocked() {
|
||||
if (this.running.length > 0) throw new Error('Environment is already running')
|
||||
this.environment = await this.loader(this.environmentRoot)
|
||||
try {
|
||||
const startupGroups = this.startupSequence()
|
||||
const recoveries = []
|
||||
for (const component of this.environment.components) await this.phase(component, 'prepare')
|
||||
for (const group of startupGroups) {
|
||||
const results = await Promise.allSettled(group.map(component => this.startComponentUnlocked(component)))
|
||||
const failures = []
|
||||
for (const [index, result] of results.entries()) {
|
||||
if (result.status === 'fulfilled') continue
|
||||
const component = group[index]
|
||||
if (!this.recoverable(component)) {
|
||||
failures.push(result.reason)
|
||||
continue
|
||||
}
|
||||
this.emitLifecycle('component.isolated', {
|
||||
componentId: component.id,
|
||||
error: result.reason instanceof Error ? result.reason.message : String(result.reason),
|
||||
level: 'error',
|
||||
})
|
||||
recoveries.push({ component, error: result.reason })
|
||||
}
|
||||
if (failures.length === 1) throw failures[0]
|
||||
if (failures.length > 1) throw new AggregateError(failures, 'Startup group failed')
|
||||
}
|
||||
for (const recovery of recoveries) this.scheduleRecovery(recovery.component, recovery.error)
|
||||
return this.status()
|
||||
} catch (error) {
|
||||
try {
|
||||
await this.stopUnlocked()
|
||||
} catch (cleanupError) {
|
||||
this.emitLifecycle('environment.cleanup.failed', {
|
||||
error: cleanupError,
|
||||
level: 'warning',
|
||||
originalError: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async startComponentUnlocked(component, prepare = false) {
|
||||
if (this.running.some(running => running.component.id === component.id)) {
|
||||
throw new Error(`component ${component.id} is already running`)
|
||||
}
|
||||
const startedAt = Date.now()
|
||||
this.emitLifecycle('component.starting', { componentId: component.id, componentType: component.type })
|
||||
let running
|
||||
let releaseService = () => {}
|
||||
try {
|
||||
if (prepare) await this.phase(component, 'prepare')
|
||||
await this.phase(component, 'preStart')
|
||||
if (component.type === 'service') {
|
||||
const options = this.commandOptions(component)
|
||||
const launch = await this.prepareService(component)
|
||||
let released = false
|
||||
const release = () => {
|
||||
if (released) return
|
||||
released = true
|
||||
launch.release?.()
|
||||
}
|
||||
releaseService = release
|
||||
const child = this.spawnImpl(component.command.executable, component.command.args, {
|
||||
env: { ...options.environment, ...launch.environment },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
})
|
||||
this.capture(child, component.id, component.logging)
|
||||
await Promise.race([
|
||||
once(child, 'spawn'),
|
||||
once(child, 'error').then(([error]) => { throw error }),
|
||||
])
|
||||
running = { component, child, ready: false, release }
|
||||
this.emitLifecycle('component.spawned', { componentId: component.id, pid: child.pid ?? null })
|
||||
child.once('exit', (code, signal) => {
|
||||
release()
|
||||
if (running.ready && !this.stopping && this.running.includes(running)) {
|
||||
const error = new ComponentExitError(component.id, code, signal)
|
||||
this.emitLifecycle('component.exited', {
|
||||
componentId: component.id,
|
||||
code,
|
||||
signal,
|
||||
error: error.message,
|
||||
level: 'error',
|
||||
})
|
||||
if (this.recoverable(component)) this.scheduleRecovery(component, error)
|
||||
else this.emitFatal(error)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
await runCommand(component.command, this.commandOptions(component))
|
||||
running = { component, child: undefined }
|
||||
}
|
||||
this.running.push(running)
|
||||
this.running.sort((left, right) => (
|
||||
this.environment.components.indexOf(left.component) - this.environment.components.indexOf(right.component)
|
||||
))
|
||||
await waitForHealth(component, running, this.commandOptions(component))
|
||||
await this.phase(component, 'postStart')
|
||||
if (running.child !== undefined && running.child.exitCode !== null) {
|
||||
throw new Error(`${component.id} exited before startup completed`)
|
||||
}
|
||||
running.ready = true
|
||||
this.emitLifecycle('component.ready', { componentId: component.id, elapsedMs: Date.now() - startedAt })
|
||||
return running
|
||||
} catch (error) {
|
||||
this.emitLifecycle('component.start.failed', {
|
||||
componentId: component.id,
|
||||
elapsedMs: Date.now() - startedAt,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
level: 'error',
|
||||
})
|
||||
if (running !== undefined) {
|
||||
try {
|
||||
await this.stopComponentUnlocked(running)
|
||||
} catch (cleanupError) {
|
||||
this.emitLifecycle('component.cleanup.failed', {
|
||||
componentId: component.id,
|
||||
error: cleanupError,
|
||||
level: 'warning',
|
||||
originalError: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
} else if (component.type === 'service') {
|
||||
releaseService()
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async stopComponentUnlocked(running) {
|
||||
const { component, child } = running
|
||||
const failures = []
|
||||
this.emitLifecycle('component.stopping', { componentId: component.id, pid: child?.pid ?? null })
|
||||
this.running = this.running.filter(value => value !== running)
|
||||
for (const phase of ['preStop', 'stop']) {
|
||||
try { await this.phase(component, phase) } catch (error) { failures.push(error) }
|
||||
}
|
||||
if (child !== undefined) {
|
||||
try { await terminate(child) } catch (error) { failures.push(error) }
|
||||
}
|
||||
try { await this.phase(component, 'postStop') } catch (error) { failures.push(error) }
|
||||
if (failures.length > 0) {
|
||||
this.emitLifecycle('component.stop.failed', {
|
||||
componentId: component.id,
|
||||
errors: failures.map(error => error instanceof Error ? error.message : String(error)),
|
||||
level: 'error',
|
||||
})
|
||||
throw new AggregateError(failures, `component ${component.id} shutdown failed`)
|
||||
}
|
||||
this.emitLifecycle('component.stopped', { componentId: component.id })
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.stopping = true
|
||||
return this.serialized(() => this.stopUnlocked())
|
||||
}
|
||||
|
||||
async stopUnlocked() {
|
||||
this.stopping = true
|
||||
const failures = []
|
||||
for (const running of [...this.running].reverse()) {
|
||||
try { await this.stopComponentUnlocked(running) } catch (error) { failures.push(error) }
|
||||
}
|
||||
this.running = []
|
||||
this.stopping = false
|
||||
if (failures.length > 0) throw new AggregateError(failures, 'Environment shutdown failed')
|
||||
}
|
||||
|
||||
reload() {
|
||||
return this.serialized(async () => {
|
||||
await this.stopUnlocked()
|
||||
return this.startUnlocked()
|
||||
})
|
||||
}
|
||||
|
||||
suspend(componentId) {
|
||||
return this.serialized(async () => {
|
||||
const running = this.running.find(value => value.component.id === componentId)
|
||||
if (running === undefined) throw new Error(`component ${componentId} is not running`)
|
||||
if (running.component.type !== 'service') throw new Error(`component ${componentId} is not a service`)
|
||||
await this.stopComponentUnlocked(running)
|
||||
return this.status()
|
||||
})
|
||||
}
|
||||
|
||||
pause(componentId, { allowMissing = false } = {}) {
|
||||
return this.serialized(async () => {
|
||||
const running = this.running.find(value => value.component.id === componentId)
|
||||
const component = running?.component ?? this.environment?.components.find(value => value.id === componentId)
|
||||
if (component === undefined) {
|
||||
if (allowMissing) return this.status()
|
||||
throw new Error(`component ${componentId} does not exist`)
|
||||
}
|
||||
if (component.type !== 'service') throw new Error(`component ${componentId} is not a service`)
|
||||
if (running === undefined) return this.status()
|
||||
await this.stopComponentUnlocked(running)
|
||||
return this.status()
|
||||
})
|
||||
}
|
||||
|
||||
resume(componentId) {
|
||||
return this.serialized(async () => {
|
||||
if (this.environment === undefined) throw new Error('Environment is not loaded')
|
||||
const component = this.environment.components.find(value => value.id === componentId)
|
||||
if (component === undefined) throw new Error(`component ${componentId} does not exist`)
|
||||
if (this.running.some(value => value.component.id === componentId)) return this.status()
|
||||
await this.startComponentUnlocked(component, true)
|
||||
return this.status()
|
||||
})
|
||||
}
|
||||
|
||||
restart(componentId, { beforeStart, recoverStart = false, onStartFailure } = {}) {
|
||||
return this.serialized(async () => {
|
||||
if (this.environment === undefined) this.environment = await this.loader(this.environmentRoot)
|
||||
const running = this.running.find(value => value.component.id === componentId)
|
||||
const component = running?.component
|
||||
?? this.environment.components.find(value => value.id === componentId)
|
||||
if (component === undefined) throw new Error(`component ${componentId} does not exist`)
|
||||
if (component.type !== 'service') throw new Error(`component ${componentId} is not a service`)
|
||||
if (running !== undefined) await this.stopComponentUnlocked(running)
|
||||
try {
|
||||
await beforeStart?.()
|
||||
await this.startComponentUnlocked(component, true)
|
||||
return this.status()
|
||||
} catch (error) {
|
||||
const failures = [error]
|
||||
try { await onStartFailure?.(error) } catch (recoveryError) { failures.push(recoveryError) }
|
||||
if (recoverStart && failures.length === 1) {
|
||||
try { await this.startComponentUnlocked(component, true) } catch (recoveryError) { failures.push(recoveryError) }
|
||||
}
|
||||
if (failures.length > 1) throw new AggregateError(failures, `component ${componentId} restart recovery failed`)
|
||||
throw error
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
health() {
|
||||
return this.serialized(async () => {
|
||||
const components = await Promise.all((this.environment?.components ?? []).map(async component => {
|
||||
const running = this.running.find(value => value.component.id === component.id)
|
||||
if (running === undefined) return { id: component.id, running: false, healthy: false }
|
||||
const healthy = component.health === null
|
||||
? running.child === undefined || running.child.exitCode === null
|
||||
: component.health.type === 'http'
|
||||
? await probeHttp(component.health)
|
||||
: await runCommand(component.health.command, this.commandOptions(component), false).then(() => true, () => false)
|
||||
return { id: component.id, running: true, healthy }
|
||||
}))
|
||||
return { healthy: components.length > 0 && components.every(component => component.healthy), components }
|
||||
})
|
||||
}
|
||||
|
||||
status() {
|
||||
return Object.freeze({
|
||||
environmentVersion: this.environment?.manifest.version ?? null,
|
||||
components: this.running.map(({ component, child }) => ({
|
||||
id: component.id,
|
||||
type: component.type,
|
||||
pid: child?.pid ?? null,
|
||||
})),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
export class BootstrapRuntime {
|
||||
constructor({
|
||||
controlPlane,
|
||||
environment,
|
||||
validateDeployment = async () => {},
|
||||
prepareDeployment = async () => {},
|
||||
beforeReady = async () => {},
|
||||
onEnvironmentVerified = async () => {},
|
||||
onDshAvailable = async () => {},
|
||||
onEnvironmentFatal = async () => {},
|
||||
onDshRecovered = async () => {},
|
||||
ownsDshLifecycle = async () => false,
|
||||
recoveryDelaysMs = [0, 2_000, 5_000],
|
||||
sleep = (milliseconds, signal) => new Promise(resolve => {
|
||||
if (milliseconds === 0 || signal?.aborted) return resolve()
|
||||
const timer = setTimeout(resolve, milliseconds)
|
||||
timer.unref?.()
|
||||
signal?.addEventListener('abort', () => { clearTimeout(timer); resolve() }, { once: true })
|
||||
}),
|
||||
report = async () => {},
|
||||
}) {
|
||||
if (!Array.isArray(recoveryDelaysMs) || recoveryDelaysMs.length !== 3
|
||||
|| recoveryDelaysMs.some(value => !Number.isFinite(value) || value < 0)) {
|
||||
throw new Error('DSH recovery delays must contain three non-negative durations')
|
||||
}
|
||||
this.controlPlane = controlPlane
|
||||
this.environment = environment
|
||||
this.fatal = controlPlane.fatal
|
||||
this.recoveryMode = null
|
||||
this.startupComplete = false
|
||||
this.dshLifecycle = Object.freeze({
|
||||
state: 'starting', action: 'start', taskId: null, attempt: 0, maxAttempts: 3,
|
||||
error: null, updatedAt: new Date().toISOString(),
|
||||
})
|
||||
this.recovery = Promise.resolve()
|
||||
this.recoveryAbort = new AbortController()
|
||||
this.validateDeployment = validateDeployment
|
||||
this.prepareDeployment = prepareDeployment
|
||||
this.beforeReady = beforeReady
|
||||
this.onEnvironmentVerified = onEnvironmentVerified
|
||||
this.onDshAvailable = onDshAvailable
|
||||
this.onEnvironmentFatal = onEnvironmentFatal
|
||||
this.onDshRecovered = onDshRecovered
|
||||
this.ownsDshLifecycle = ownsDshLifecycle
|
||||
this.recoveryDelaysMs = Object.freeze([...recoveryDelaysMs])
|
||||
this.sleep = sleep
|
||||
this.report = report
|
||||
const handleEnvironmentFatal = error => {
|
||||
const recover = () => this.handleEnvironmentFatal(error)
|
||||
this.recovery = this.recovery.then(recover, recover).catch(recoveryError => this.record(
|
||||
'environment.recovery.failed', { error: recoveryError, originalError: String(error) },
|
||||
))
|
||||
}
|
||||
if (typeof environment.onFatal === 'function') {
|
||||
this.offEnvironmentFatal = environment.onFatal(handleEnvironmentFatal)
|
||||
} else {
|
||||
void environment.fatal.then(handleEnvironmentFatal)
|
||||
this.offEnvironmentFatal = undefined
|
||||
}
|
||||
}
|
||||
|
||||
async lifecycleOwned() {
|
||||
try {
|
||||
return await this.ownsDshLifecycle()
|
||||
} catch (error) {
|
||||
await this.record('dsh.recovery-ownership.failed', { error, level: 'warning' })
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
async handleEnvironmentFatal(error) {
|
||||
if (error?.componentId !== 'dsh-runtime') return this.isolateEnvironmentFailure(error)
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
if (await this.lifecycleOwned()) {
|
||||
this.publishDshLifecycle({ state: 'recovering', action: 'auto-recover', attempt: 0, error: message })
|
||||
await this.record('dsh.recovery.deferred', { error, reason: 'deployment-transaction-active', level: 'warning' })
|
||||
return
|
||||
}
|
||||
for (let index = 0; index < this.recoveryDelaysMs.length; index += 1) {
|
||||
const attempt = index + 1
|
||||
await this.sleep(this.recoveryDelaysMs[index], this.recoveryAbort.signal)
|
||||
if (this.recoveryAbort.signal.aborted) return
|
||||
if (await this.lifecycleOwned()) {
|
||||
this.publishDshLifecycle({ state: 'recovering', action: 'auto-recover', attempt: attempt - 1, error: message })
|
||||
await this.record('dsh.recovery.deferred', { error, reason: 'deployment-transaction-active', level: 'warning' })
|
||||
return
|
||||
}
|
||||
this.publishDshLifecycle({ state: 'recovering', action: 'auto-recover', attempt, error: message })
|
||||
await this.record('dsh.recovery.attempt.started', { attempt, maxAttempts: this.recoveryDelaysMs.length })
|
||||
try {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
await this.environment.restart('dsh-runtime')
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
let recoveryStatusPublished = true
|
||||
try {
|
||||
await this.onDshRecovered()
|
||||
} catch (statusError) {
|
||||
recoveryStatusPublished = false
|
||||
await this.record('dsh.recovery-status.failed', { error: statusError, level: 'warning' })
|
||||
}
|
||||
if (recoveryStatusPublished) await this.onDshAvailable()
|
||||
await this.record('dsh.recovery.completed', { attempt, maxAttempts: this.recoveryDelaysMs.length })
|
||||
return
|
||||
} catch (recoveryError) {
|
||||
await this.record('dsh.recovery.attempt.failed', {
|
||||
attempt,
|
||||
error: recoveryError,
|
||||
maxAttempts: this.recoveryDelaysMs.length,
|
||||
level: 'warning',
|
||||
})
|
||||
}
|
||||
}
|
||||
await this.isolateEnvironmentFailure(error)
|
||||
}
|
||||
|
||||
async isolateEnvironmentFailure(error) {
|
||||
this.recoveryMode = error instanceof Error ? error.message : String(error)
|
||||
this.publishDshLifecycle({
|
||||
state: 'failed', action: null,
|
||||
attempt: error?.componentId === 'dsh-runtime' ? this.recoveryDelaysMs.length : 0,
|
||||
error: this.recoveryMode,
|
||||
})
|
||||
try {
|
||||
await this.onEnvironmentFatal(error)
|
||||
} catch (reportError) {
|
||||
await this.record('environment.recovery-report.failed', { error: reportError, originalError: String(error) })
|
||||
}
|
||||
try {
|
||||
await this.environment.stop()
|
||||
} catch (stopError) {
|
||||
await this.record('environment.recovery-stop.failed', { error: stopError, originalError: String(error) })
|
||||
}
|
||||
}
|
||||
|
||||
record(message, fields = {}) {
|
||||
return Promise.resolve().then(() => this.report(message, fields)).catch(() => {})
|
||||
}
|
||||
|
||||
publishDshLifecycle(value) {
|
||||
this.dshLifecycle = Object.freeze({
|
||||
...this.dshLifecycle,
|
||||
...value,
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
return this.dshLifecycle
|
||||
}
|
||||
|
||||
async stopControlPlane(phase, cause) {
|
||||
try {
|
||||
await this.controlPlane.stop()
|
||||
} catch (error) {
|
||||
await this.record('control-plane.cleanup.failed', {
|
||||
error,
|
||||
phase,
|
||||
originalError: cause instanceof Error ? cause.message : String(cause),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async stopEnvironment(phase, cause) {
|
||||
try {
|
||||
await this.environment.stop()
|
||||
} catch (error) {
|
||||
await this.record('environment.cleanup.failed', {
|
||||
error,
|
||||
phase,
|
||||
originalError: cause instanceof Error ? cause.message : String(cause),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async startEnvironment({ onEnvironmentFailure, allowRecovery }) {
|
||||
try {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
await this.environment.start()
|
||||
await this.onEnvironmentVerified()
|
||||
} catch (error) {
|
||||
let retry
|
||||
try {
|
||||
retry = await onEnvironmentFailure?.(error)
|
||||
} catch (recoveryError) {
|
||||
const failure = new AggregateError([error, recoveryError], 'Deployment failed and its fallback could not be resolved')
|
||||
if (allowRecovery) {
|
||||
this.recoveryMode = failure.message
|
||||
return
|
||||
}
|
||||
throw failure
|
||||
}
|
||||
if (retry === true) {
|
||||
try {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
await this.environment.start()
|
||||
await this.onEnvironmentVerified()
|
||||
} catch (fallbackError) {
|
||||
const failure = new AggregateError([error, fallbackError], 'Deployment candidate and fallback both failed')
|
||||
if (allowRecovery) this.recoveryMode = failure.message
|
||||
else throw failure
|
||||
}
|
||||
} else if (allowRecovery) {
|
||||
this.recoveryMode = error instanceof Error ? error.message : 'Deployment failed to start'
|
||||
} else throw error
|
||||
}
|
||||
}
|
||||
|
||||
async start({ onEnvironmentFailure, allowRecovery = false } = {}) {
|
||||
const readiness = Promise.resolve().then(async () => {
|
||||
await this.controlPlane.start()
|
||||
await this.beforeReady()
|
||||
})
|
||||
const environment = Promise.resolve().then(() => this.startEnvironment({ onEnvironmentFailure, allowRecovery }))
|
||||
const results = await Promise.allSettled([readiness, environment])
|
||||
const failures = results.filter(result => result.status === 'rejected').map(result => result.reason)
|
||||
if (failures.length > 0) {
|
||||
const failure = failures.length === 1 ? failures[0] : new AggregateError(failures, 'Platform startup failed')
|
||||
await Promise.all([
|
||||
this.stopEnvironment('concurrent-start', failure),
|
||||
this.stopControlPlane('concurrent-start', failure),
|
||||
])
|
||||
throw failure
|
||||
}
|
||||
return this.status()
|
||||
}
|
||||
|
||||
async stop() {
|
||||
this.offEnvironmentFatal?.()
|
||||
this.recoveryAbort.abort()
|
||||
await this.recovery
|
||||
const failures = []
|
||||
try { await this.environment.stop() } catch (error) { failures.push(error) }
|
||||
try { await this.controlPlane.stop() } catch (error) { failures.push(error) }
|
||||
if (failures.length > 0) throw new AggregateError(failures, 'Bootstrap shutdown failed')
|
||||
}
|
||||
|
||||
async reload() {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
const status = await this.environment.reload()
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
if (this.environment.status().components.some(value => value.id === 'dsh-runtime')) {
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
await this.onDshAvailable()
|
||||
}
|
||||
return status
|
||||
}
|
||||
async suspend(componentId) {
|
||||
if (componentId !== 'dsh-runtime') return this.environment.suspend(componentId)
|
||||
this.publishDshLifecycle({ state: 'stopping', action: 'stop', attempt: 0, error: null })
|
||||
try {
|
||||
const status = await this.environment.suspend(componentId)
|
||||
this.publishDshLifecycle({ state: 'stopped', action: null, attempt: 0, error: null })
|
||||
return status
|
||||
} catch (error) {
|
||||
this.publishDshLifecycle({
|
||||
state: 'failed', action: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
async pause(componentId) {
|
||||
if (componentId !== 'dsh-runtime') return this.environment.pause(componentId)
|
||||
this.publishDshLifecycle({ state: 'stopping', action: 'stop', attempt: 0, error: null })
|
||||
try {
|
||||
const status = await this.environment.pause(componentId, { allowMissing: true })
|
||||
this.publishDshLifecycle({ state: 'stopped', action: null, attempt: 0, error: null })
|
||||
return status
|
||||
} catch (error) {
|
||||
this.publishDshLifecycle({
|
||||
state: 'failed', action: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
async resume(componentId, options) {
|
||||
if (componentId === 'dsh-runtime') this.publishDshLifecycle({ state: 'starting', action: 'start', attempt: 0, error: null })
|
||||
try {
|
||||
if (componentId === 'dsh-runtime') {
|
||||
if (options?.skipValidation !== true) await this.validateDeployment()
|
||||
if (options?.skipPreparation !== true) await this.prepareDeployment()
|
||||
}
|
||||
const status = await this.environment.resume(componentId)
|
||||
if (componentId === 'dsh-runtime') {
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
await this.onDshAvailable()
|
||||
}
|
||||
return status
|
||||
} catch (error) {
|
||||
if (componentId === 'dsh-runtime') this.enterRecovery(error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
async restart(componentId, options) {
|
||||
if (componentId === 'dsh-runtime') this.publishDshLifecycle({ state: 'restarting', action: 'restart', attempt: 0, error: null })
|
||||
try {
|
||||
if (componentId === 'dsh-runtime') {
|
||||
if (options?.skipValidation !== true) await this.validateDeployment()
|
||||
if (options?.skipPreparation !== true) await this.prepareDeployment()
|
||||
}
|
||||
const status = await this.environment.restart(componentId, options)
|
||||
if (componentId === 'dsh-runtime') {
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
await this.onDshAvailable()
|
||||
}
|
||||
return status
|
||||
} catch (error) {
|
||||
if (componentId === 'dsh-runtime') this.enterRecovery(error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
health() { return this.environment.health() }
|
||||
|
||||
markStartupComplete() {
|
||||
this.startupComplete = true
|
||||
if (this.recoveryMode === null && this.environment.status().components.some(value => value.id === 'dsh-runtime')) {
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
} else if (this.recoveryMode !== null) {
|
||||
this.publishDshLifecycle({ state: 'failed', action: null, error: this.recoveryMode })
|
||||
}
|
||||
}
|
||||
|
||||
enterRecovery(error) {
|
||||
this.recoveryMode = error instanceof Error ? error.message : String(error)
|
||||
this.publishDshLifecycle({ state: 'failed', action: null, attempt: 0, error: this.recoveryMode })
|
||||
}
|
||||
|
||||
status() {
|
||||
const environment = this.environment.status()
|
||||
return Object.freeze({
|
||||
...environment,
|
||||
controlPlane: this.controlPlane.status().components,
|
||||
recoveryMode: this.recoveryMode,
|
||||
dshLifecycle: this.dshLifecycle,
|
||||
startupComplete: this.startupComplete,
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user