#!/usr/bin/env node import { mkdir, readFile, realpath } from 'node:fs/promises' import { join } from 'node:path' import { EnvironmentRunner, loadControlPlane } from './lib/lifecycle.mjs' import { BootstrapRuntime } from './lib/runtime.mjs' import { createBootstrapControl, listenBootstrapControl } from './lib/control.mjs' import { createDshLifecycleServer, DshLifecycleBroker, listenDshLifecycle } from './lib/dsh-lifecycle.mjs' import { JsonlLogManager } from '../../control-plane/modules/log-manager/index.mjs' import { PlatformPaths } from '../lib/paths.mjs' import { parseImageInventory, recordsFromImageInventory } from '../lib/deployment-contracts.mjs' import { DeploymentManager, DeploymentResolutionError } from './lib/deployments.mjs' import { LocalApiClient } from '../../control-plane/modules/updater/lib/client.mjs' import { discardSystemPluginSelection, linkSystemPluginScope, listManagedSystemPlugins, materializeSystemPluginSelection, pruneSystemPluginSelectionViews, SystemPluginSelectionStore, } from '../../control-plane/modules/plugin-manager/system.mjs' import { replaceSystemPluginView } from '../lib/paths.mjs' import { verifyRuntimePatches } from '../../control-plane/modules/patch-manager/index.mjs' import { materializeSystemSkillCatalog, retainLegacySystemSkillCatalog, SystemSkillManager, } from '../../control-plane/modules/skill-manager/index.mjs' import { SnapshotClient } from '../../control-plane/modules/updater/lib/snapshot-client.mjs' import { recoverPlatformUpdateBeforeDshStart } from '../../control-plane/hooks/recovery/index.mjs' import { recoverUserPluginBeforeDshStart } from '../../control-plane/modules/plugin-manager/user-transaction.mjs' import { UserPluginJournal } from '../../control-plane/modules/plugin-manager/user-journal.mjs' import { UserPluginSelectionStore } from '../../control-plane/modules/plugin-manager/user-state.mjs' import { outboundProxyEnvironment, outboundProxyScopeEnabled, outboundProxyUrl, clearOutboundProxyEnvironment, parseOutboundProxyEnvironment, } from '../lib/outbound-proxy.mjs' import { parseAccessEvidence } from '../lib/access-evidence.mjs' const dataRoot = process.env.DSH_PLATFORM_DATA ?? '/data/platform' const runRoot = process.env.DSH_PLATFORM_RUN ?? '/run/dsh-platform' const paths = new PlatformPaths(dataRoot, runRoot) const logs = new JsonlLogManager({ root: paths.logsRoot, maxBytes: Number(process.env.DSH_LOG_MAX_BYTES ?? 104857600), retentionDays: Number(process.env.DSH_LOG_RETENTION_DAYS ?? 14), output: { stdout: process.stdout, stderr: process.stderr }, }) logs.on('error', error => { void logs.diagnostic('log-manager', 'capture.failed', { error }) }) const seedRoot = process.env.DSH_PLATFORM_SEED ?? '/opt/dsh-platform/seed' const inventory = parseImageInventory(await readFile(join(seedRoot, 'inventory.json'))) const imageRecords = recordsFromImageInventory(inventory) const dshHome = process.env.DSH_HOME ?? '/data/dsh' const accessEvidence = parseAccessEvidence(process.env.DSH_LEGACY_INSTALLATION_EVIDENCE) await logs.diagnostic('bootstrap', 'bootstrap.starting', { bootstrapVersion: process.env.DSH_BOOTSTRAP_VERSION ?? null, imageBuildId: inventory.imageBuildId, targetSequence: inventory.targetSequence, }) const deployments = new DeploymentManager({ paths, seedRoot, inventory }) const trust = new LocalApiClient(paths.trustSocket) const outboundProxy = new LocalApiClient(paths.proxyControlSocket) const snapshots = new SnapshotClient(new LocalApiClient(paths.snapshotSocket), 'dsh') const access = new LocalApiClient(paths.accessSocket) try { await deployments.recoverActivation(trust) } catch (error) { await logs.diagnostic('bootstrap', 'deployment.activation-recovery.failed', { error }) throw error } let imagePlan let previousPlan let planningError = null try { imagePlan = await deployments.prepareImage(imageRecords.deployment) await deployments.publishStatus({ plan: imagePlan, currentId: imagePlan.target }) } catch (error) { planningError = error await logs.diagnostic('bootstrap', 'deployment.image-plan.failed', { error }) if (error instanceof DeploymentResolutionError) { try { previousPlan = await deployments.preparePreviousRecovery() await deployments.publishStatus({ currentId: previousPlan.target, operation: 'recovering' }) } catch (recoveryError) { planningError = new AggregateError([error, recoveryError], 'Deployment resolution and previous recovery failed') await logs.diagnostic('bootstrap', 'deployment.previous-plan.failed', { error: planningError }) } } if (previousPlan === undefined) await deployments.publishStatus({ recoveryMode: { reason: error instanceof Error ? error.message : 'Deployment resolution failed', failedRecordId: null, }, }) } await linkSystemPluginScope({ dshHome: process.env.DSH_HOME ?? '/data/dsh', viewRoot: join(paths.viewsRoot, 'system-plugins'), }) const systemPluginSelections = new SystemPluginSelectionStore(join(paths.deploymentStateRoot, 'system-plugins.json')) await mkdir(paths.systemSkillCatalogsRoot, { recursive: true }) const legacySystemSkillRoot = () => retainLegacySystemSkillCatalog({ sourceRoot: join(seedRoot, 'bootstrap', inventory.bootstrap.id, 'control-plane', 'skills'), outputRoot: paths.systemSkillCatalogsRoot, }) const selectedSystemSkillRoot = async () => { const selected = await deployments.selected() if (selected === null) throw new Error('no Deployment is selected') return await materializeSystemSkillCatalog({ environmentRoot: selected.paths.environment, outputRoot: paths.systemSkillCatalogsRoot, }) ?? await legacySystemSkillRoot() } const systemSkillManager = new SystemSkillManager({ sourceRoot: await selectedSystemSkillRoot(), viewRoot: paths.systemSkillsView, statePath: paths.systemSkillStatePath, }) await systemSkillManager.initialize() const systemSkills = { list: () => systemSkillManager.list(), configure: (skillId, action) => deployments.exclusive(() => systemSkillManager.configure(skillId, action)), } const applySystemPluginSelection = async () => { const selected = await deployments.selected() if (selected === null) return [] const materialized = await materializeSystemPluginSelection({ environmentRoot: selected.paths.environment, sourceRoot: selected.paths['system-plugins'], outputRoot: paths.systemPluginViewsRoot, selectionStore: systemPluginSelections, }) await replaceSystemPluginView(paths, materialized.path) await pruneSystemPluginSelectionViews({ outputRoot: paths.systemPluginViewsRoot, keepPath: materialized.path }) await logs.diagnostic('bootstrap', 'system-plugin.view.applied', { pluginCount: materialized.plugins.length, }) return materialized.plugins } const applyDeploymentResources = async () => { await applySystemPluginSelection() const skills = await systemSkillManager.useSourceRoot(await selectedSystemSkillRoot()) await logs.diagnostic('bootstrap', 'system-skill.view.applied', { skillCount: skills.length }) } const prepareSystemPluginSelection = async () => { const previous = await realpath(join(paths.systemPluginViewsRoot, 'current')) const selected = await deployments.selected() if (selected === null) throw new Error('no Deployment is selected') const materialized = await materializeSystemPluginSelection({ environmentRoot: selected.paths.environment, sourceRoot: selected.paths['system-plugins'], outputRoot: paths.systemPluginViewsRoot, selectionStore: systemPluginSelections, }) await logs.diagnostic('bootstrap', 'system-plugin.view.prepared', { pluginCount: materialized.plugins.length, }) return Object.freeze({ activate: async () => { await replaceSystemPluginView(paths, materialized.path) await logs.diagnostic('bootstrap', 'system-plugin.view.activated') }, commit: async () => { await pruneSystemPluginSelectionViews({ outputRoot: paths.systemPluginViewsRoot, keepPath: materialized.path, }) await logs.diagnostic('bootstrap', 'system-plugin.view.committed') }, rollback: async () => { await replaceSystemPluginView(paths, previous) await pruneSystemPluginSelectionViews({ outputRoot: paths.systemPluginViewsRoot, keepPath: previous }) await logs.diagnostic('bootstrap', 'system-plugin.view.rolled-back', { level: 'warning' }) }, }) } const capture = (child, source, declaration) => logs.capture( child, source, declaration, { acceptForwarded: ['gateway', 'platform-management'].includes(source) }, ) const proxyLaunchToken = process.env.DSH_PROXY_LAUNCH_TOKEN delete process.env.DSH_PROXY_LAUNCH_TOKEN const accessLaunchToken = process.env.DSH_ACCESS_LAUNCH_TOKEN delete process.env.DSH_ACCESS_LAUNCH_TOKEN const reportLifecycle = (message, fields) => logs.diagnostic(fields.componentId ?? 'bootstrap', message, fields) const managedNetworkEnvironment = async scope => { try { const result = await outboundProxy.request('GET', `/v1/environment?scope=${encodeURIComponent(scope)}`) return clearOutboundProxyEnvironment(parseOutboundProxyEnvironment(result.environment, scope)) } catch (error) { await logs.diagnostic('bootstrap', 'outbound-proxy.environment.fallback', { scope, error, level: 'warning', }) try { const state = JSON.parse(await readFile(paths.proxyRoutingStatePath, 'utf8')) return outboundProxyEnvironment(scope, { enabled: outboundProxyScopeEnabled(state, scope) }) } catch { return outboundProxyEnvironment(scope) } } } let runtime const dshLifecycleBroker = new DshLifecycleBroker({ report: reportLifecycle, shouldTerminate: async () => { const lifecycle = runtime?.status().dshLifecycle return ['stopping', 'restarting'].includes(lifecycle?.state) || environment?.stopping === true || await deployments.activation() !== undefined }, }) const controlPlane = new EnvironmentRunner({ environmentRoot: join(import.meta.dirname, '..', '..', 'control-plane'), loader: loadControlPlane, capture, prepareService: async component => { if (component.id === 'access-manager') return { environment: { DSH_ACCESS_LAUNCH_TOKEN: accessLaunchToken }, release: () => {}, } if (component.id === 'outbound-proxy') return { environment: { DSH_PROXY_LAUNCH_TOKEN: proxyLaunchToken }, release: () => {}, } const environment = component.id === 'gateway' ? outboundProxyEnvironment('platform', { enabled: false }) : await managedNetworkEnvironment('platform') return { environment: { ...environment, NODE_USE_ENV_PROXY: '1' }, release: () => {} } }, recoverableComponents: ['access-manager', 'outbound-proxy'], report: reportLifecycle, startupGroups: [ ['gateway', 'access-manager', 'outbound-proxy'], ['platform-recovery'], ['platform-management'], ], }) const environment = new EnvironmentRunner({ environmentRoot: join(paths.viewsRoot, 'environment'), capture, prepareService: async component => { const launch = dshLifecycleBroker.prepareLaunch(component.id) if (component.id !== 'dsh-runtime') return launch const environment = await managedNetworkEnvironment('sharedDsh') return { ...launch, environment: { ...launch.environment, ...environment, NODE_USE_ENV_PROXY: '1', DSH_PLATFORM_AGENT_PROXY_URL: outboundProxyUrl('agentNetwork'), }, } }, report: reportLifecycle, }) runtime = new BootstrapRuntime({ controlPlane, environment, validateDeployment: async () => { await verifyRuntimePatches({ runtimeRoot: join(paths.viewsRoot, 'runtime'), environmentRoot: join(paths.viewsRoot, 'environment'), }) }, prepareDeployment: applyDeploymentResources, beforeReady: async () => { await access.request('POST', '/v1/classify', { token: accessLaunchToken, evidence: accessEvidence }) await logs.diagnostic('bootstrap', 'access.classification.submitted') }, onEnvironmentVerified: async () => dshLifecycleBroker.verify(), onDshAvailable: async () => dshLifecycleBroker.publish(), ownsDshLifecycle: async () => await deployments.activation() !== undefined, onDshRecovered: async () => { await deployments.publishStatus({ recoveryMode: null }) await logs.diagnostic('bootstrap', 'dsh.recovery-mode.cleared') }, report: reportLifecycle, onEnvironmentFatal: async error => { const state = await deployments.state().catch(() => ({ current: null })) const reason = error instanceof Error ? error.message : String(error) await deployments.publishStatus({ recoveryMode: { reason, failedRecordId: state.current }, }) await logs.diagnostic('bootstrap', 'environment.recovery-mode.entered', { error, failedRecordId: state.current }) }, }) const systemPlugins = { list: async () => { const deploymentRoot = await realpath(paths.deploymentView).catch(error => ( error?.code === 'ENOENT' ? undefined : Promise.reject(error) )) if (deploymentRoot === undefined) return [] const activeRoot = await realpath(join(paths.systemPluginViewsRoot, 'current')).catch(error => ( error?.code === 'ENOENT' ? undefined : Promise.reject(error) )) return listManagedSystemPlugins({ environmentRoot: join(deploymentRoot, 'environment'), sourceRoot: join(deploymentRoot, 'system-plugins'), selectionStore: systemPluginSelections, activeRoot, }) }, mutate: (pluginId, action, recovery = false) => deployments.exclusive(async () => { const resolved = await deployments.selected() if (resolved === null) throw new Error('no Deployment is selected') const before = await systemPlugins.list() const pluginIds = before.map(plugin => plugin.id) const previousSelection = await systemPluginSelections.read(pluginIds) try { await systemPluginSelections[recovery ? 'recover' : 'configure'](pluginIds, pluginId, action) return systemPlugins.list() } catch (error) { try { await systemPluginSelections.write(pluginIds, previousSelection) } catch (rollbackError) { throw new AggregateError([error, rollbackError], 'System Plugin operation and rollback failed') } throw error } }), discard: () => deployments.exclusive(async () => { const resolved = await deployments.selected() if (resolved === null) throw new Error('no Deployment is selected') const activeRoot = await realpath(join(paths.systemPluginViewsRoot, 'current')) return discardSystemPluginSelection({ environmentRoot: resolved.paths.environment, sourceRoot: resolved.paths['system-plugins'], selectionStore: systemPluginSelections, activeRoot, }) }), prepare: prepareSystemPluginSelection, } systemPlugins.configure = (pluginId, action) => systemPlugins.mutate(pluginId, action) systemPlugins.recover = (pluginId, action) => systemPlugins.mutate(pluginId, action, true) const server = createBootstrapControl(runtime, { deployments, trust, systemPlugins, systemSkills, snapshots }) const dshLifecycleServer = createDshLifecycleServer(dshLifecycleBroker) await listenBootstrapControl(server, paths.bootstrapSocket) await listenDshLifecycle(dshLifecycleServer, paths.dshLifecycleSocket) let imageCandidateHealthy = true try { const recoveryStartedAt = Date.now() const updateRecovery = await recoverPlatformUpdateBeforeDshStart({ dataRoot, runRoot, dshHome }) const userPluginRecovery = await recoverUserPluginBeforeDshStart({ journal: new UserPluginJournal(paths.userPluginJournalPath), report: (message, fields) => logs.diagnostic('user-plugin-manager', message, fields), selectionStore: new UserPluginSelectionStore(paths.userPluginStatePath), snapshots: new SnapshotClient(new LocalApiClient(paths.snapshotSocket), 'user-plugin'), }) await logs.diagnostic('bootstrap', 'startup-recovery.completed', { elapsedMs: Date.now() - recoveryStartedAt, updatePhase: updateRecovery?.phase ?? null, userPluginPhase: userPluginRecovery?.phase ?? null, }) await runtime.start({ allowRecovery: true, onEnvironmentFailure: async () => { imageCandidateHealthy = false if (previousPlan !== undefined) return false if (planningError !== null) return false if (imagePlan?.fallback !== null && imagePlan?.fallback !== undefined) { return deployments.rejectImage(imagePlan) } try { previousPlan = await deployments.preparePreviousRecovery(imagePlan?.target) return true } catch (error) { await logs.diagnostic('bootstrap', 'deployment.previous-plan.failed', { error }) return false } }, }) } catch (error) { await logs.diagnostic('bootstrap', 'bootstrap.start.failed', { error }) server.close() dshLifecycleServer.close() throw error } if (runtime.recoveryMode === null) { if (imageCandidateHealthy && imagePlan !== undefined) await deployments.acceptImage(imagePlan) else if (previousPlan !== undefined) { try { await deployments.acceptPreviousRecovery(previousPlan, tokens => trust.activate(tokens)) planningError = null } catch (error) { await runtime.suspend('dsh-runtime').catch(suspendError => logs.diagnostic('bootstrap', 'dsh-runtime.suspend.failed', { error: suspendError, cause: error instanceof Error ? error.message : String(error), })) await logs.diagnostic('bootstrap', 'deployment.previous-activation.failed', { error }) runtime.enterRecovery(error) } } } const recoveryReason = runtime.recoveryMode ?? (previousPlan === undefined && planningError instanceof Error ? planningError.message : null) const recoveryMode = recoveryReason === null ? null : { reason: recoveryReason, failedRecordId: imagePlan?.target ?? null, } await deployments.publishStatus({ plan: imagePlan, recoveryMode }) runtime.markStartupComplete() if (recoveryMode === null) await dshLifecycleBroker.publish() await logs.diagnostic('bootstrap', 'platform.ready', { recoveryMode: recoveryMode !== null }) process.send?.({ type: 'ready', bootstrapApi: 1 }) const onRecoveryMessage = message => { if (message?.type !== 'recover-image-baseline' || typeof message.requestId !== 'string') return void deployments.recoverImageBaseline(imageRecords.deployment, { healthCheck: () => runtime.reload(), activateReceipts: tokens => trust.activate(tokens), }).then( async slots => { await logs.diagnostic('bootstrap', 'image-baseline.recovery.completed', { requestId: message.requestId }) process.send?.({ type: 'recovery-result', requestId: message.requestId, slots }) }, async error => { await logs.diagnostic('bootstrap', 'image-baseline.recovery.failed', { error, requestId: message.requestId }) process.send?.({ type: 'recovery-result', requestId: message.requestId, error: error instanceof Error ? error.message : 'image baseline recovery failed', }) }, ) } process.on('message', onRecoveryMessage) let resolveSignal const signal = new Promise(resolve => { resolveSignal = resolve }) const onSignal = () => resolveSignal({ type: 'signal' }) process.once('SIGINT', onSignal) process.once('SIGTERM', onSignal) const outcome = await Promise.race([ signal, runtime.fatal.then(error => ({ type: 'fatal', error })), ]) if (outcome.type === 'fatal') await logs.diagnostic('bootstrap', 'bootstrap.fatal', { error: outcome.error }) else await logs.diagnostic('bootstrap', 'bootstrap.stopping') dshLifecycleBroker.beginShutdown() server.close() dshLifecycleServer.close() await runtime.stop().catch(error => logs.diagnostic('bootstrap', 'bootstrap.stop.failed', { error })) await logs.diagnostic('bootstrap', 'bootstrap.stopped') process.off('message', onRecoveryMessage) if (process.connected) process.disconnect() if (outcome.type === 'fatal') throw outcome.error