dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,347 @@
|
||||
export class BootstrapRuntime {
|
||||
constructor({
|
||||
controlPlane,
|
||||
environment,
|
||||
validateDeployment = async () => {},
|
||||
prepareDeployment = async () => {},
|
||||
beforeReady = async () => {},
|
||||
onEnvironmentVerified = async () => {},
|
||||
onDshAvailable = async () => {},
|
||||
onEnvironmentFatal = async () => {},
|
||||
onDshRecovered = async () => {},
|
||||
ownsDshLifecycle = async () => false,
|
||||
recoveryDelaysMs = [0, 2_000, 5_000],
|
||||
sleep = (milliseconds, signal) => new Promise(resolve => {
|
||||
if (milliseconds === 0 || signal?.aborted) return resolve()
|
||||
const timer = setTimeout(resolve, milliseconds)
|
||||
timer.unref?.()
|
||||
signal?.addEventListener('abort', () => { clearTimeout(timer); resolve() }, { once: true })
|
||||
}),
|
||||
report = async () => {},
|
||||
}) {
|
||||
if (!Array.isArray(recoveryDelaysMs) || recoveryDelaysMs.length !== 3
|
||||
|| recoveryDelaysMs.some(value => !Number.isFinite(value) || value < 0)) {
|
||||
throw new Error('DSH recovery delays must contain three non-negative durations')
|
||||
}
|
||||
this.controlPlane = controlPlane
|
||||
this.environment = environment
|
||||
this.fatal = controlPlane.fatal
|
||||
this.recoveryMode = null
|
||||
this.startupComplete = false
|
||||
this.dshLifecycle = Object.freeze({
|
||||
state: 'starting', action: 'start', taskId: null, attempt: 0, maxAttempts: 3,
|
||||
error: null, updatedAt: new Date().toISOString(),
|
||||
})
|
||||
this.recovery = Promise.resolve()
|
||||
this.recoveryAbort = new AbortController()
|
||||
this.validateDeployment = validateDeployment
|
||||
this.prepareDeployment = prepareDeployment
|
||||
this.beforeReady = beforeReady
|
||||
this.onEnvironmentVerified = onEnvironmentVerified
|
||||
this.onDshAvailable = onDshAvailable
|
||||
this.onEnvironmentFatal = onEnvironmentFatal
|
||||
this.onDshRecovered = onDshRecovered
|
||||
this.ownsDshLifecycle = ownsDshLifecycle
|
||||
this.recoveryDelaysMs = Object.freeze([...recoveryDelaysMs])
|
||||
this.sleep = sleep
|
||||
this.report = report
|
||||
const handleEnvironmentFatal = error => {
|
||||
const recover = () => this.handleEnvironmentFatal(error)
|
||||
this.recovery = this.recovery.then(recover, recover).catch(recoveryError => this.record(
|
||||
'environment.recovery.failed', { error: recoveryError, originalError: String(error) },
|
||||
))
|
||||
}
|
||||
if (typeof environment.onFatal === 'function') {
|
||||
this.offEnvironmentFatal = environment.onFatal(handleEnvironmentFatal)
|
||||
} else {
|
||||
void environment.fatal.then(handleEnvironmentFatal)
|
||||
this.offEnvironmentFatal = undefined
|
||||
}
|
||||
}
|
||||
|
||||
async lifecycleOwned() {
|
||||
try {
|
||||
return await this.ownsDshLifecycle()
|
||||
} catch (error) {
|
||||
await this.record('dsh.recovery-ownership.failed', { error, level: 'warning' })
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
async handleEnvironmentFatal(error) {
|
||||
if (error?.componentId !== 'dsh-runtime') return this.isolateEnvironmentFailure(error)
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
if (await this.lifecycleOwned()) {
|
||||
this.publishDshLifecycle({ state: 'recovering', action: 'auto-recover', attempt: 0, error: message })
|
||||
await this.record('dsh.recovery.deferred', { error, reason: 'deployment-transaction-active', level: 'warning' })
|
||||
return
|
||||
}
|
||||
for (let index = 0; index < this.recoveryDelaysMs.length; index += 1) {
|
||||
const attempt = index + 1
|
||||
await this.sleep(this.recoveryDelaysMs[index], this.recoveryAbort.signal)
|
||||
if (this.recoveryAbort.signal.aborted) return
|
||||
if (await this.lifecycleOwned()) {
|
||||
this.publishDshLifecycle({ state: 'recovering', action: 'auto-recover', attempt: attempt - 1, error: message })
|
||||
await this.record('dsh.recovery.deferred', { error, reason: 'deployment-transaction-active', level: 'warning' })
|
||||
return
|
||||
}
|
||||
this.publishDshLifecycle({ state: 'recovering', action: 'auto-recover', attempt, error: message })
|
||||
await this.record('dsh.recovery.attempt.started', { attempt, maxAttempts: this.recoveryDelaysMs.length })
|
||||
try {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
await this.environment.restart('dsh-runtime')
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
let recoveryStatusPublished = true
|
||||
try {
|
||||
await this.onDshRecovered()
|
||||
} catch (statusError) {
|
||||
recoveryStatusPublished = false
|
||||
await this.record('dsh.recovery-status.failed', { error: statusError, level: 'warning' })
|
||||
}
|
||||
if (recoveryStatusPublished) await this.onDshAvailable()
|
||||
await this.record('dsh.recovery.completed', { attempt, maxAttempts: this.recoveryDelaysMs.length })
|
||||
return
|
||||
} catch (recoveryError) {
|
||||
await this.record('dsh.recovery.attempt.failed', {
|
||||
attempt,
|
||||
error: recoveryError,
|
||||
maxAttempts: this.recoveryDelaysMs.length,
|
||||
level: 'warning',
|
||||
})
|
||||
}
|
||||
}
|
||||
await this.isolateEnvironmentFailure(error)
|
||||
}
|
||||
|
||||
async isolateEnvironmentFailure(error) {
|
||||
this.recoveryMode = error instanceof Error ? error.message : String(error)
|
||||
this.publishDshLifecycle({
|
||||
state: 'failed', action: null,
|
||||
attempt: error?.componentId === 'dsh-runtime' ? this.recoveryDelaysMs.length : 0,
|
||||
error: this.recoveryMode,
|
||||
})
|
||||
try {
|
||||
await this.onEnvironmentFatal(error)
|
||||
} catch (reportError) {
|
||||
await this.record('environment.recovery-report.failed', { error: reportError, originalError: String(error) })
|
||||
}
|
||||
try {
|
||||
await this.environment.stop()
|
||||
} catch (stopError) {
|
||||
await this.record('environment.recovery-stop.failed', { error: stopError, originalError: String(error) })
|
||||
}
|
||||
}
|
||||
|
||||
record(message, fields = {}) {
|
||||
return Promise.resolve().then(() => this.report(message, fields)).catch(() => {})
|
||||
}
|
||||
|
||||
publishDshLifecycle(value) {
|
||||
this.dshLifecycle = Object.freeze({
|
||||
...this.dshLifecycle,
|
||||
...value,
|
||||
updatedAt: new Date().toISOString(),
|
||||
})
|
||||
return this.dshLifecycle
|
||||
}
|
||||
|
||||
async stopControlPlane(phase, cause) {
|
||||
try {
|
||||
await this.controlPlane.stop()
|
||||
} catch (error) {
|
||||
await this.record('control-plane.cleanup.failed', {
|
||||
error,
|
||||
phase,
|
||||
originalError: cause instanceof Error ? cause.message : String(cause),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async stopEnvironment(phase, cause) {
|
||||
try {
|
||||
await this.environment.stop()
|
||||
} catch (error) {
|
||||
await this.record('environment.cleanup.failed', {
|
||||
error,
|
||||
phase,
|
||||
originalError: cause instanceof Error ? cause.message : String(cause),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async startEnvironment({ onEnvironmentFailure, allowRecovery }) {
|
||||
try {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
await this.environment.start()
|
||||
await this.onEnvironmentVerified()
|
||||
} catch (error) {
|
||||
let retry
|
||||
try {
|
||||
retry = await onEnvironmentFailure?.(error)
|
||||
} catch (recoveryError) {
|
||||
const failure = new AggregateError([error, recoveryError], 'Deployment failed and its fallback could not be resolved')
|
||||
if (allowRecovery) {
|
||||
this.recoveryMode = failure.message
|
||||
return
|
||||
}
|
||||
throw failure
|
||||
}
|
||||
if (retry === true) {
|
||||
try {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
await this.environment.start()
|
||||
await this.onEnvironmentVerified()
|
||||
} catch (fallbackError) {
|
||||
const failure = new AggregateError([error, fallbackError], 'Deployment candidate and fallback both failed')
|
||||
if (allowRecovery) this.recoveryMode = failure.message
|
||||
else throw failure
|
||||
}
|
||||
} else if (allowRecovery) {
|
||||
this.recoveryMode = error instanceof Error ? error.message : 'Deployment failed to start'
|
||||
} else throw error
|
||||
}
|
||||
}
|
||||
|
||||
async start({ onEnvironmentFailure, allowRecovery = false } = {}) {
|
||||
const readiness = Promise.resolve().then(async () => {
|
||||
await this.controlPlane.start()
|
||||
await this.beforeReady()
|
||||
})
|
||||
const environment = Promise.resolve().then(() => this.startEnvironment({ onEnvironmentFailure, allowRecovery }))
|
||||
const results = await Promise.allSettled([readiness, environment])
|
||||
const failures = results.filter(result => result.status === 'rejected').map(result => result.reason)
|
||||
if (failures.length > 0) {
|
||||
const failure = failures.length === 1 ? failures[0] : new AggregateError(failures, 'Platform startup failed')
|
||||
await Promise.all([
|
||||
this.stopEnvironment('concurrent-start', failure),
|
||||
this.stopControlPlane('concurrent-start', failure),
|
||||
])
|
||||
throw failure
|
||||
}
|
||||
return this.status()
|
||||
}
|
||||
|
||||
async stop() {
|
||||
this.offEnvironmentFatal?.()
|
||||
this.recoveryAbort.abort()
|
||||
await this.recovery
|
||||
const failures = []
|
||||
try { await this.environment.stop() } catch (error) { failures.push(error) }
|
||||
try { await this.controlPlane.stop() } catch (error) { failures.push(error) }
|
||||
if (failures.length > 0) throw new AggregateError(failures, 'Bootstrap shutdown failed')
|
||||
}
|
||||
|
||||
async reload() {
|
||||
await this.validateDeployment()
|
||||
await this.prepareDeployment()
|
||||
const status = await this.environment.reload()
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
if (this.environment.status().components.some(value => value.id === 'dsh-runtime')) {
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
await this.onDshAvailable()
|
||||
}
|
||||
return status
|
||||
}
|
||||
async suspend(componentId) {
|
||||
if (componentId !== 'dsh-runtime') return this.environment.suspend(componentId)
|
||||
this.publishDshLifecycle({ state: 'stopping', action: 'stop', attempt: 0, error: null })
|
||||
try {
|
||||
const status = await this.environment.suspend(componentId)
|
||||
this.publishDshLifecycle({ state: 'stopped', action: null, attempt: 0, error: null })
|
||||
return status
|
||||
} catch (error) {
|
||||
this.publishDshLifecycle({
|
||||
state: 'failed', action: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
async pause(componentId) {
|
||||
if (componentId !== 'dsh-runtime') return this.environment.pause(componentId)
|
||||
this.publishDshLifecycle({ state: 'stopping', action: 'stop', attempt: 0, error: null })
|
||||
try {
|
||||
const status = await this.environment.pause(componentId, { allowMissing: true })
|
||||
this.publishDshLifecycle({ state: 'stopped', action: null, attempt: 0, error: null })
|
||||
return status
|
||||
} catch (error) {
|
||||
this.publishDshLifecycle({
|
||||
state: 'failed', action: null,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
async resume(componentId, options) {
|
||||
if (componentId === 'dsh-runtime') this.publishDshLifecycle({ state: 'starting', action: 'start', attempt: 0, error: null })
|
||||
try {
|
||||
if (componentId === 'dsh-runtime') {
|
||||
if (options?.skipValidation !== true) await this.validateDeployment()
|
||||
if (options?.skipPreparation !== true) await this.prepareDeployment()
|
||||
}
|
||||
const status = await this.environment.resume(componentId)
|
||||
if (componentId === 'dsh-runtime') {
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
await this.onDshAvailable()
|
||||
}
|
||||
return status
|
||||
} catch (error) {
|
||||
if (componentId === 'dsh-runtime') this.enterRecovery(error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
async restart(componentId, options) {
|
||||
if (componentId === 'dsh-runtime') this.publishDshLifecycle({ state: 'restarting', action: 'restart', attempt: 0, error: null })
|
||||
try {
|
||||
if (componentId === 'dsh-runtime') {
|
||||
if (options?.skipValidation !== true) await this.validateDeployment()
|
||||
if (options?.skipPreparation !== true) await this.prepareDeployment()
|
||||
}
|
||||
const status = await this.environment.restart(componentId, options)
|
||||
if (componentId === 'dsh-runtime') {
|
||||
await this.onEnvironmentVerified()
|
||||
this.recoveryMode = null
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
await this.onDshAvailable()
|
||||
}
|
||||
return status
|
||||
} catch (error) {
|
||||
if (componentId === 'dsh-runtime') this.enterRecovery(error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
health() { return this.environment.health() }
|
||||
|
||||
markStartupComplete() {
|
||||
this.startupComplete = true
|
||||
if (this.recoveryMode === null && this.environment.status().components.some(value => value.id === 'dsh-runtime')) {
|
||||
this.publishDshLifecycle({ state: 'running', action: null, attempt: 0, error: null })
|
||||
} else if (this.recoveryMode !== null) {
|
||||
this.publishDshLifecycle({ state: 'failed', action: null, error: this.recoveryMode })
|
||||
}
|
||||
}
|
||||
|
||||
enterRecovery(error) {
|
||||
this.recoveryMode = error instanceof Error ? error.message : String(error)
|
||||
this.publishDshLifecycle({ state: 'failed', action: null, attempt: 0, error: this.recoveryMode })
|
||||
}
|
||||
|
||||
status() {
|
||||
const environment = this.environment.status()
|
||||
return Object.freeze({
|
||||
...environment,
|
||||
controlPlane: this.controlPlane.status().components,
|
||||
recoveryMode: this.recoveryMode,
|
||||
dshLifecycle: this.dshLifecycle,
|
||||
startupComplete: this.startupComplete,
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user