dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,808 @@
|
||||
import { lstat, mkdir, readFile, rename, rm, symlink } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { canonicalJson } from '../../lib/canonical-json.mjs'
|
||||
import {
|
||||
deriveRecordId,
|
||||
parseDeploymentRecord,
|
||||
parseSlots,
|
||||
recordsFromImageInventory,
|
||||
} from '../../lib/deployment-contracts.mjs'
|
||||
import { durableCreate, durableReplace } from '../../lib/atomic.mjs'
|
||||
import { readDeploymentStatus, writeDeploymentStatus } from '../../lib/deployment-status.mjs'
|
||||
import { replaceDeploymentView, replaceSystemPluginView } from '../../lib/paths.mjs'
|
||||
import { hashTree, hashTreeContent } from '../../lib/tree-hash.mjs'
|
||||
import { compareDshVersions } from '../../lib/supported-target.mjs'
|
||||
import { TrustError } from '../../lib/validation.mjs'
|
||||
import { buildRuntime, cloneTree, loadEnvironmentPatchSet } from '../../../control-plane/modules/patch-manager/index.mjs'
|
||||
|
||||
const KINDS = Object.freeze(['environment', 'pristine', 'runtime', 'system-plugins'])
|
||||
|
||||
export class DeploymentResolutionError extends Error {
|
||||
constructor(message, cause) {
|
||||
super(message, { cause })
|
||||
this.name = 'DeploymentResolutionError'
|
||||
}
|
||||
}
|
||||
|
||||
function sameDeployment(left, right) {
|
||||
return left.dshVersion === right.dshVersion
|
||||
&& left.environmentVersion === right.environmentVersion
|
||||
&& KINDS.every(kind => {
|
||||
const field = kind === 'system-plugins' ? 'systemPlugins' : kind
|
||||
return left[field].sha256 === right[field].sha256
|
||||
})
|
||||
}
|
||||
|
||||
function validateCandidate(from, candidate, experimental) {
|
||||
if (experimental) {
|
||||
if (candidate.authority !== 'experimental') throw new TrustError('probation candidate must use Experimental authority')
|
||||
if (candidate.targetSequence !== from.targetSequence) throw new TrustError('Experimental candidate must retain the Stable targetSequence')
|
||||
if (compareDshVersions(candidate.dshVersion, from.dshVersion) <= 0) {
|
||||
throw new TrustError('Experimental candidate DSH must advance the current version')
|
||||
}
|
||||
if (
|
||||
candidate.environment.sha256 !== from.environment.sha256
|
||||
|| candidate.systemPlugins.sha256 !== from.systemPlugins.sha256
|
||||
) throw new TrustError('Experimental candidate must retain the current Environment and System Plugins')
|
||||
return
|
||||
}
|
||||
if (candidate.authority !== 'stable') throw new TrustError('managed release activation requires Stable authority')
|
||||
if (candidate.targetSequence < from.targetSequence) throw new TrustError('Stable candidate targetSequence would roll back')
|
||||
if (from.authority === 'stable' && candidate.targetSequence === from.targetSequence && !sameDeployment(from, candidate)) {
|
||||
throw new TrustError('Stable candidate conflicts with current content at the same targetSequence')
|
||||
}
|
||||
if (from.authority === 'experimental' && compareDshVersions(candidate.dshVersion, from.dshVersion) < 0) {
|
||||
throw new TrustError('Stable candidate has not caught up with Experimental DSH')
|
||||
}
|
||||
}
|
||||
|
||||
async function readOptional(path) {
|
||||
try { return await readFile(path) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
async function replaceRuntimeDirectory(staging, destination) {
|
||||
const backup = `${destination}.${randomUUID()}.reset-backup`
|
||||
const hadDestination = await exists(destination)
|
||||
if (hadDestination) await rename(destination, backup)
|
||||
try {
|
||||
await rename(staging, destination)
|
||||
} catch (error) {
|
||||
if (hadDestination) await rename(backup, destination).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
return Object.freeze({
|
||||
commit: () => hadDestination ? rm(backup, { recursive: true, force: true }) : Promise.resolve(),
|
||||
rollback: async () => {
|
||||
await rm(destination, { recursive: true, force: true })
|
||||
if (hadDestination) await rename(backup, destination)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export class DeploymentManager {
|
||||
constructor({ paths, seedRoot, inventory }) {
|
||||
this.paths = paths
|
||||
this.seedRoot = resolve(seedRoot)
|
||||
this.inventory = inventory
|
||||
this.imageRecord = recordsFromImageInventory(inventory).deployment
|
||||
this.stateRoot = paths.deploymentStateRoot
|
||||
this.recordsRoot = join(this.stateRoot, 'records')
|
||||
this.slotsPath = join(this.stateRoot, 'slots.json')
|
||||
this.activationPath = join(this.stateRoot, 'activation.json')
|
||||
this.queue = Promise.resolve()
|
||||
}
|
||||
|
||||
exclusive(operation) {
|
||||
const result = this.queue.then(operation, operation)
|
||||
this.queue = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
recordPath(id) {
|
||||
return join(this.recordsRoot, `${id}.json`)
|
||||
}
|
||||
|
||||
async writeRecord(value) {
|
||||
const record = parseDeploymentRecord(value)
|
||||
const bytes = canonicalJson(record)
|
||||
const path = this.recordPath(record.id)
|
||||
const existing = await readOptional(path)
|
||||
if (existing === undefined) await durableCreate(path, bytes)
|
||||
else if (!existing.equals(bytes)) throw new TrustError('Deployment Record ID identifies conflicting content')
|
||||
return record
|
||||
}
|
||||
|
||||
async repairImageRecord(value) {
|
||||
const record = parseDeploymentRecord(value)
|
||||
const bytes = canonicalJson(record)
|
||||
if (!bytes.equals(canonicalJson(this.imageRecord))) {
|
||||
throw new TrustError('image recovery Record differs from the current image inventory')
|
||||
}
|
||||
await durableReplace(this.recordPath(record.id), bytes)
|
||||
return record
|
||||
}
|
||||
|
||||
async record(id) {
|
||||
const bytes = await readOptional(this.recordPath(id))
|
||||
if (bytes === undefined) throw new TrustError(`Deployment Record ${id} does not exist`)
|
||||
return parseDeploymentRecord(bytes)
|
||||
}
|
||||
|
||||
async state() {
|
||||
const bytes = await readOptional(this.slotsPath)
|
||||
if (bytes === undefined) return Object.freeze({ generation: 0, current: null, previous: null })
|
||||
return parseSlots(bytes, 'deployment-record', 'Deployment slots')
|
||||
}
|
||||
|
||||
availableInCurrentImage(record) {
|
||||
return KINDS.every(kind => {
|
||||
const reference = record[kind === 'system-plugins' ? 'systemPlugins' : kind]
|
||||
return reference.storage !== 'image' || reference.imageBuildId === this.inventory.imageBuildId
|
||||
})
|
||||
}
|
||||
|
||||
imageRecordFromAnotherImage(record) {
|
||||
return KINDS.every(kind => {
|
||||
const reference = record[kind === 'system-plugins' ? 'systemPlugins' : kind]
|
||||
return reference.storage === 'image' && reference.imageBuildId !== this.inventory.imageBuildId
|
||||
})
|
||||
}
|
||||
|
||||
async rollbackPlan() {
|
||||
const state = await this.state()
|
||||
const current = state.current === null ? null : await this.record(state.current)
|
||||
let previous = state.previous === null ? null : await this.record(state.previous)
|
||||
if (previous !== null && !this.availableInCurrentImage(previous)) previous = null
|
||||
return Object.freeze({ current, previous })
|
||||
}
|
||||
|
||||
async commit(current, previous) {
|
||||
const state = await this.state()
|
||||
if (current === previous) previous = null
|
||||
const slots = parseSlots({
|
||||
schema: 1,
|
||||
generation: state.generation + 1,
|
||||
current,
|
||||
previous,
|
||||
}, 'deployment-record', 'Deployment slots')
|
||||
await durableReplace(this.slotsPath, canonicalJson(slots))
|
||||
return slots
|
||||
}
|
||||
|
||||
imageAsset(reference) {
|
||||
if (reference.imageBuildId !== this.inventory.imageBuildId) {
|
||||
throw new TrustError('Deployment Image Reference belongs to a different image')
|
||||
}
|
||||
const expected = reference.kind === 'system-plugins'
|
||||
? this.inventory.deployment.systemPlugins
|
||||
: this.inventory.deployment[reference.kind]
|
||||
if (expected === undefined || reference.id !== expected.id || reference.sha256 !== expected.sha256) {
|
||||
throw new TrustError(`Deployment Image Reference for ${reference.kind} is absent from inventory`)
|
||||
}
|
||||
const roots = {
|
||||
environment: 'environments',
|
||||
pristine: 'pristine',
|
||||
runtime: 'runtimes',
|
||||
'system-plugins': 'system-plugins',
|
||||
}
|
||||
return join(this.seedRoot, roots[reference.kind], reference.id)
|
||||
}
|
||||
|
||||
storeAsset(reference) {
|
||||
const roots = {
|
||||
environment: this.paths.environmentsRoot,
|
||||
pristine: this.paths.pristineRoot,
|
||||
runtime: this.paths.runtimesRoot,
|
||||
'system-plugins': this.paths.systemPluginsRoot,
|
||||
}
|
||||
return join(roots[reference.kind], reference.id)
|
||||
}
|
||||
|
||||
async resolveReference(reference) {
|
||||
const path = reference.storage === 'image' ? this.imageAsset(reference) : this.storeAsset(reference)
|
||||
const details = await lstat(path)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) {
|
||||
throw new TrustError(`resolved ${reference.kind} must be an immutable directory`)
|
||||
}
|
||||
// Image bytes are bound to the signed inventory during image assembly. Rehashing a large
|
||||
// Runtime on every boot would defeat zero-copy startup; managed objects are checked here.
|
||||
if (reference.storage === 'store' && await hashTree(path) !== reference.sha256) {
|
||||
throw new TrustError(`resolved ${reference.kind} content hash differs from its Record`)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
async resolveRecord(recordId) {
|
||||
const record = await this.record(recordId)
|
||||
const entries = await Promise.all(KINDS.map(async kind => [kind, await this.resolveReference(record[kind === 'system-plugins' ? 'systemPlugins' : kind])]))
|
||||
return Object.freeze({ record, paths: Object.freeze(Object.fromEntries(entries)) })
|
||||
}
|
||||
|
||||
async sameImageDeployment(current, image) {
|
||||
if (sameDeployment(current, image)) return true
|
||||
if (
|
||||
current.dshVersion !== image.dshVersion
|
||||
|| current.environmentVersion !== image.environmentVersion
|
||||
|| current.pristine.sha256 !== image.pristine.sha256
|
||||
|| current.runtime.sha256 !== image.runtime.sha256
|
||||
|| current.systemPlugins.sha256 !== image.systemPlugins.sha256
|
||||
) return false
|
||||
const [currentEnvironment, imageEnvironment] = await Promise.all([
|
||||
this.resolveReference(current.environment),
|
||||
this.resolveReference(image.environment),
|
||||
])
|
||||
const [currentContent, imageContent] = await Promise.all([
|
||||
hashTreeContent(currentEnvironment),
|
||||
hashTreeContent(imageEnvironment),
|
||||
])
|
||||
return currentContent === imageContent
|
||||
}
|
||||
|
||||
async selected() {
|
||||
const recordId = this.selectedRecordId ?? (await this.state()).current
|
||||
return recordId === null ? null : this.resolveRecord(recordId)
|
||||
}
|
||||
|
||||
async prepareView(recordId) {
|
||||
const resolved = await this.resolveRecord(recordId)
|
||||
const destination = join(this.paths.deploymentViewsRoot, recordId)
|
||||
const temporary = join(this.paths.deploymentViewsRoot, `.${recordId}.${randomUUID()}.tmp`)
|
||||
await mkdir(temporary, { recursive: false })
|
||||
try {
|
||||
for (const kind of KINDS) await symlink(resolved.paths[kind], join(temporary, kind), 'dir')
|
||||
try {
|
||||
await rename(temporary, destination)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
|
||||
}
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true })
|
||||
}
|
||||
return Object.freeze({ ...resolved, view: destination })
|
||||
}
|
||||
|
||||
async select(recordId) {
|
||||
const candidate = await this.prepareView(recordId)
|
||||
await replaceDeploymentView(this.paths, candidate.view)
|
||||
await replaceSystemPluginView(this.paths)
|
||||
this.selectedRecordId = recordId
|
||||
return candidate
|
||||
}
|
||||
|
||||
async initialize(record) {
|
||||
const plan = await this.prepareImage(record)
|
||||
return this.acceptImage(plan)
|
||||
}
|
||||
|
||||
async prepareImage(record) {
|
||||
return this.exclusive(async () => {
|
||||
const image = await this.writeRecord(record)
|
||||
const state = await this.state()
|
||||
if (state.current === null) {
|
||||
await this.select(image.id)
|
||||
return Object.freeze({
|
||||
baseGeneration: 0,
|
||||
target: image.id,
|
||||
fallback: null,
|
||||
action: 'initialize',
|
||||
imageBehindCurrent: false,
|
||||
requiresExperimentalRebuild: false,
|
||||
})
|
||||
}
|
||||
|
||||
const current = await this.record(state.current)
|
||||
let target = current.id
|
||||
let action = 'retain'
|
||||
let imageBehindCurrent = false
|
||||
let requiresExperimentalRebuild = false
|
||||
|
||||
if (
|
||||
image.authority === 'development'
|
||||
&& image.targetSequence === 0
|
||||
&& current.id !== image.id
|
||||
&& compareDshVersions(image.dshVersion, current.dshVersion) >= 0
|
||||
) {
|
||||
target = image.id
|
||||
action = 'development-refresh'
|
||||
} else if (
|
||||
image.targetSequence < current.targetSequence
|
||||
&& current.receiptTokens.length === 0
|
||||
&& this.imageRecordFromAnotherImage(current)
|
||||
) {
|
||||
target = image.id
|
||||
action = 'image-rollback'
|
||||
} else if (current.authority === 'experimental') {
|
||||
const dshCaughtUp = compareDshVersions(image.dshVersion, current.dshVersion) >= 0
|
||||
if (image.targetSequence >= current.targetSequence && dshCaughtUp) {
|
||||
target = image.id
|
||||
action = 'stable-caught-up'
|
||||
} else {
|
||||
imageBehindCurrent = image.targetSequence < current.targetSequence
|
||||
requiresExperimentalRebuild = image.targetSequence > current.targetSequence && !dshCaughtUp
|
||||
}
|
||||
} else if (image.targetSequence > current.targetSequence) {
|
||||
target = image.id
|
||||
action = 'image-forward'
|
||||
} else if (image.targetSequence < current.targetSequence) {
|
||||
imageBehindCurrent = true
|
||||
} else if (!await this.sameImageDeployment(current, image)) {
|
||||
throw new TrustError('Deployment image conflicts with current content at the same targetSequence')
|
||||
} else if (current.id !== image.id) {
|
||||
target = image.id
|
||||
action = 'prefer-image'
|
||||
}
|
||||
|
||||
let candidateError = null
|
||||
try {
|
||||
await this.select(target)
|
||||
} catch (error) {
|
||||
if (target === current.id) {
|
||||
throw new DeploymentResolutionError('current Deployment assets could not be resolved', error)
|
||||
}
|
||||
try {
|
||||
await this.select(current.id)
|
||||
} catch (fallbackError) {
|
||||
throw new DeploymentResolutionError(
|
||||
'image candidate and current Deployment assets could not be resolved',
|
||||
new AggregateError([error, fallbackError]),
|
||||
)
|
||||
}
|
||||
target = current.id
|
||||
action = 'image-rejected'
|
||||
candidateError = error instanceof Error ? error.message : 'image candidate could not be resolved'
|
||||
}
|
||||
return Object.freeze({
|
||||
baseGeneration: state.generation,
|
||||
target,
|
||||
fallback: target === current.id || action === 'development-refresh' ? null : current.id,
|
||||
action,
|
||||
imageBehindCurrent,
|
||||
requiresExperimentalRebuild,
|
||||
candidateError,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async acceptImage(plan) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.generation !== plan.baseGeneration) throw new TrustError('Deployment slots changed during image startup')
|
||||
if (state.current === plan.target) return state
|
||||
return this.commit(plan.target, plan.action === 'development-refresh' ? null : state.current)
|
||||
})
|
||||
}
|
||||
|
||||
async rejectImage(plan) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.generation !== plan.baseGeneration) throw new TrustError('Deployment slots changed during image startup')
|
||||
if (plan.fallback === null) return false
|
||||
await this.select(plan.fallback)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
async preparePreviousRecovery(failedRecordId) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
const failed = failedRecordId ?? state.current
|
||||
if (failed === null || state.current !== failed) {
|
||||
throw new TrustError('failed Deployment is no longer current')
|
||||
}
|
||||
if (state.previous === null || state.previous === failed) {
|
||||
throw new TrustError('no previous Deployment exists')
|
||||
}
|
||||
await this.record(failed)
|
||||
await this.select(state.previous)
|
||||
return Object.freeze({
|
||||
baseGeneration: state.generation,
|
||||
failed,
|
||||
target: state.previous,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async acceptPreviousRecovery(plan, activateReceipts) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.generation !== plan.baseGeneration || state.current !== plan.failed || state.previous !== plan.target) {
|
||||
throw new TrustError('Deployment slots changed during automatic recovery')
|
||||
}
|
||||
const previous = await this.record(plan.target)
|
||||
await activateReceipts(previous.receiptTokens)
|
||||
await this.writeActivation({ phase: 'receipts-active', from: plan.failed, to: plan.target })
|
||||
const committed = await this.commit(plan.target, plan.failed)
|
||||
await this.clearActivation()
|
||||
return committed
|
||||
})
|
||||
}
|
||||
|
||||
async activate(recordId, healthCheck) {
|
||||
return this.exclusive(async () => {
|
||||
await this.record(recordId)
|
||||
const state = await this.state()
|
||||
if (state.current === recordId) {
|
||||
await this.select(recordId)
|
||||
return state
|
||||
}
|
||||
await this.select(recordId)
|
||||
try {
|
||||
await healthCheck()
|
||||
} catch (error) {
|
||||
if (state.current !== null) {
|
||||
await this.select(state.current)
|
||||
await healthCheck().catch(() => {})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
return this.commit(recordId, state.current)
|
||||
})
|
||||
}
|
||||
|
||||
async rollback({ healthCheck, activateReceipts }) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.previous === null) throw new TrustError('no previous Deployment exists')
|
||||
const from = await this.record(state.current)
|
||||
const previous = await this.record(state.previous)
|
||||
await this.writeActivation({ phase: 'prepared', from: from.id, to: previous.id })
|
||||
await this.publishStatus({ operation: 'recovering' })
|
||||
let receiptsActive = false
|
||||
try {
|
||||
await this.select(previous.id)
|
||||
await healthCheck()
|
||||
await this.writeActivation({ phase: 'healthy', from: from.id, to: previous.id })
|
||||
await activateReceipts(previous.receiptTokens)
|
||||
receiptsActive = true
|
||||
await this.writeActivation({ phase: 'receipts-active', from: from.id, to: previous.id })
|
||||
const committed = await this.commit(previous.id, from.id)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return committed
|
||||
} catch (error) {
|
||||
if (receiptsActive) await activateReceipts(from.receiptTokens).catch(() => {})
|
||||
await this.select(from.id).catch(() => {})
|
||||
await healthCheck().catch(() => {})
|
||||
await this.clearActivation().catch(() => {})
|
||||
await this.publishStatus().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async materializeRecord(recordId) {
|
||||
const record = await this.record(recordId)
|
||||
const fields = {
|
||||
environment: 'environment',
|
||||
pristine: 'pristine',
|
||||
runtime: 'runtime',
|
||||
systemPlugins: 'system-plugins',
|
||||
}
|
||||
const references = {}
|
||||
for (const [field, kind] of Object.entries(fields)) {
|
||||
const reference = record[field]
|
||||
if (reference.storage === 'store') {
|
||||
references[field] = reference
|
||||
continue
|
||||
}
|
||||
const source = await this.resolveReference(reference)
|
||||
const id = `image-${kind}-${reference.sha256}`
|
||||
const destination = this.storeAsset({ kind, id })
|
||||
if (!await exists(destination)) {
|
||||
const temporary = `${destination}.${randomUUID()}.tmp`
|
||||
try {
|
||||
await cloneTree(source, temporary)
|
||||
if (await hashTree(temporary) !== reference.sha256) throw new TrustError(`materialized ${kind} differs from Image Reference`)
|
||||
await rename(temporary, destination)
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true })
|
||||
}
|
||||
} else if (await hashTree(destination) !== reference.sha256) {
|
||||
throw new TrustError(`materialized ${kind} conflicts with Managed Store`)
|
||||
}
|
||||
references[field] = { storage: 'store', kind, id, sha256: reference.sha256 }
|
||||
}
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: record.authority,
|
||||
targetSequence: record.targetSequence,
|
||||
dshVersion: record.dshVersion,
|
||||
environmentVersion: record.environmentVersion,
|
||||
...references,
|
||||
receiptTokens: [...record.receiptTokens],
|
||||
snapshotId: record.snapshotId,
|
||||
}
|
||||
return this.writeRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
}
|
||||
|
||||
async materializeCurrent() {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.current === null) throw new TrustError('no current Deployment exists')
|
||||
const current = await this.record(state.current)
|
||||
const references = [current.environment, current.pristine, current.runtime, current.systemPlugins]
|
||||
if (references.every(reference => reference.storage === 'store')) return current
|
||||
const materialized = await this.materializeRecord(current.id)
|
||||
await this.select(materialized.id)
|
||||
await this.commit(materialized.id, state.previous)
|
||||
return materialized
|
||||
})
|
||||
}
|
||||
|
||||
async resetCurrentRuntime({ pauseDsh, restartDsh }) {
|
||||
return this.exclusive(async () => {
|
||||
const state = await this.state()
|
||||
if (state.current === null) throw new TrustError('no current Deployment exists')
|
||||
const current = await this.record(state.current)
|
||||
const [pristineRoot, environmentRoot] = await Promise.all([
|
||||
this.resolveReference(current.pristine),
|
||||
this.resolveReference(current.environment),
|
||||
])
|
||||
const patchSet = await loadEnvironmentPatchSet(environmentRoot)
|
||||
const stagingId = `runtime-reset-${randomUUID()}`
|
||||
const versionsRoot = this.paths.runtimesRoot
|
||||
const staging = join(versionsRoot, stagingId)
|
||||
const destinationId = current.runtime.storage === 'store'
|
||||
? current.runtime.id
|
||||
: `reset-${current.runtime.sha256}`
|
||||
const destination = join(this.paths.runtimesRoot, destinationId)
|
||||
let replacement
|
||||
let paused = false
|
||||
await this.publishStatus({ operation: 'runtime-reset-building' })
|
||||
try {
|
||||
await buildRuntime({
|
||||
pristineRoot,
|
||||
versionsRoot,
|
||||
runtimeId: stagingId,
|
||||
patchPaths: patchSet.paths,
|
||||
})
|
||||
await this.publishStatus({ operation: 'runtime-reset-verifying' })
|
||||
const sha256 = await hashTree(staging)
|
||||
if (sha256 !== current.runtime.sha256) {
|
||||
throw new TrustError('rebuilt Runtime differs from the current Deployment Record')
|
||||
}
|
||||
|
||||
await this.publishStatus({ operation: 'runtime-reset-switching' })
|
||||
await pauseDsh()
|
||||
paused = true
|
||||
replacement = await replaceRuntimeDirectory(staging, destination)
|
||||
const runtime = Object.freeze({
|
||||
storage: 'store', kind: 'runtime', id: destinationId, sha256,
|
||||
})
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: current.authority,
|
||||
targetSequence: current.targetSequence,
|
||||
dshVersion: current.dshVersion,
|
||||
environmentVersion: current.environmentVersion,
|
||||
environment: current.environment,
|
||||
pristine: current.pristine,
|
||||
runtime,
|
||||
systemPlugins: current.systemPlugins,
|
||||
receiptTokens: [...current.receiptTokens],
|
||||
snapshotId: current.snapshotId,
|
||||
}
|
||||
const repaired = current.runtime.storage === 'store'
|
||||
? current
|
||||
: await this.writeRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
await this.select(repaired.id)
|
||||
await this.publishStatus({ operation: 'runtime-reset-starting' })
|
||||
await restartDsh()
|
||||
const slots = repaired.id === current.id
|
||||
? state
|
||||
: await this.commit(repaired.id, state.previous)
|
||||
await replacement.commit()
|
||||
await this.publishStatus({ recoveryMode: null })
|
||||
return Object.freeze({ recordId: repaired.id, slots })
|
||||
} catch (error) {
|
||||
if (replacement !== undefined) await replacement.rollback().catch(() => {})
|
||||
if (paused) {
|
||||
await this.publishStatus({ operation: 'runtime-reset-recovering' }).catch(() => {})
|
||||
await this.select(current.id).catch(() => {})
|
||||
await restartDsh().catch(() => {})
|
||||
}
|
||||
await this.publishStatus({ operation: 'runtime-reset-failed' }).catch(() => {})
|
||||
throw error
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async activation() {
|
||||
const bytes = await readOptional(this.activationPath)
|
||||
if (bytes === undefined) return undefined
|
||||
const value = JSON.parse(bytes.toString('utf8'))
|
||||
if (
|
||||
value?.schema !== 1
|
||||
|| !['prepared', 'healthy', 'probation', 'receipts-active'].includes(value.phase)
|
||||
|| typeof value.from !== 'string'
|
||||
|| typeof value.to !== 'string'
|
||||
) throw new TrustError('Deployment activation journal is invalid')
|
||||
return Object.freeze(value)
|
||||
}
|
||||
|
||||
async writeActivation(value) {
|
||||
const journal = Object.freeze({ schema: 1, ...value, updatedAt: new Date().toISOString() })
|
||||
await durableReplace(this.activationPath, canonicalJson(journal))
|
||||
return journal
|
||||
}
|
||||
|
||||
async clearActivation() {
|
||||
await rm(this.activationPath, { force: true })
|
||||
}
|
||||
|
||||
async recoverActivation(trust) {
|
||||
const journal = await this.activation()
|
||||
if (journal === undefined) return null
|
||||
const [from, to, active] = await Promise.all([
|
||||
this.record(journal.from),
|
||||
this.record(journal.to),
|
||||
trust.activeReceipts(),
|
||||
])
|
||||
const activeTokens = new Set(active.receipts.map(receipt => receipt.token))
|
||||
const candidateReceiptsActive = to.receiptTokens.every(token => activeTokens.has(token))
|
||||
if (journal.phase === 'receipts-active' || (journal.phase === 'healthy' && candidateReceiptsActive)) {
|
||||
await this.select(to.id)
|
||||
const state = await this.state()
|
||||
if (state.current !== to.id) await this.commit(to.id, from.id)
|
||||
await this.clearActivation()
|
||||
return Object.freeze({ action: 'committed', recordId: to.id })
|
||||
}
|
||||
await trust.activate(from.receiptTokens)
|
||||
await this.select(from.id)
|
||||
const state = await this.state()
|
||||
if (state.current !== from.id) await this.commit(from.id, state.current)
|
||||
await this.clearActivation()
|
||||
return Object.freeze({ action: 'rolled-back', recordId: from.id })
|
||||
}
|
||||
|
||||
async activateManaged(value, { healthCheck, activateReceipts }) {
|
||||
const candidate = await this.writeRecord(value)
|
||||
await this.materializeCurrent()
|
||||
const state = await this.state()
|
||||
const from = await this.record(state.current)
|
||||
if (candidate.id === from.id) return state
|
||||
validateCandidate(from, candidate, false)
|
||||
await this.writeActivation({ phase: 'prepared', from: from.id, to: candidate.id })
|
||||
await this.publishStatus({ operation: 'switching' })
|
||||
let receiptsActive = false
|
||||
try {
|
||||
await this.select(candidate.id)
|
||||
await healthCheck()
|
||||
await this.writeActivation({ phase: 'healthy', from: from.id, to: candidate.id })
|
||||
await activateReceipts(candidate.receiptTokens)
|
||||
receiptsActive = true
|
||||
await this.writeActivation({ phase: 'receipts-active', from: from.id, to: candidate.id })
|
||||
const committed = await this.commit(candidate.id, from.id)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return committed
|
||||
} catch (error) {
|
||||
await this.publishStatus({ operation: 'recovering' }).catch(() => {})
|
||||
if (receiptsActive) await activateReceipts(from.receiptTokens).catch(() => {})
|
||||
await this.select(from.id).catch(() => {})
|
||||
await healthCheck().catch(() => {})
|
||||
await this.clearActivation().catch(() => {})
|
||||
await this.publishStatus().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async stageCandidate(value, healthCheck) {
|
||||
const candidate = await this.writeRecord(value)
|
||||
await this.materializeCurrent()
|
||||
const state = await this.state()
|
||||
const from = await this.record(state.current)
|
||||
validateCandidate(from, candidate, true)
|
||||
await this.writeActivation({ phase: 'prepared', from: from.id, to: candidate.id })
|
||||
try {
|
||||
await this.select(candidate.id)
|
||||
await healthCheck()
|
||||
await this.writeActivation({ phase: 'probation', from: from.id, to: candidate.id })
|
||||
return candidate
|
||||
} catch (error) {
|
||||
await this.select(from.id).catch(() => {})
|
||||
await this.clearActivation().catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async commitCandidate(recordId, activateReceipts) {
|
||||
const journal = await this.activation()
|
||||
if (journal?.phase !== 'probation' || journal.to !== recordId) {
|
||||
throw new TrustError('Deployment candidate is not in probation')
|
||||
}
|
||||
const candidate = await this.record(recordId)
|
||||
await activateReceipts(candidate.receiptTokens)
|
||||
await this.writeActivation({ phase: 'receipts-active', from: journal.from, to: journal.to })
|
||||
const committed = await this.commit(candidate.id, journal.from)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return committed
|
||||
}
|
||||
|
||||
async cancelCandidate() {
|
||||
const journal = await this.activation()
|
||||
if (journal === undefined) return Object.freeze({ cancelled: false, slots: await this.state() })
|
||||
const from = await this.record(journal.from)
|
||||
await this.select(from.id)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus()
|
||||
return Object.freeze({ cancelled: true, slots: await this.state() })
|
||||
}
|
||||
|
||||
async recoverImageBaseline(value, { healthCheck, activateReceipts }) {
|
||||
const previousStatus = await readDeploymentStatus(this.paths.deploymentStatusPath)
|
||||
await this.publishStatus({ operation: 'recovering', recoveryMode: previousStatus.recoveryMode })
|
||||
const image = await this.repairImageRecord(value)
|
||||
await this.prepareView(image.id)
|
||||
const state = await this.state()
|
||||
await this.select(image.id)
|
||||
try {
|
||||
await healthCheck()
|
||||
await activateReceipts(image.receiptTokens)
|
||||
const committed = state.current === image.id ? state : await this.commit(image.id, state.current)
|
||||
await this.clearActivation()
|
||||
await this.publishStatus({ recoveryMode: null })
|
||||
return committed
|
||||
} catch (error) {
|
||||
if (state.current !== null) await this.select(state.current).catch(() => {})
|
||||
await this.publishStatus({
|
||||
recoveryMode: error instanceof Error ? error.message : 'image baseline recovery failed',
|
||||
}).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
describe(record) {
|
||||
const references = [record.environment, record.pristine, record.runtime, record.systemPlugins]
|
||||
return Object.freeze({
|
||||
recordId: record.id,
|
||||
source: references.every(reference => reference.storage === 'image') ? 'image' : 'managed',
|
||||
authority: record.authority,
|
||||
targetSequence: record.targetSequence,
|
||||
dsh: record.dshVersion,
|
||||
environment: record.environmentVersion,
|
||||
})
|
||||
}
|
||||
|
||||
async publishStatus({ plan = null, recoveryMode = null, currentId, operation = null } = {}) {
|
||||
const state = await this.state().catch(() => ({ current: null }))
|
||||
const selectedId = currentId === undefined ? state.current : currentId
|
||||
const current = selectedId === null
|
||||
? null
|
||||
: await this.record(selectedId).then(record => this.describe(record), () => null)
|
||||
const value = Object.freeze({
|
||||
platformLayout: 1,
|
||||
imageBaseline: Object.freeze({
|
||||
imageBuildId: this.inventory.imageBuildId,
|
||||
targetSequence: this.inventory.targetSequence,
|
||||
dsh: this.inventory.deployment.dshVersion,
|
||||
environment: this.inventory.deployment.environmentVersion,
|
||||
}),
|
||||
current,
|
||||
imageBehindCurrent: plan?.imageBehindCurrent
|
||||
?? (current !== null && current.targetSequence > this.inventory.targetSequence),
|
||||
recoveryMode,
|
||||
operation,
|
||||
})
|
||||
return writeDeploymentStatus(this.paths.deploymentStatusPath, value)
|
||||
}
|
||||
|
||||
async setOperation(operation) {
|
||||
const status = await readDeploymentStatus(this.paths.deploymentStatusPath)
|
||||
return writeDeploymentStatus(this.paths.deploymentStatusPath, Object.freeze({
|
||||
...status,
|
||||
operation,
|
||||
}))
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user