dsh/upstream/container/platform/bootstrap/lib/deployments.mjs

import { lstat, mkdir, readFile, rename, rm, symlink } from 'node:fs/promises'
import { join, resolve } from 'node:path'
import { randomUUID } from 'node:crypto'
import { canonicalJson } from '../../lib/canonical-json.mjs'
import {
  deriveRecordId,
  parseDeploymentRecord,
  parseSlots,
  recordsFromImageInventory,
} from '../../lib/deployment-contracts.mjs'
import { durableCreate, durableReplace } from '../../lib/atomic.mjs'
import { readDeploymentStatus, writeDeploymentStatus } from '../../lib/deployment-status.mjs'
import { replaceDeploymentView, replaceSystemPluginView } from '../../lib/paths.mjs'
import { hashTree, hashTreeContent } from '../../lib/tree-hash.mjs'
import { compareDshVersions } from '../../lib/supported-target.mjs'
import { TrustError } from '../../lib/validation.mjs'
import { buildRuntime, cloneTree, loadEnvironmentPatchSet } from '../../../control-plane/modules/patch-manager/index.mjs'

const KINDS = Object.freeze(['environment', 'pristine', 'runtime', 'system-plugins'])

export class DeploymentResolutionError extends Error {
  constructor(message, cause) {
    super(message, { cause })
    this.name = 'DeploymentResolutionError'
  }
}

function sameDeployment(left, right) {
  return left.dshVersion === right.dshVersion
    && left.environmentVersion === right.environmentVersion
    && KINDS.every(kind => {
      const field = kind === 'system-plugins' ? 'systemPlugins' : kind
      return left[field].sha256 === right[field].sha256
    })
}

function validateCandidate(from, candidate, experimental) {
  if (experimental) {
    if (candidate.authority !== 'experimental') throw new TrustError('probation candidate must use Experimental authority')
    if (candidate.targetSequence !== from.targetSequence) throw new TrustError('Experimental candidate must retain the Stable targetSequence')
    if (compareDshVersions(candidate.dshVersion, from.dshVersion) <= 0) {
      throw new TrustError('Experimental candidate DSH must advance the current version')
    }
    if (
      candidate.environment.sha256 !== from.environment.sha256
      || candidate.systemPlugins.sha256 !== from.systemPlugins.sha256
    ) throw new TrustError('Experimental candidate must retain the current Environment and System Plugins')
    return
  }
  if (candidate.authority !== 'stable') throw new TrustError('managed release activation requires Stable authority')
  if (candidate.targetSequence < from.targetSequence) throw new TrustError('Stable candidate targetSequence would roll back')
  if (from.authority === 'stable' && candidate.targetSequence === from.targetSequence && !sameDeployment(from, candidate)) {
    throw new TrustError('Stable candidate conflicts with current content at the same targetSequence')
  }
  if (from.authority === 'experimental' && compareDshVersions(candidate.dshVersion, from.dshVersion) < 0) {
    throw new TrustError('Stable candidate has not caught up with Experimental DSH')
  }
}

async function readOptional(path) {
  try { return await readFile(path) } catch (error) {
    if (error?.code === 'ENOENT') return undefined
    throw error
  }
}

async function exists(path) {
  return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
}

async function replaceRuntimeDirectory(staging, destination) {
  const backup = `${destination}.${randomUUID()}.reset-backup`
  const hadDestination = await exists(destination)
  if (hadDestination) await rename(destination, backup)
  try {
    await rename(staging, destination)
  } catch (error) {
    if (hadDestination) await rename(backup, destination).catch(() => {})
    throw error
  }
  return Object.freeze({
    commit: () => hadDestination ? rm(backup, { recursive: true, force: true }) : Promise.resolve(),
    rollback: async () => {
      await rm(destination, { recursive: true, force: true })
      if (hadDestination) await rename(backup, destination)
    },
  })
}

export class DeploymentManager {
  constructor({ paths, seedRoot, inventory }) {
    this.paths = paths
    this.seedRoot = resolve(seedRoot)
    this.inventory = inventory
    this.imageRecord = recordsFromImageInventory(inventory).deployment
    this.stateRoot = paths.deploymentStateRoot
    this.recordsRoot = join(this.stateRoot, 'records')
    this.slotsPath = join(this.stateRoot, 'slots.json')
    this.activationPath = join(this.stateRoot, 'activation.json')
    this.queue = Promise.resolve()
  }

  exclusive(operation) {
    const result = this.queue.then(operation, operation)
    this.queue = result.then(() => undefined, () => undefined)
    return result
  }

  recordPath(id) {
    return join(this.recordsRoot, `${id}.json`)
  }

  async writeRecord(value) {
    const record = parseDeploymentRecord(value)
    const bytes = canonicalJson(record)
    const path = this.recordPath(record.id)
    const existing = await readOptional(path)
    if (existing === undefined) await durableCreate(path, bytes)
    else if (!existing.equals(bytes)) throw new TrustError('Deployment Record ID identifies conflicting content')
    return record
  }

  async repairImageRecord(value) {
    const record = parseDeploymentRecord(value)
    const bytes = canonicalJson(record)
    if (!bytes.equals(canonicalJson(this.imageRecord))) {
      throw new TrustError('image recovery Record differs from the current image inventory')
    }
    await durableReplace(this.recordPath(record.id), bytes)
    return record
  }

  async record(id) {
    const bytes = await readOptional(this.recordPath(id))
    if (bytes === undefined) throw new TrustError(`Deployment Record ${id} does not exist`)
    return parseDeploymentRecord(bytes)
  }

  async state() {
    const bytes = await readOptional(this.slotsPath)
    if (bytes === undefined) return Object.freeze({ generation: 0, current: null, previous: null })
    return parseSlots(bytes, 'deployment-record', 'Deployment slots')
  }

  availableInCurrentImage(record) {
    return KINDS.every(kind => {
      const reference = record[kind === 'system-plugins' ? 'systemPlugins' : kind]
      return reference.storage !== 'image' || reference.imageBuildId === this.inventory.imageBuildId
    })
  }

  imageRecordFromAnotherImage(record) {
    return KINDS.every(kind => {
      const reference = record[kind === 'system-plugins' ? 'systemPlugins' : kind]
      return reference.storage === 'image' && reference.imageBuildId !== this.inventory.imageBuildId
    })
  }

  async rollbackPlan() {
    const state = await this.state()
    const current = state.current === null ? null : await this.record(state.current)
    let previous = state.previous === null ? null : await this.record(state.previous)
    if (previous !== null && !this.availableInCurrentImage(previous)) previous = null
    return Object.freeze({ current, previous })
  }

  async commit(current, previous) {
    const state = await this.state()
    if (current === previous) previous = null
    const slots = parseSlots({
      schema: 1,
      generation: state.generation + 1,
      current,
      previous,
    }, 'deployment-record', 'Deployment slots')
    await durableReplace(this.slotsPath, canonicalJson(slots))
    return slots
  }

  imageAsset(reference) {
    if (reference.imageBuildId !== this.inventory.imageBuildId) {
      throw new TrustError('Deployment Image Reference belongs to a different image')
    }
    const expected = reference.kind === 'system-plugins'
      ? this.inventory.deployment.systemPlugins
      : this.inventory.deployment[reference.kind]
    if (expected === undefined || reference.id !== expected.id || reference.sha256 !== expected.sha256) {
      throw new TrustError(`Deployment Image Reference for ${reference.kind} is absent from inventory`)
    }
    const roots = {
      environment: 'environments',
      pristine: 'pristine',
      runtime: 'runtimes',
      'system-plugins': 'system-plugins',
    }
    return join(this.seedRoot, roots[reference.kind], reference.id)
  }

  storeAsset(reference) {
    const roots = {
      environment: this.paths.environmentsRoot,
      pristine: this.paths.pristineRoot,
      runtime: this.paths.runtimesRoot,
      'system-plugins': this.paths.systemPluginsRoot,
    }
    return join(roots[reference.kind], reference.id)
  }

  async resolveReference(reference) {
    const path = reference.storage === 'image' ? this.imageAsset(reference) : this.storeAsset(reference)
    const details = await lstat(path)
    if (!details.isDirectory() || details.isSymbolicLink()) {
      throw new TrustError(`resolved ${reference.kind} must be an immutable directory`)
    }
    // Image bytes are bound to the signed inventory during image assembly. Rehashing a large
    // Runtime on every boot would defeat zero-copy startup; managed objects are checked here.
    if (reference.storage === 'store' && await hashTree(path) !== reference.sha256) {
      throw new TrustError(`resolved ${reference.kind} content hash differs from its Record`)
    }
    return path
  }

  async resolveRecord(recordId) {
    const record = await this.record(recordId)
    const entries = await Promise.all(KINDS.map(async kind => [kind, await this.resolveReference(record[kind === 'system-plugins' ? 'systemPlugins' : kind])]))
    return Object.freeze({ record, paths: Object.freeze(Object.fromEntries(entries)) })
  }

  async sameImageDeployment(current, image) {
    if (sameDeployment(current, image)) return true
    if (
      current.dshVersion !== image.dshVersion
      || current.environmentVersion !== image.environmentVersion
      || current.pristine.sha256 !== image.pristine.sha256
      || current.runtime.sha256 !== image.runtime.sha256
      || current.systemPlugins.sha256 !== image.systemPlugins.sha256
    ) return false
    const [currentEnvironment, imageEnvironment] = await Promise.all([
      this.resolveReference(current.environment),
      this.resolveReference(image.environment),
    ])
    const [currentContent, imageContent] = await Promise.all([
      hashTreeContent(currentEnvironment),
      hashTreeContent(imageEnvironment),
    ])
    return currentContent === imageContent
  }

  async selected() {
    const recordId = this.selectedRecordId ?? (await this.state()).current
    return recordId === null ? null : this.resolveRecord(recordId)
  }

  async prepareView(recordId) {
    const resolved = await this.resolveRecord(recordId)
    const destination = join(this.paths.deploymentViewsRoot, recordId)
    const temporary = join(this.paths.deploymentViewsRoot, `.${recordId}.${randomUUID()}.tmp`)
    await mkdir(temporary, { recursive: false })
    try {
      for (const kind of KINDS) await symlink(resolved.paths[kind], join(temporary, kind), 'dir')
      try {
        await rename(temporary, destination)
      } catch (error) {
        if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
      }
    } finally {
      await rm(temporary, { recursive: true, force: true })
    }
    return Object.freeze({ ...resolved, view: destination })
  }

  async select(recordId) {
    const candidate = await this.prepareView(recordId)
    await replaceDeploymentView(this.paths, candidate.view)
    await replaceSystemPluginView(this.paths)
    this.selectedRecordId = recordId
    return candidate
  }

  async initialize(record) {
    const plan = await this.prepareImage(record)
    return this.acceptImage(plan)
  }

  async prepareImage(record) {
    return this.exclusive(async () => {
      const image = await this.writeRecord(record)
      const state = await this.state()
      if (state.current === null) {
        await this.select(image.id)
        return Object.freeze({
          baseGeneration: 0,
          target: image.id,
          fallback: null,
          action: 'initialize',
          imageBehindCurrent: false,
          requiresExperimentalRebuild: false,
        })
      }

      const current = await this.record(state.current)
      let target = current.id
      let action = 'retain'
      let imageBehindCurrent = false
      let requiresExperimentalRebuild = false

      if (
        image.authority === 'development'
        && image.targetSequence === 0
        && current.id !== image.id
        && compareDshVersions(image.dshVersion, current.dshVersion) >= 0
      ) {
        target = image.id
        action = 'development-refresh'
      } else if (
        image.targetSequence < current.targetSequence
        && current.receiptTokens.length === 0
        && this.imageRecordFromAnotherImage(current)
      ) {
        target = image.id
        action = 'image-rollback'
      } else if (current.authority === 'experimental') {
        const dshCaughtUp = compareDshVersions(image.dshVersion, current.dshVersion) >= 0
        if (image.targetSequence >= current.targetSequence && dshCaughtUp) {
          target = image.id
          action = 'stable-caught-up'
        } else {
          imageBehindCurrent = image.targetSequence < current.targetSequence
          requiresExperimentalRebuild = image.targetSequence > current.targetSequence && !dshCaughtUp
        }
      } else if (image.targetSequence > current.targetSequence) {
        target = image.id
        action = 'image-forward'
      } else if (image.targetSequence < current.targetSequence) {
        imageBehindCurrent = true
      } else if (!await this.sameImageDeployment(current, image)) {
        throw new TrustError('Deployment image conflicts with current content at the same targetSequence')
      } else if (current.id !== image.id) {
        target = image.id
        action = 'prefer-image'
      }

      let candidateError = null
      try {
        await this.select(target)
      } catch (error) {
        if (target === current.id) {
          throw new DeploymentResolutionError('current Deployment assets could not be resolved', error)
        }
        try {
          await this.select(current.id)
        } catch (fallbackError) {
          throw new DeploymentResolutionError(
            'image candidate and current Deployment assets could not be resolved',
            new AggregateError([error, fallbackError]),
          )
        }
        target = current.id
        action = 'image-rejected'
        candidateError = error instanceof Error ? error.message : 'image candidate could not be resolved'
      }
      return Object.freeze({
        baseGeneration: state.generation,
        target,
        fallback: target === current.id || action === 'development-refresh' ? null : current.id,
        action,
        imageBehindCurrent,
        requiresExperimentalRebuild,
        candidateError,
      })
    })
  }

  async acceptImage(plan) {
    return this.exclusive(async () => {
      const state = await this.state()
      if (state.generation !== plan.baseGeneration) throw new TrustError('Deployment slots changed during image startup')
      if (state.current === plan.target) return state
      return this.commit(plan.target, plan.action === 'development-refresh' ? null : state.current)
    })
  }

  async rejectImage(plan) {
    return this.exclusive(async () => {
      const state = await this.state()
      if (state.generation !== plan.baseGeneration) throw new TrustError('Deployment slots changed during image startup')
      if (plan.fallback === null) return false
      await this.select(plan.fallback)
      return true
    })
  }

  async preparePreviousRecovery(failedRecordId) {
    return this.exclusive(async () => {
      const state = await this.state()
      const failed = failedRecordId ?? state.current
      if (failed === null || state.current !== failed) {
        throw new TrustError('failed Deployment is no longer current')
      }
      if (state.previous === null || state.previous === failed) {
        throw new TrustError('no previous Deployment exists')
      }
      await this.record(failed)
      await this.select(state.previous)
      return Object.freeze({
        baseGeneration: state.generation,
        failed,
        target: state.previous,
      })
    })
  }

  async acceptPreviousRecovery(plan, activateReceipts) {
    return this.exclusive(async () => {
      const state = await this.state()
      if (state.generation !== plan.baseGeneration || state.current !== plan.failed || state.previous !== plan.target) {
        throw new TrustError('Deployment slots changed during automatic recovery')
      }
      const previous = await this.record(plan.target)
      await activateReceipts(previous.receiptTokens)
      await this.writeActivation({ phase: 'receipts-active', from: plan.failed, to: plan.target })
      const committed = await this.commit(plan.target, plan.failed)
      await this.clearActivation()
      return committed
    })
  }

  async activate(recordId, healthCheck) {
    return this.exclusive(async () => {
      await this.record(recordId)
      const state = await this.state()
      if (state.current === recordId) {
        await this.select(recordId)
        return state
      }
      await this.select(recordId)
      try {
        await healthCheck()
      } catch (error) {
        if (state.current !== null) {
          await this.select(state.current)
          await healthCheck().catch(() => {})
        }
        throw error
      }
      return this.commit(recordId, state.current)
    })
  }

  async rollback({ healthCheck, activateReceipts }) {
    return this.exclusive(async () => {
      const state = await this.state()
      if (state.previous === null) throw new TrustError('no previous Deployment exists')
      const from = await this.record(state.current)
      const previous = await this.record(state.previous)
      await this.writeActivation({ phase: 'prepared', from: from.id, to: previous.id })
      await this.publishStatus({ operation: 'recovering' })
      let receiptsActive = false
      try {
        await this.select(previous.id)
        await healthCheck()
        await this.writeActivation({ phase: 'healthy', from: from.id, to: previous.id })
        await activateReceipts(previous.receiptTokens)
        receiptsActive = true
        await this.writeActivation({ phase: 'receipts-active', from: from.id, to: previous.id })
        const committed = await this.commit(previous.id, from.id)
        await this.clearActivation()
        await this.publishStatus()
        return committed
      } catch (error) {
        if (receiptsActive) await activateReceipts(from.receiptTokens).catch(() => {})
        await this.select(from.id).catch(() => {})
        await healthCheck().catch(() => {})
        await this.clearActivation().catch(() => {})
        await this.publishStatus().catch(() => {})
        throw error
      }
    })
  }

  async materializeRecord(recordId) {
    const record = await this.record(recordId)
    const fields = {
      environment: 'environment',
      pristine: 'pristine',
      runtime: 'runtime',
      systemPlugins: 'system-plugins',
    }
    const references = {}
    for (const [field, kind] of Object.entries(fields)) {
      const reference = record[field]
      if (reference.storage === 'store') {
        references[field] = reference
        continue
      }
      const source = await this.resolveReference(reference)
      const id = `image-${kind}-${reference.sha256}`
      const destination = this.storeAsset({ kind, id })
      if (!await exists(destination)) {
        const temporary = `${destination}.${randomUUID()}.tmp`
        try {
          await cloneTree(source, temporary)
          if (await hashTree(temporary) !== reference.sha256) throw new TrustError(`materialized ${kind} differs from Image Reference`)
          await rename(temporary, destination)
        } finally {
          await rm(temporary, { recursive: true, force: true })
        }
      } else if (await hashTree(destination) !== reference.sha256) {
        throw new TrustError(`materialized ${kind} conflicts with Managed Store`)
      }
      references[field] = { storage: 'store', kind, id, sha256: reference.sha256 }
    }
    const content = {
      schema: 1,
      authority: record.authority,
      targetSequence: record.targetSequence,
      dshVersion: record.dshVersion,
      environmentVersion: record.environmentVersion,
      ...references,
      receiptTokens: [...record.receiptTokens],
      snapshotId: record.snapshotId,
    }
    return this.writeRecord({ ...content, id: deriveRecordId('deployment-record', content) })
  }

  async materializeCurrent() {
    return this.exclusive(async () => {
      const state = await this.state()
      if (state.current === null) throw new TrustError('no current Deployment exists')
      const current = await this.record(state.current)
      const references = [current.environment, current.pristine, current.runtime, current.systemPlugins]
      if (references.every(reference => reference.storage === 'store')) return current
      const materialized = await this.materializeRecord(current.id)
      await this.select(materialized.id)
      await this.commit(materialized.id, state.previous)
      return materialized
    })
  }

  async resetCurrentRuntime({ pauseDsh, restartDsh }) {
    return this.exclusive(async () => {
      const state = await this.state()
      if (state.current === null) throw new TrustError('no current Deployment exists')
      const current = await this.record(state.current)
      const [pristineRoot, environmentRoot] = await Promise.all([
        this.resolveReference(current.pristine),
        this.resolveReference(current.environment),
      ])
      const patchSet = await loadEnvironmentPatchSet(environmentRoot)
      const stagingId = `runtime-reset-${randomUUID()}`
      const versionsRoot = this.paths.runtimesRoot
      const staging = join(versionsRoot, stagingId)
      const destinationId = current.runtime.storage === 'store'
        ? current.runtime.id
        : `reset-${current.runtime.sha256}`
      const destination = join(this.paths.runtimesRoot, destinationId)
      let replacement
      let paused = false
      await this.publishStatus({ operation: 'runtime-reset-building' })
      try {
        await buildRuntime({
          pristineRoot,
          versionsRoot,
          runtimeId: stagingId,
          patchPaths: patchSet.paths,
        })
        await this.publishStatus({ operation: 'runtime-reset-verifying' })
        const sha256 = await hashTree(staging)
        if (sha256 !== current.runtime.sha256) {
          throw new TrustError('rebuilt Runtime differs from the current Deployment Record')
        }

        await this.publishStatus({ operation: 'runtime-reset-switching' })
        await pauseDsh()
        paused = true
        replacement = await replaceRuntimeDirectory(staging, destination)
        const runtime = Object.freeze({
          storage: 'store', kind: 'runtime', id: destinationId, sha256,
        })
        const content = {
          schema: 1,
          authority: current.authority,
          targetSequence: current.targetSequence,
          dshVersion: current.dshVersion,
          environmentVersion: current.environmentVersion,
          environment: current.environment,
          pristine: current.pristine,
          runtime,
          systemPlugins: current.systemPlugins,
          receiptTokens: [...current.receiptTokens],
          snapshotId: current.snapshotId,
        }
        const repaired = current.runtime.storage === 'store'
          ? current
          : await this.writeRecord({ ...content, id: deriveRecordId('deployment-record', content) })
        await this.select(repaired.id)
        await this.publishStatus({ operation: 'runtime-reset-starting' })
        await restartDsh()
        const slots = repaired.id === current.id
          ? state
          : await this.commit(repaired.id, state.previous)
        await replacement.commit()
        await this.publishStatus({ recoveryMode: null })
        return Object.freeze({ recordId: repaired.id, slots })
      } catch (error) {
        if (replacement !== undefined) await replacement.rollback().catch(() => {})
        if (paused) {
          await this.publishStatus({ operation: 'runtime-reset-recovering' }).catch(() => {})
          await this.select(current.id).catch(() => {})
          await restartDsh().catch(() => {})
        }
        await this.publishStatus({ operation: 'runtime-reset-failed' }).catch(() => {})
        throw error
      } finally {
        await rm(staging, { recursive: true, force: true })
      }
    })
  }

  async activation() {
    const bytes = await readOptional(this.activationPath)
    if (bytes === undefined) return undefined
    const value = JSON.parse(bytes.toString('utf8'))
    if (
      value?.schema !== 1
      || !['prepared', 'healthy', 'probation', 'receipts-active'].includes(value.phase)
      || typeof value.from !== 'string'
      || typeof value.to !== 'string'
    ) throw new TrustError('Deployment activation journal is invalid')
    return Object.freeze(value)
  }

  async writeActivation(value) {
    const journal = Object.freeze({ schema: 1, ...value, updatedAt: new Date().toISOString() })
    await durableReplace(this.activationPath, canonicalJson(journal))
    return journal
  }

  async clearActivation() {
    await rm(this.activationPath, { force: true })
  }

  async recoverActivation(trust) {
    const journal = await this.activation()
    if (journal === undefined) return null
    const [from, to, active] = await Promise.all([
      this.record(journal.from),
      this.record(journal.to),
      trust.activeReceipts(),
    ])
    const activeTokens = new Set(active.receipts.map(receipt => receipt.token))
    const candidateReceiptsActive = to.receiptTokens.every(token => activeTokens.has(token))
    if (journal.phase === 'receipts-active' || (journal.phase === 'healthy' && candidateReceiptsActive)) {
      await this.select(to.id)
      const state = await this.state()
      if (state.current !== to.id) await this.commit(to.id, from.id)
      await this.clearActivation()
      return Object.freeze({ action: 'committed', recordId: to.id })
    }
    await trust.activate(from.receiptTokens)
    await this.select(from.id)
    const state = await this.state()
    if (state.current !== from.id) await this.commit(from.id, state.current)
    await this.clearActivation()
    return Object.freeze({ action: 'rolled-back', recordId: from.id })
  }

  async activateManaged(value, { healthCheck, activateReceipts }) {
    const candidate = await this.writeRecord(value)
    await this.materializeCurrent()
    const state = await this.state()
    const from = await this.record(state.current)
    if (candidate.id === from.id) return state
    validateCandidate(from, candidate, false)
    await this.writeActivation({ phase: 'prepared', from: from.id, to: candidate.id })
    await this.publishStatus({ operation: 'switching' })
    let receiptsActive = false
    try {
      await this.select(candidate.id)
      await healthCheck()
      await this.writeActivation({ phase: 'healthy', from: from.id, to: candidate.id })
      await activateReceipts(candidate.receiptTokens)
      receiptsActive = true
      await this.writeActivation({ phase: 'receipts-active', from: from.id, to: candidate.id })
      const committed = await this.commit(candidate.id, from.id)
      await this.clearActivation()
      await this.publishStatus()
      return committed
    } catch (error) {
      await this.publishStatus({ operation: 'recovering' }).catch(() => {})
      if (receiptsActive) await activateReceipts(from.receiptTokens).catch(() => {})
      await this.select(from.id).catch(() => {})
      await healthCheck().catch(() => {})
      await this.clearActivation().catch(() => {})
      await this.publishStatus().catch(() => {})
      throw error
    }
  }

  async stageCandidate(value, healthCheck) {
    const candidate = await this.writeRecord(value)
    await this.materializeCurrent()
    const state = await this.state()
    const from = await this.record(state.current)
    validateCandidate(from, candidate, true)
    await this.writeActivation({ phase: 'prepared', from: from.id, to: candidate.id })
    try {
      await this.select(candidate.id)
      await healthCheck()
      await this.writeActivation({ phase: 'probation', from: from.id, to: candidate.id })
      return candidate
    } catch (error) {
      await this.select(from.id).catch(() => {})
      await this.clearActivation().catch(() => {})
      throw error
    }
  }

  async commitCandidate(recordId, activateReceipts) {
    const journal = await this.activation()
    if (journal?.phase !== 'probation' || journal.to !== recordId) {
      throw new TrustError('Deployment candidate is not in probation')
    }
    const candidate = await this.record(recordId)
    await activateReceipts(candidate.receiptTokens)
    await this.writeActivation({ phase: 'receipts-active', from: journal.from, to: journal.to })
    const committed = await this.commit(candidate.id, journal.from)
    await this.clearActivation()
    await this.publishStatus()
    return committed
  }

  async cancelCandidate() {
    const journal = await this.activation()
    if (journal === undefined) return Object.freeze({ cancelled: false, slots: await this.state() })
    const from = await this.record(journal.from)
    await this.select(from.id)
    await this.clearActivation()
    await this.publishStatus()
    return Object.freeze({ cancelled: true, slots: await this.state() })
  }

  async recoverImageBaseline(value, { healthCheck, activateReceipts }) {
    const previousStatus = await readDeploymentStatus(this.paths.deploymentStatusPath)
    await this.publishStatus({ operation: 'recovering', recoveryMode: previousStatus.recoveryMode })
    const image = await this.repairImageRecord(value)
    await this.prepareView(image.id)
    const state = await this.state()
    await this.select(image.id)
    try {
      await healthCheck()
      await activateReceipts(image.receiptTokens)
      const committed = state.current === image.id ? state : await this.commit(image.id, state.current)
      await this.clearActivation()
      await this.publishStatus({ recoveryMode: null })
      return committed
    } catch (error) {
      if (state.current !== null) await this.select(state.current).catch(() => {})
      await this.publishStatus({
        recoveryMode: error instanceof Error ? error.message : 'image baseline recovery failed',
      }).catch(() => {})
      throw error
    }
  }

  describe(record) {
    const references = [record.environment, record.pristine, record.runtime, record.systemPlugins]
    return Object.freeze({
      recordId: record.id,
      source: references.every(reference => reference.storage === 'image') ? 'image' : 'managed',
      authority: record.authority,
      targetSequence: record.targetSequence,
      dsh: record.dshVersion,
      environment: record.environmentVersion,
    })
  }

  async publishStatus({ plan = null, recoveryMode = null, currentId, operation = null } = {}) {
    const state = await this.state().catch(() => ({ current: null }))
    const selectedId = currentId === undefined ? state.current : currentId
    const current = selectedId === null
      ? null
      : await this.record(selectedId).then(record => this.describe(record), () => null)
    const value = Object.freeze({
      platformLayout: 1,
      imageBaseline: Object.freeze({
        imageBuildId: this.inventory.imageBuildId,
        targetSequence: this.inventory.targetSequence,
        dsh: this.inventory.deployment.dshVersion,
        environment: this.inventory.deployment.environmentVersion,
      }),
      current,
      imageBehindCurrent: plan?.imageBehindCurrent
        ?? (current !== null && current.targetSequence > this.inventory.targetSequence),
      recoveryMode,
      operation,
    })
    return writeDeploymentStatus(this.paths.deploymentStatusPath, value)
  }

  async setOperation(operation) {
    const status = await readDeploymentStatus(this.paths.deploymentStatusPath)
    return writeDeploymentStatus(this.paths.deploymentStatusPath, Object.freeze({
      ...status,
      operation,
    }))
  }
}