dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x

upstream/container/control-plane/modules/updater/lib/snapshots.mjs

@@ -0,0 +1,199 @@
+import { spawn } from 'node:child_process'
+import { createHash, randomUUID } from 'node:crypto'
+import { lstat, mkdir, open, readFile, readdir, rename, rm } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { canonicalJson } from '../../../../platform/lib/canonical-json.mjs'
+import { exactKeys, isoTimestamp, parseJsonDocument, plainObject, TrustError } from '../../../../platform/lib/validation.mjs'
+
+function validId(value) {
+  if (typeof value !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value)) {
+    throw new TrustError('snapshot ID is invalid')
+  }
+  return value
+}
+
+function version(value, label) {
+  if (typeof value !== 'string' || value === '' || value.length > 128 || /[\0/\\]/.test(value)) {
+    throw new TrustError(`${label} is invalid`)
+  }
+  return value
+}
+
+function parseManifest(bytes) {
+  const value = parseJsonDocument(bytes, 'snapshot manifest')
+  exactKeys(value, [
+    'archiveSha256', 'archiveSize', 'createdAt', 'dshVersion', 'environmentVersion',
+    'id', 'runtimeId', 'schema', 'source',
+  ], 'snapshot manifest')
+  if (value.schema !== 1) throw new TrustError('snapshot manifest schema must be 1')
+  validId(value.id)
+  if (value.source !== '/data/dsh') throw new TrustError('snapshot source is invalid')
+  if (typeof value.archiveSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.archiveSha256)) {
+    throw new TrustError('snapshot archive SHA-256 is invalid')
+  }
+  if (!Number.isSafeInteger(value.archiveSize) || value.archiveSize < 0) {
+    throw new TrustError('snapshot archive size is invalid')
+  }
+  isoTimestamp(value.createdAt, 'snapshot creation time')
+  version(value.runtimeId, 'snapshot runtime ID')
+  version(value.environmentVersion, 'snapshot Environment version')
+  version(value.dshVersion, 'snapshot DSH version')
+  return Object.freeze(value)
+}
+
+function run(command, args) {
+  return new Promise((resolveRun, reject) => {
+    const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
+    const stdout = []
+    const stderr = []
+    child.stdout.on('data', chunk => stdout.push(chunk))
+    child.stderr.on('data', chunk => stderr.push(chunk))
+    child.once('error', reject)
+    child.once('exit', code => code === 0
+      ? resolveRun(Buffer.concat(stdout).toString('utf8'))
+      : reject(new Error(`${command} failed: ${Buffer.concat(stderr).toString('utf8').trim()}`)))
+  })
+}
+
+async function hashFile(path) {
+  const handle = await open(path, 'r')
+  const hash = createHash('sha256')
+  let size = 0
+  try {
+    for await (const chunk of handle.createReadStream({ autoClose: false })) {
+      hash.update(chunk)
+      size += chunk.byteLength
+    }
+  } finally {
+    await handle.close()
+  }
+  return { sha256: hash.digest('hex'), size }
+}
+
+async function syncDirectory(path) {
+  const handle = await open(path, 'r')
+  try { await handle.sync() } finally { await handle.close() }
+}
+
+async function countTree(path) {
+  const details = await lstat(path)
+  if (!details.isDirectory() || details.isSymbolicLink()) return 1
+  let count = 0
+  for (const entry of await readdir(path)) count += await countTree(join(path, entry))
+  return count
+}
+
+export class PersistentStateSnapshots {
+  constructor({ root, sourceRoot = '/data/dsh', now = () => new Date(), runImpl = run }) {
+    this.root = resolve(root)
+    this.sourceRoot = resolve(sourceRoot)
+    this.now = now
+    this.run = runImpl
+  }
+
+  path(id) {
+    return join(this.root, 'versions', validId(id))
+  }
+
+  async create({ id = randomUUID(), runtimeId, environmentVersion, dshVersion, onProgress = async () => {} }) {
+    validId(id)
+    const sourceDetails = await lstat(this.sourceRoot)
+    if (!sourceDetails.isDirectory() || sourceDetails.isSymbolicLink()) {
+      throw new TrustError('snapshot source must be a real directory')
+    }
+    const versions = join(this.root, 'versions')
+    const destination = this.path(id)
+    await lstat(destination).then(
+      () => { throw new TrustError(`snapshot ${id} already exists`) },
+      error => { if (error?.code !== 'ENOENT') throw error },
+    )
+    await mkdir(versions, { recursive: true })
+    const staging = join(versions, `.${id}.${randomUUID()}.tmp`)
+    await mkdir(staging)
+    const archive = join(staging, 'data.tar.gz')
+    try {
+      const totalItems = await countTree(this.sourceRoot)
+      await onProgress({ processedItems: 0, totalItems })
+      await this.run('tar', ['-czpf', archive, '-C', this.sourceRoot, '.'])
+      await onProgress({ processedItems: totalItems, totalItems })
+      const measured = await hashFile(archive)
+      const manifest = canonicalJson({
+        schema: 1,
+        id,
+        createdAt: this.now().toISOString(),
+        source: '/data/dsh',
+        runtimeId: version(runtimeId, 'snapshot runtime ID'),
+        environmentVersion: version(environmentVersion, 'snapshot Environment version'),
+        dshVersion: version(dshVersion, 'snapshot DSH version'),
+        archiveSha256: measured.sha256,
+        archiveSize: measured.size,
+      })
+      parseManifest(manifest)
+      const handle = await open(join(staging, 'manifest.json'), 'wx', 0o600)
+      try { await handle.writeFile(manifest); await handle.sync() } finally { await handle.close() }
+      await syncDirectory(staging)
+      await rename(staging, destination)
+      await syncDirectory(versions)
+      return Object.freeze({ ...parseManifest(manifest), path: destination })
+    } catch (error) {
+      await rm(staging, { recursive: true, force: true })
+      throw error
+    }
+  }
+
+  async inspect(id) {
+    const root = this.path(id)
+    const manifest = parseManifest(await readFile(join(root, 'manifest.json')))
+    if (manifest.id !== id) throw new TrustError('snapshot manifest ID does not match its directory')
+    const archive = join(root, 'data.tar.gz')
+    const measured = await hashFile(archive)
+    if (measured.sha256 !== manifest.archiveSha256 || measured.size !== manifest.archiveSize) {
+      throw new TrustError('snapshot archive does not match its manifest', 'TRUST_ARTIFACT_MISMATCH')
+    }
+    const entries = (await this.run('tar', ['-tzf', archive])).split('\n').filter(Boolean)
+    if (entries.length === 0) throw new TrustError('snapshot archive is empty')
+    for (const entry of entries) {
+      if (entry.startsWith('/') || entry.split('/').includes('..')) {
+        throw new TrustError(`snapshot archive path is unsafe: ${entry}`)
+      }
+    }
+    return Object.freeze({ ...manifest, archive })
+  }
+
+  async restore(id, { onProgress = async () => {} } = {}) {
+    const snapshot = await this.inspect(id)
+    await mkdir(this.sourceRoot, { recursive: true })
+    for (const entry of await readdir(this.sourceRoot)) {
+      await rm(join(this.sourceRoot, entry), { recursive: true, force: true })
+    }
+    const entries = (await this.run('tar', ['-tzf', snapshot.archive])).split('\n').filter(Boolean)
+    await onProgress({ processedItems: 0, totalItems: entries.length })
+    const ownership = process.getuid?.() === 0 ? [] : ['--no-same-owner']
+    await this.run('tar', ['-xzpf', snapshot.archive, ...ownership, '-C', this.sourceRoot])
+    await onProgress({ processedItems: entries.length, totalItems: entries.length })
+    await syncDirectory(this.sourceRoot)
+    return snapshot
+  }
+
+  async list() {
+    let names
+    try { names = await readdir(join(this.root, 'versions')) } catch (error) {
+      if (error?.code === 'ENOENT') return []
+      throw error
+    }
+    const snapshots = []
+    for (const name of names.filter(name => !name.startsWith('.')).sort()) snapshots.push(await this.inspect(name))
+    return Object.freeze(snapshots)
+  }
+
+  async remove(id) {
+    try {
+      await this.inspect(id)
+    } catch (error) {
+      if (error?.code === 'ENOENT') return false
+      throw error
+    }
+    await rm(this.path(id), { recursive: true, force: true })
+    return true
+  }
+}