dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,110 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { lstat, mkdir, readdir, readlink, rm } from 'node:fs/promises'
|
||||
import { dirname, join, relative, resolve, sep } from 'node:path'
|
||||
import { FileManagerError } from './index.mjs'
|
||||
|
||||
export const ARCHIVE_FORMATS = Object.freeze({
|
||||
zip: { extension: '.zip', command: 'zip' },
|
||||
'7z': { extension: '.7z', command: '7z' },
|
||||
'tar.gz': { extension: '.tar.gz', command: 'tar' },
|
||||
})
|
||||
|
||||
export function normalizeArchiveFormat(value) {
|
||||
if (!Object.hasOwn(ARCHIVE_FORMATS, value)) throw new FileManagerError('archive format is invalid')
|
||||
return value
|
||||
}
|
||||
|
||||
function run(command, args, { cwd, signal, capture = false } = {}) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const child = spawn(command, args, { cwd, signal, stdio: ['ignore', capture ? 'pipe' : 'ignore', 'pipe'] })
|
||||
let errorOutput = ''
|
||||
let output = ''
|
||||
if (capture) {
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stdout.on('data', chunk => { if (output.length < 8 * 1024 * 1024) output += chunk })
|
||||
}
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', chunk => { if (errorOutput.length < 16_384) errorOutput += chunk })
|
||||
child.once('error', reject)
|
||||
child.once('exit', (code, signalName) => {
|
||||
if (code === 0) resolvePromise(output)
|
||||
else reject(new FileManagerError(
|
||||
`archive tool failed${errorOutput.trim() === '' ? '' : `: ${errorOutput.trim()}`}`,
|
||||
400,
|
||||
signalName === 'SIGTERM' ? 'FILE_TASK_CANCELLED' : 'ARCHIVE_TOOL_FAILED',
|
||||
))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function validateEntryNames(names) {
|
||||
for (const original of names) {
|
||||
const name = original.replace(/^(\.\/)+/u, '')
|
||||
if (name === '') continue
|
||||
if (name.includes('\0') || name.startsWith('/') || name.split('/').includes('..')) {
|
||||
throw new FileManagerError('archive contains an unsafe path', 400, 'ARCHIVE_UNSAFE')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function inside(root, path) {
|
||||
const value = relative(root, path)
|
||||
return value === '' || (!value.startsWith(`..${sep}`) && value !== '..')
|
||||
}
|
||||
|
||||
export async function validateExtractedTree(root) {
|
||||
const pending = [root]
|
||||
let entries = 0
|
||||
let bytes = 0
|
||||
while (pending.length > 0) {
|
||||
const path = pending.pop()
|
||||
const details = await lstat(path)
|
||||
if (details.isDirectory() && !details.isSymbolicLink()) {
|
||||
for (const name of await readdir(path)) pending.push(join(path, name))
|
||||
} else if (details.isSymbolicLink()) {
|
||||
const target = await readlink(path)
|
||||
if (target.startsWith('/') || !inside(root, resolve(dirname(path), target))) {
|
||||
throw new FileManagerError('archive contains a symbolic link outside its destination', 400, 'ARCHIVE_UNSAFE')
|
||||
}
|
||||
} else if (!details.isFile()) {
|
||||
throw new FileManagerError('archive contains an unsupported special file', 415, 'ARCHIVE_UNSAFE')
|
||||
}
|
||||
entries += 1
|
||||
if (details.isFile()) bytes += details.size
|
||||
}
|
||||
return { entries: Math.max(0, entries - 1), bytes }
|
||||
}
|
||||
|
||||
export async function createArchive({ format, sourceRoot, output, entries = ['.'], signal }) {
|
||||
normalizeArchiveFormat(format)
|
||||
if (!Array.isArray(entries) || entries.length === 0 || entries.some(entry => typeof entry !== 'string' || entry === '' || entry.includes('\0') || entry.includes('/'))) {
|
||||
throw new FileManagerError('archive entry names are invalid')
|
||||
}
|
||||
if (format === 'zip') await run('zip', ['-q', '-r', output, '--', ...entries], { cwd: sourceRoot, signal })
|
||||
else if (format === '7z') await run('7z', ['a', '-bd', '-y', '-t7z', output, '--', ...entries], { cwd: sourceRoot, signal })
|
||||
else await run('tar', ['-czf', output, '-C', sourceRoot, '--', ...entries], { signal })
|
||||
}
|
||||
|
||||
export async function extractArchive({ format, archive, output, signal }) {
|
||||
normalizeArchiveFormat(format)
|
||||
await mkdir(output, { recursive: true })
|
||||
try {
|
||||
if (format === 'zip') {
|
||||
validateEntryNames((await run('unzip', ['-Z1', archive], { signal, capture: true })).split('\n'))
|
||||
await run('unzip', ['-qq', archive, '-d', output], { signal })
|
||||
} else if (format === '7z') {
|
||||
const listed = await run('7z', ['l', '-slt', '--', archive], { signal, capture: true })
|
||||
validateEntryNames(listed.split('\n').filter(line => line.startsWith('Path = ')).slice(1).map(line => line.slice(7)))
|
||||
await run('7z', ['x', '-bd', '-y', `-o${output}`, '--', archive], { signal })
|
||||
} else {
|
||||
validateEntryNames((await run('tar', ['-tzf', archive], { signal, capture: true })).split('\n'))
|
||||
await run('tar', ['--extract', '--gzip', '--file', archive, '--directory', output, '--no-same-owner', '--no-same-permissions'], { signal })
|
||||
}
|
||||
return await validateExtractedTree(output)
|
||||
} catch (error) {
|
||||
await rm(output, { recursive: true, force: true }).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export const archiveInternals = Object.freeze({ inside, run, validateEntryNames })
|
||||
@@ -0,0 +1,78 @@
|
||||
import { link, open, rename, rm, stat } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
FileManagerError, FileRevisionConflictError, MAX_TEXT_BYTES, fileError,
|
||||
fileManagerInternals, isManagedPath, normalizeAbsolutePath,
|
||||
} from './index.mjs'
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
function bytes(value) {
|
||||
if (typeof value !== 'string') throw new FileManagerError('content must be a string')
|
||||
const result = Buffer.from(value, 'utf8')
|
||||
if (result.byteLength > MAX_TEXT_BYTES) throw new FileManagerError('file is too large to edit', 413, 'FILE_TOO_LARGE')
|
||||
if (result.includes(0)) throw new FileManagerError('content contains NUL', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
return result
|
||||
}
|
||||
|
||||
export class AtomicFileEditor {
|
||||
constructor({ isManaged = isManagedPath } = {}) {
|
||||
this.isManaged = isManaged
|
||||
}
|
||||
|
||||
async write(value, content, revision, { create = false } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
const body = bytes(content)
|
||||
const parent = dirname(path)
|
||||
let original
|
||||
try {
|
||||
original = await stat(path, { bigint: true })
|
||||
if (!original.isFile()) throw new FileManagerError('path is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
if (create) throw new FileManagerError('file already exists', 409, 'FILE_EXISTS')
|
||||
if (typeof revision !== 'string' || revision !== fileManagerInternals.revisionFor(path, original)) {
|
||||
throw new FileRevisionConflictError()
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw fileError(error)
|
||||
if (!create) throw fileError(error)
|
||||
if (revision !== null && revision !== undefined) throw new FileManagerError('new file revision must be null')
|
||||
}
|
||||
const temporary = join(parent, `.dsh-edit-${randomUUID()}.tmp`)
|
||||
let handle
|
||||
try {
|
||||
handle = await open(temporary, 'wx', original === undefined ? 0o644 : Number(original.mode & 0o7777n))
|
||||
await handle.writeFile(body)
|
||||
await handle.sync()
|
||||
if (original !== undefined) {
|
||||
await handle.chmod(Number(original.mode & 0o7777n))
|
||||
const current = await handle.stat({ bigint: true })
|
||||
if (current.uid !== original.uid || current.gid !== original.gid) {
|
||||
try { await handle.chown(Number(original.uid), Number(original.gid)) } catch {
|
||||
throw new FileManagerError('cannot preserve file ownership; use the terminal', 403, 'OWNERSHIP_PRESERVE_FAILED')
|
||||
}
|
||||
}
|
||||
}
|
||||
await handle.close()
|
||||
handle = undefined
|
||||
if (original === undefined) {
|
||||
await link(temporary, path)
|
||||
await rm(temporary)
|
||||
} else await rename(temporary, path)
|
||||
await syncDirectory(parent)
|
||||
const saved = await stat(path, { bigint: true })
|
||||
return {
|
||||
path, revision: fileManagerInternals.revisionFor(path, saved), size: body.byteLength,
|
||||
mode: Number(saved.mode & 0o7777n), modifiedAt: new Date(Number(saved.mtimeNs / 1_000_000n)).toISOString(),
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {})
|
||||
await rm(temporary, { force: true }).catch(() => {})
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { constants } from 'node:fs'
|
||||
import { access, lstat, readFile, readdir, readlink, realpath, stat } from 'node:fs/promises'
|
||||
import { basename, dirname, join, normalize, resolve } from 'node:path'
|
||||
|
||||
export const MAX_PATH_BYTES = 4096
|
||||
export const MAX_TEXT_BYTES = 2 * 1024 * 1024
|
||||
export const DEFAULT_LIST_LIMIT = 200
|
||||
export const MAX_LIST_LIMIT = 1000
|
||||
export const SEARCH_MAX_DEPTH = 16
|
||||
export const SEARCH_MAX_SCANNED = 10_000
|
||||
export const SEARCH_MAX_RESULTS = 1000
|
||||
|
||||
const SORTS = new Set(['name', 'size', 'type', 'modifiedAt'])
|
||||
const ORDERS = new Set(['asc', 'desc'])
|
||||
|
||||
export class FileManagerError extends Error {
|
||||
constructor(message, statusCode = 400, code = 'FILE_INVALID') {
|
||||
super(message)
|
||||
this.name = 'FileManagerError'
|
||||
this.statusCode = statusCode
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
export class FileRevisionConflictError extends FileManagerError {
|
||||
constructor(message = 'file revision changed') {
|
||||
super(message, 409, 'REVISION_CONFLICT')
|
||||
this.name = 'FileRevisionConflictError'
|
||||
}
|
||||
}
|
||||
|
||||
function statusCode(error) {
|
||||
if (['EACCES', 'EPERM', 'EROFS'].includes(error?.code)) return 403
|
||||
if (['ENOENT', 'ENOTDIR'].includes(error?.code)) return 404
|
||||
if (error?.code === 'EFBIG') return 413
|
||||
return 400
|
||||
}
|
||||
|
||||
export function fileError(error, fallback = 'file operation failed') {
|
||||
if (error instanceof FileManagerError) return error
|
||||
const wrapped = new FileManagerError(error instanceof Error ? error.message : fallback, statusCode(error), error?.code ?? 'FILE_ERROR')
|
||||
if (error instanceof Error) wrapped.cause = error
|
||||
return wrapped
|
||||
}
|
||||
|
||||
export function normalizeAbsolutePath(value) {
|
||||
if (typeof value !== 'string' || value.length === 0) throw new FileManagerError('path must be a non-empty string')
|
||||
if (!value.startsWith('/')) throw new FileManagerError('path must be absolute')
|
||||
if (Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES) throw new FileManagerError('path is too long')
|
||||
if(/[\u0000-\u001f\u007f]/u.test(value)) throw new FileManagerError('path contains control characters')
|
||||
const path = normalize(value)
|
||||
if (!path.startsWith('/')) throw new FileManagerError('path must be absolute')
|
||||
return path
|
||||
}
|
||||
|
||||
export function createManagedPathMatcher(platformData = '/data/platform') {
|
||||
const data = normalizeAbsolutePath(platformData)
|
||||
const roots = Object.freeze(['/opt/dsh-platform', '/run/dsh-platform', join(data, 'state'), join(data, 'store')])
|
||||
return value => {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
return roots.some(root => path === root || path.startsWith(`${root}/`))
|
||||
}
|
||||
}
|
||||
|
||||
export const isManagedPath = createManagedPathMatcher()
|
||||
|
||||
export function fileManagerLocations({
|
||||
platformData = '/data/platform', dshHome = '/data/dsh', defaultWorkspace = '/workspace',
|
||||
} = {}) {
|
||||
const defaultPath = normalizeAbsolutePath(defaultWorkspace)
|
||||
const shortcuts = [...new Set([defaultPath, normalizeAbsolutePath(dshHome), normalizeAbsolutePath(platformData), '/'])]
|
||||
return Object.freeze({ defaultPath, shortcuts: Object.freeze(shortcuts) })
|
||||
}
|
||||
|
||||
function parseIdentityFile(value, idIndex) {
|
||||
const result = new Map()
|
||||
for (const line of value.split('\n')) {
|
||||
if (line === '' || line.startsWith('#')) continue
|
||||
const fields = line.split(':')
|
||||
const id = Number(fields[idIndex])
|
||||
if (fields[0] !== '' && Number.isSafeInteger(id) && id >= 0 && !result.has(id)) result.set(id, fields[0])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export class UnixIdentityResolver {
|
||||
constructor({ passwdPath = '/etc/passwd', groupPath = '/etc/group' } = {}) {
|
||||
this.passwdPath = passwdPath
|
||||
this.groupPath = groupPath
|
||||
this.loading = undefined
|
||||
}
|
||||
|
||||
async #identities() {
|
||||
this.loading ??= Promise.all([
|
||||
readFile(this.passwdPath, 'utf8').then(value => parseIdentityFile(value, 2), () => new Map()),
|
||||
readFile(this.groupPath, 'utf8').then(value => parseIdentityFile(value, 2), () => new Map()),
|
||||
])
|
||||
return this.loading
|
||||
}
|
||||
|
||||
async resolve(uid, gid) {
|
||||
const [users, groups] = await this.#identities()
|
||||
return { user: users.get(uid) ?? String(uid), group: groups.get(gid) ?? String(gid) }
|
||||
}
|
||||
|
||||
async userId(value) { return this.#identityId(value, 0, 'user') }
|
||||
|
||||
async groupId(value) { return this.#identityId(value, 1, 'group') }
|
||||
|
||||
async #identityId(value, index, label) {
|
||||
if (typeof value !== 'string' || value === '' || value.length > 256 || /[:\u0000-\u001f\u007f]/u.test(value)) {
|
||||
throw new FileManagerError(`${label} is invalid`, 400, 'FILE_IDENTITY_INVALID')
|
||||
}
|
||||
if (/^[0-9]+$/u.test(value)) {
|
||||
const id = Number(value)
|
||||
if (Number.isSafeInteger(id) && id >= 0 && id < 0xffff_ffff) return id
|
||||
throw new FileManagerError(`${label} id is invalid`, 400, 'FILE_IDENTITY_INVALID')
|
||||
}
|
||||
const identities = (await this.#identities())[index]
|
||||
for (const [id, name] of identities) if (name === value) return id
|
||||
throw new FileManagerError(`${label} was not found`, 400, 'FILE_IDENTITY_INVALID')
|
||||
}
|
||||
}
|
||||
|
||||
function kind(stats) {
|
||||
if (stats.isFile()) return 'file'
|
||||
if (stats.isDirectory()) return 'directory'
|
||||
if (stats.isSymbolicLink()) return 'symlink'
|
||||
if (stats.isBlockDevice()) return 'block-device'
|
||||
if (stats.isCharacterDevice()) return 'character-device'
|
||||
if (stats.isFIFO()) return 'fifo'
|
||||
if (stats.isSocket()) return 'socket'
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
function metadata(stats) {
|
||||
return {
|
||||
type: kind(stats),
|
||||
size: Number(stats.size),
|
||||
mode: Number(stats.mode & 0o7777n),
|
||||
uid: Number(stats.uid),
|
||||
gid: Number(stats.gid),
|
||||
modifiedAt: new Date(Number(stats.mtimeNs / 1_000_000n)).toISOString(),
|
||||
createdAt: new Date(Number(stats.birthtimeNs / 1_000_000n)).toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
function revisionFor(path, stats, extra = '') {
|
||||
return `sha256:${createHash('sha256').update(JSON.stringify([
|
||||
path, stats.dev.toString(), stats.ino.toString(), kind(stats), stats.size.toString(),
|
||||
stats.mode.toString(), stats.uid.toString(), stats.gid.toString(), stats.mtimeNs.toString(), stats.ctimeNs.toString(), extra,
|
||||
])).digest('hex')}`
|
||||
}
|
||||
|
||||
async function permissions(path) {
|
||||
const allowed = async mode => access(path, mode).then(() => true, () => false)
|
||||
const [readable, writable] = await Promise.all([allowed(constants.R_OK), allowed(constants.W_OK)])
|
||||
return { readable, writable }
|
||||
}
|
||||
|
||||
async function symlinkDetails(path, stats) {
|
||||
if (!stats.isSymbolicLink()) return {}
|
||||
const linkTarget = await readlink(path)
|
||||
try {
|
||||
const target = await stat(path, { bigint: true })
|
||||
return { linkTarget, targetExists: true, targetType: kind(target), realPath: await realpath(path) }
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT' || error?.code === 'ELOOP') return { linkTarget, targetExists: false, targetType: null, realPath: null }
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function describe(path, stats = undefined, managed = isManagedPath, identity = new UnixIdentityResolver()) {
|
||||
const value = stats ?? await lstat(path, { bigint: true })
|
||||
const link = await symlinkDetails(path, value)
|
||||
const owner = await identity.resolve(Number(value.uid), Number(value.gid))
|
||||
return {
|
||||
name: basename(path) || '/',
|
||||
path,
|
||||
...metadata(value),
|
||||
...owner,
|
||||
...await permissions(path),
|
||||
managed: managed(path),
|
||||
revision: revisionFor(path, value, link.linkTarget ?? ''),
|
||||
...link,
|
||||
}
|
||||
}
|
||||
|
||||
function compareEntries(left, right, sort, order) {
|
||||
if (left.type === 'directory' && right.type !== 'directory') return -1
|
||||
if (left.type !== 'directory' && right.type === 'directory') return 1
|
||||
let result
|
||||
if (sort === 'size') result = left.size - right.size
|
||||
else if (sort === 'modifiedAt') result = left.modifiedAt.localeCompare(right.modifiedAt)
|
||||
else if (sort === 'type') result = left.type.localeCompare(right.type)
|
||||
else result = left.name.localeCompare(right.name, undefined, { numeric: true, sensitivity: 'base' })
|
||||
if (result === 0) result = left.name.localeCompare(right.name)
|
||||
return order === 'desc' ? -result : result
|
||||
}
|
||||
|
||||
function encodeCursor(revision, offset) {
|
||||
return Buffer.from(JSON.stringify({ revision, offset })).toString('base64url')
|
||||
}
|
||||
|
||||
function decodeCursor(value, revision) {
|
||||
if (value === undefined || value === null || value === '') return 0
|
||||
try {
|
||||
const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8'))
|
||||
if (parsed.revision !== revision) throw new FileRevisionConflictError('directory changed while paging')
|
||||
if (!Number.isSafeInteger(parsed.offset) || parsed.offset < 0) throw new Error('invalid offset')
|
||||
return parsed.offset
|
||||
} catch (error) {
|
||||
if (error instanceof FileRevisionConflictError) throw error
|
||||
throw new FileManagerError('cursor is invalid')
|
||||
}
|
||||
}
|
||||
|
||||
function listRevision(path, stats, entries) {
|
||||
const digest = createHash('sha256')
|
||||
for (const entry of entries) digest.update(`${entry.name}\0${entry.revision ?? entry.type}\0`)
|
||||
return revisionFor(path, stats, digest.digest('hex'))
|
||||
}
|
||||
|
||||
function directoryEntryType(entry) {
|
||||
if (entry.isDirectory()) return 'directory'
|
||||
if (entry.isFile()) return 'file'
|
||||
if (entry.isSymbolicLink()) return 'symlink'
|
||||
if (entry.isFIFO()) return 'fifo'
|
||||
if (entry.isSocket()) return 'socket'
|
||||
if (entry.isBlockDevice()) return 'block-device'
|
||||
if (entry.isCharacterDevice()) return 'character-device'
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
function compareNames(left, right, order) {
|
||||
if (left.type === 'directory' && right.type !== 'directory') return -1
|
||||
if (left.type !== 'directory' && right.type === 'directory') return 1
|
||||
const result = left.name.localeCompare(right.name, undefined, { numeric: true, sensitivity: 'base' })
|
||||
|| left.name.localeCompare(right.name)
|
||||
return order === 'desc' ? -result : result
|
||||
}
|
||||
|
||||
function parseLimit(value, fallback = DEFAULT_LIST_LIMIT) {
|
||||
if (value === undefined || value === null || value === '') return fallback
|
||||
const limit = Number(value)
|
||||
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST_LIMIT) throw new FileManagerError(`limit must be between 1 and ${MAX_LIST_LIMIT}`)
|
||||
return limit
|
||||
}
|
||||
|
||||
function parseOffset(value) {
|
||||
if (value === undefined || value === null || value === '') return undefined
|
||||
const offset = Number(value)
|
||||
if (!Number.isSafeInteger(offset) || offset < 0) throw new FileManagerError('offset must be a non-negative integer')
|
||||
return offset
|
||||
}
|
||||
|
||||
export class FileInventory {
|
||||
constructor({ isManaged = isManagedPath, identity = new UnixIdentityResolver() } = {}) {
|
||||
this.isManaged = isManaged
|
||||
this.identity = identity
|
||||
}
|
||||
|
||||
async stat(value) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
try {
|
||||
const item = await describe(path, undefined, this.isManaged, this.identity)
|
||||
return { ...item, realPath: item.realPath ?? await realpath(path).catch(() => path), parent: dirname(path) }
|
||||
} catch (error) {
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async list(value, { cursor, offset: requestedOffset, limit, sort = 'name', order = 'asc' } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
if (!SORTS.has(sort)) throw new FileManagerError('sort is invalid')
|
||||
if (!ORDERS.has(order)) throw new FileManagerError('order is invalid')
|
||||
try {
|
||||
const root = await lstat(path, { bigint: true })
|
||||
if (!root.isDirectory()) throw new FileManagerError('path is not a directory', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const pageSize = parseLimit(limit)
|
||||
const directOffset = parseOffset(requestedOffset)
|
||||
if (directOffset !== undefined && cursor !== undefined && cursor !== null && cursor !== '') throw new FileManagerError('cursor and offset cannot be combined')
|
||||
const names = await readdir(path, { withFileTypes: true })
|
||||
let entries
|
||||
let revision
|
||||
let offset
|
||||
if (sort === 'name') {
|
||||
const summary = names.map(entry => ({ name: entry.name, type: directoryEntryType(entry) }))
|
||||
summary.sort((left, right) => compareNames(left, right, order))
|
||||
revision = listRevision(path, root, summary)
|
||||
offset = directOffset ?? decodeCursor(cursor, revision)
|
||||
entries = await Promise.all(summary.slice(offset, offset + pageSize).map(entry => (
|
||||
describe(resolve(path, entry.name), undefined, this.isManaged, this.identity)
|
||||
)))
|
||||
} else {
|
||||
entries = await Promise.all(names.map(entry => describe(resolve(path, entry.name), undefined, this.isManaged, this.identity)))
|
||||
entries.sort((left, right) => compareEntries(left, right, sort, order))
|
||||
revision = listRevision(path, root, entries)
|
||||
offset = directOffset ?? decodeCursor(cursor, revision)
|
||||
}
|
||||
const page = sort === 'name' ? entries : entries.slice(offset, offset + pageSize)
|
||||
const nextOffset = offset + page.length
|
||||
return {
|
||||
path,
|
||||
realPath: await realpath(path),
|
||||
revision,
|
||||
entries: page,
|
||||
nextCursor: nextOffset < names.length ? encodeCursor(revision, nextOffset) : null,
|
||||
total: names.length,
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async content(value) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
try {
|
||||
const before = await stat(path, { bigint: true })
|
||||
if (!before.isFile()) throw new FileManagerError('path is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
if (before.size > BigInt(MAX_TEXT_BYTES)) throw new FileManagerError('file is too large to edit', 413, 'FILE_TOO_LARGE')
|
||||
const bytes = await readFile(path)
|
||||
const after = await stat(path, { bigint: true })
|
||||
const beforeRevision = revisionFor(path, before)
|
||||
if (beforeRevision !== revisionFor(path, after)) throw new FileRevisionConflictError('file changed while reading')
|
||||
if (bytes.includes(0)) throw new FileManagerError('file is not UTF-8 text', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const content = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
|
||||
return {
|
||||
path,
|
||||
realPath: await realpath(path),
|
||||
content,
|
||||
encoding: 'utf-8',
|
||||
newline: content.includes('\r\n') ? 'crlf' : 'lf',
|
||||
revision: beforeRevision,
|
||||
mode: Number(before.mode & 0o7777n),
|
||||
size: bytes.byteLength,
|
||||
modifiedAt: new Date(Number(before.mtimeNs / 1_000_000n)).toISOString(),
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class FileSearchManager {
|
||||
constructor({ inventory = new FileInventory(), onState = () => {} } = {}) {
|
||||
this.inventory = inventory
|
||||
this.onState = onState
|
||||
this.tasks = new Map()
|
||||
}
|
||||
|
||||
start({ path: value, revision, query }) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
if (typeof query !== 'string' || query.trim().length === 0 || query.length > 256) throw new FileManagerError('search query is invalid')
|
||||
const task = {
|
||||
taskId: randomUUID(), operation: 'search', status: 'running', path, revision, query,
|
||||
scanned: 0, matches: [], truncated: false, error: null, cancelRequested: false,
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#publish(task)
|
||||
task.completion = this.#run(task).catch(error => {
|
||||
task.status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
this.#publish(task)
|
||||
})
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
get(taskId, { cursor, limit } = {}) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
const offset = cursor === undefined || cursor === '' ? 0 : Number(cursor)
|
||||
if (!Number.isSafeInteger(offset) || offset < 0) throw new FileManagerError('task cursor is invalid')
|
||||
const size = parseLimit(limit)
|
||||
return { ...this.public(task), results: task.matches.slice(offset, offset + size), nextCursor: offset + size < task.matches.length ? String(offset + size) : null }
|
||||
}
|
||||
|
||||
list() {
|
||||
return [...this.tasks.values()].map(task => this.public(task)).reverse()
|
||||
}
|
||||
|
||||
cancel(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
if (task.status !== 'running') throw new FileManagerError('file task is already complete', 409, 'FILE_TASK_COMPLETE')
|
||||
task.cancelRequested = true
|
||||
this.#publish(task)
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
public(task) {
|
||||
const { completion, matches, cancelRequested, ...value } = task
|
||||
return { ...value, resultCount: matches.length, cancelRequested }
|
||||
}
|
||||
|
||||
#publish(task) {
|
||||
task.updatedAt = new Date().toISOString()
|
||||
this.onState(this.public(task))
|
||||
}
|
||||
|
||||
async #run(task) {
|
||||
const root = await this.inventory.list(task.path, { limit: 1 })
|
||||
if (task.revision !== undefined && task.revision !== null && task.revision !== root.revision) throw new FileRevisionConflictError('search root changed')
|
||||
task.revision = root.revision
|
||||
const needle = task.query.toLocaleLowerCase()
|
||||
const pending = [{ path: task.path, depth: 0 }]
|
||||
while (pending.length > 0 && task.scanned < SEARCH_MAX_SCANNED && task.matches.length < SEARCH_MAX_RESULTS) {
|
||||
if (task.cancelRequested) {
|
||||
task.status = 'cancelled'
|
||||
this.#publish(task)
|
||||
return
|
||||
}
|
||||
const current = pending.shift()
|
||||
let names
|
||||
try { names = await readdir(current.path, { withFileTypes: true }) } catch (error) {
|
||||
if (['EACCES', 'EPERM', 'ENOENT'].includes(error?.code)) continue
|
||||
throw error
|
||||
}
|
||||
for (const entry of names) {
|
||||
task.scanned += 1
|
||||
const child = resolve(current.path, entry.name)
|
||||
const relative = child.slice(task.path === '/' ? 1 : task.path.length + 1)
|
||||
if (relative.toLocaleLowerCase().includes(needle)) task.matches.push(await describe(child, undefined, this.inventory.isManaged, this.inventory.identity))
|
||||
if (entry.isDirectory() && !entry.isSymbolicLink() && current.depth < SEARCH_MAX_DEPTH) pending.push({ path: child, depth: current.depth + 1 })
|
||||
if (task.scanned >= SEARCH_MAX_SCANNED || task.matches.length >= SEARCH_MAX_RESULTS) break
|
||||
}
|
||||
this.#publish(task)
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
task.truncated = pending.length > 0 || task.scanned >= SEARCH_MAX_SCANNED || task.matches.length >= SEARCH_MAX_RESULTS
|
||||
task.status = 'success'
|
||||
this.#publish(task)
|
||||
}
|
||||
}
|
||||
|
||||
export class FileSizeManager {
|
||||
constructor({ inventory = new FileInventory(), onState = () => {} } = {}) {
|
||||
this.inventory = inventory
|
||||
this.onState = onState
|
||||
this.tasks = new Map()
|
||||
}
|
||||
|
||||
start({ path: value, revision } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
const task = {
|
||||
taskId: randomUUID(), operation: 'size', status: 'running', path, revision,
|
||||
bytes: 0, entries: 0, error: null, cancelRequested: false,
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#publish(task)
|
||||
task.completion = this.#run(task).catch(error => {
|
||||
task.status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
this.#publish(task)
|
||||
})
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
get(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
list() { return [...this.tasks.values()].map(task => this.public(task)).reverse() }
|
||||
|
||||
cancel(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
if (task.status !== 'running') throw new FileManagerError('file task is already complete', 409, 'FILE_TASK_COMPLETE')
|
||||
task.cancelRequested = true
|
||||
this.#publish(task)
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
public(task) {
|
||||
const { completion, cancelRequested, ...value } = task
|
||||
return { ...value, cancelRequested }
|
||||
}
|
||||
|
||||
#publish(task) {
|
||||
task.updatedAt = new Date().toISOString()
|
||||
this.onState(this.public(task))
|
||||
}
|
||||
|
||||
async #run(task) {
|
||||
const root = await this.inventory.stat(task.path)
|
||||
if (root.type !== 'directory') throw new FileManagerError('size target is not a directory', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
if (typeof task.revision !== 'string' || task.revision !== root.revision) throw new FileRevisionConflictError('size target changed')
|
||||
const pending = [task.path]
|
||||
while (pending.length > 0) {
|
||||
if (task.cancelRequested) {
|
||||
task.status = 'cancelled'
|
||||
this.#publish(task)
|
||||
return
|
||||
}
|
||||
const directory = pending.pop()
|
||||
for (const name of await readdir(directory)) {
|
||||
const path = resolve(directory, name)
|
||||
const details = await lstat(path, { bigint: true })
|
||||
task.entries += 1
|
||||
if (details.isDirectory() && !details.isSymbolicLink()) pending.push(path)
|
||||
else if (details.isFile()) task.bytes += Number(details.size)
|
||||
}
|
||||
this.#publish(task)
|
||||
await new Promise(resolveTask => setImmediate(resolveTask))
|
||||
}
|
||||
task.status = 'success'
|
||||
this.#publish(task)
|
||||
}
|
||||
}
|
||||
|
||||
export const fileManagerInternals = Object.freeze({ describe, revisionFor, parseIdentityFile, parseLimit })
|
||||
@@ -0,0 +1,645 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { chmod, cp, lchown, lstat, mkdir, open, readdir, readFile, readlink, rename, rm } from 'node:fs/promises'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import { durableReplace } from '../../../platform/lib/atomic.mjs'
|
||||
import { createArchive, extractArchive, normalizeArchiveFormat } from './archives.mjs'
|
||||
import {
|
||||
FileInventory, FileManagerError, FileRevisionConflictError, FileSearchManager, FileSizeManager, UnixIdentityResolver,
|
||||
fileError, isManagedPath, normalizeAbsolutePath,
|
||||
} from './index.mjs'
|
||||
|
||||
const OPERATIONS = new Set(['mkdir', 'touch', 'rename', 'copy', 'move', 'delete', 'attributes', 'archive', 'extract'])
|
||||
const DEFAULT_PROTECTED_DELETE_ROOTS = ['/', '/data', '/data/dsh', '/data/platform', '/workspace', '/run/dsh-platform/deployment']
|
||||
|
||||
function now() { return new Date().toISOString() }
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
function publicTask(task) {
|
||||
const { completion, persisted, executor, resolveResult, rejectResult, cancelRequested, abortController, ...value } = task
|
||||
return { ...value, cancelRequested: cancelRequested === true }
|
||||
}
|
||||
|
||||
function cleanName(path) {
|
||||
const name = basename(path)
|
||||
if (name === '' || name === '.' || name === '..') throw new FileManagerError('destination name is invalid')
|
||||
return name
|
||||
}
|
||||
|
||||
function renamedPath(path, index) {
|
||||
const dot = basename(path).lastIndexOf('.')
|
||||
const directory = dirname(path)
|
||||
const name = basename(path)
|
||||
const stem = dot > 0 ? name.slice(0, dot) : name
|
||||
const extension = dot > 0 ? name.slice(dot) : ''
|
||||
return join(directory, `${stem} (${String(index)})${extension}`)
|
||||
}
|
||||
|
||||
async function uniqueDestination(path, conflict) {
|
||||
if (!await exists(path)) return path
|
||||
if (conflict === 'reject') throw new FileManagerError('destination already exists', 409, 'FILE_EXISTS')
|
||||
if (conflict === 'overwrite') return path
|
||||
for (let index = 1; index < 10_000; index += 1) {
|
||||
const candidate = renamedPath(path, index)
|
||||
if (!await exists(candidate)) return candidate
|
||||
}
|
||||
throw new FileManagerError('could not choose a unique destination', 409, 'FILE_EXISTS')
|
||||
}
|
||||
|
||||
async function copyEntry(source, staging) {
|
||||
const sourceStats = await lstat(source)
|
||||
if (sourceStats.isDirectory()) {
|
||||
await cp(source, staging, { recursive: true, dereference: false, errorOnExist: true, force: false, preserveTimestamps: true })
|
||||
} else if (sourceStats.isSymbolicLink()) {
|
||||
const target = await readlink(source)
|
||||
await import('node:fs/promises').then(fs => fs.symlink(target, staging))
|
||||
} else if (sourceStats.isFile()) {
|
||||
await cp(source, staging, { dereference: false, errorOnExist: true, force: false, preserveTimestamps: true })
|
||||
} else throw new FileManagerError('special files cannot be copied', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
}
|
||||
|
||||
async function treeMetrics(path) {
|
||||
const value = await lstat(path)
|
||||
if (!value.isDirectory() || value.isSymbolicLink()) return { entries: 1, bytes: value.isFile() ? value.size : 0 }
|
||||
let entries = 1
|
||||
let bytes = 0
|
||||
for (const name of await readdir(path)) {
|
||||
const child = await treeMetrics(join(path, name))
|
||||
entries += child.entries
|
||||
bytes += child.bytes
|
||||
}
|
||||
return { entries, bytes }
|
||||
}
|
||||
|
||||
function normalizeAttributes(value) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).some(key => !['user', 'group', 'mode', 'recursive'].includes(key))
|
||||
|| typeof value.user !== 'string' || typeof value.group !== 'string'
|
||||
|| typeof value.mode !== 'string' || !/^[0-7]{3,4}$/u.test(value.mode)
|
||||
|| typeof value.recursive !== 'boolean') throw new FileManagerError('file attributes are invalid')
|
||||
return { user: value.user, group: value.group, mode: value.mode, recursive: value.recursive }
|
||||
}
|
||||
|
||||
function assertAttributesApplied(path, details, { uid, gid, mode }, symbolicLink) {
|
||||
const ownershipMatches = details.uid === uid && details.gid === gid
|
||||
const modeMatches = symbolicLink || (details.mode & 0o7777) === mode
|
||||
if (ownershipMatches && modeMatches) return
|
||||
throw new FileManagerError(
|
||||
`filesystem did not apply Unix ownership or mode for ${path}; use a filesystem mounted with Unix metadata support`,
|
||||
403,
|
||||
'FILE_ATTRIBUTES_UNSUPPORTED',
|
||||
)
|
||||
}
|
||||
|
||||
export class FileTaskManager {
|
||||
constructor({
|
||||
root, inventory = new FileInventory(), isManaged = isManagedPath,
|
||||
protectedRoots = DEFAULT_PROTECTED_DELETE_ROOTS,
|
||||
onState = () => {}, platformBusy = () => false, report = async () => {}, identity,
|
||||
} = {}) {
|
||||
if (typeof root !== 'string') throw new Error('file task root is required')
|
||||
this.root = root
|
||||
this.inventory = inventory
|
||||
this.isManaged = isManaged
|
||||
this.protectedRoots = new Set(protectedRoots.map(normalizeAbsolutePath))
|
||||
this.onState = onState
|
||||
this.platformBusy = platformBusy
|
||||
this.report = report
|
||||
this.identity = identity ?? inventory.identity ?? new UnixIdentityResolver()
|
||||
this.tasks = new Map()
|
||||
this.activeMutation = undefined
|
||||
this.queue = []
|
||||
this.queueTail = Promise.resolve()
|
||||
this.search = new FileSearchManager({ inventory, onState: state => onState(state) })
|
||||
this.sizes = new FileSizeManager({ inventory, onState: state => onState(state) })
|
||||
}
|
||||
|
||||
get hasManagedMutation() { return this.queue.some(task => ['queued', 'running'].includes(task.status) && task.managed === true) }
|
||||
|
||||
wouldManage(input) {
|
||||
if (['search', 'size'].includes(input?.operation)) return false
|
||||
const sourceManaged = Array.isArray(input?.sources) && input.sources.some(source => typeof source?.path === 'string' && this.isManaged(source.path))
|
||||
return sourceManaged || (typeof input?.destination === 'string' && this.isManaged(input.destination))
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const files = (await readdir(this.root)).filter(name => name.endsWith('.json')).sort()
|
||||
for (const name of files) {
|
||||
try {
|
||||
const task = JSON.parse(await readFile(join(this.root, name), 'utf8'))
|
||||
if (task.schema !== 1 || typeof task.taskId !== 'string') continue
|
||||
this.tasks.set(task.taskId, task)
|
||||
if (task.status === 'running') await this.#recover(task)
|
||||
else if (task.status === 'queued' && ['upload', 'download'].includes(task.operation)) {
|
||||
task.status = 'running'
|
||||
await this.#recover(task)
|
||||
} else if (task.status === 'queued') this.#schedule(task)
|
||||
} catch (error) {
|
||||
this.onState({ operation: 'recovery', status: 'failed', error: error instanceof Error ? error.message : String(error) })
|
||||
}
|
||||
}
|
||||
await this.#prune()
|
||||
}
|
||||
|
||||
start(input) {
|
||||
if (input?.operation === 'search') return this.search.start(input)
|
||||
if (input?.operation === 'size') return this.sizes.start(input)
|
||||
if (input === null || typeof input !== 'object' || !OPERATIONS.has(input.operation)) throw new FileManagerError('file task operation is invalid')
|
||||
const sources = Array.isArray(input.sources) ? input.sources.map(source => ({ path: normalizeAbsolutePath(source.path), revision: source.revision })) : []
|
||||
const destination = input.destination === undefined ? null : normalizeAbsolutePath(input.destination)
|
||||
const conflict = input.conflict ?? 'reject'
|
||||
if (!['reject', 'overwrite', 'rename'].includes(conflict)) throw new FileManagerError('file task conflict mode is invalid')
|
||||
if (['copy', 'move', 'delete', 'rename', 'archive', 'extract'].includes(input.operation) && sources.length === 0) throw new FileManagerError('file task sources are required')
|
||||
if (input.operation === 'extract' && sources.length !== 1) throw new FileManagerError('archive extraction requires one source')
|
||||
if (input.operation === 'attributes' && sources.length !== 1) throw new FileManagerError('attribute changes require one source')
|
||||
if (['mkdir', 'touch', 'copy', 'move', 'rename', 'archive', 'extract'].includes(input.operation) && destination === null) throw new FileManagerError('file task destination is required')
|
||||
if (input.operation === 'delete' && sources.some(source => this.protectedRoots.has(source.path))) throw new FileManagerError('protected root cannot be deleted', 403, 'PROTECTED_ROOT')
|
||||
const managed = sources.some(source => this.isManaged(source.path)) || (destination !== null && this.isManaged(destination))
|
||||
if (managed && this.platformBusy()) throw new FileManagerError('a platform operation is already running', 409, 'FILE_TASK_CONFLICT')
|
||||
const task = {
|
||||
schema: 1, taskId: randomUUID(), operation: input.operation,
|
||||
status: this.queue.length === 0 ? 'running' : 'queued', phase: this.queue.length === 0 ? 'validating' : 'queued',
|
||||
sources, destination, destinationRevision: input.destinationRevision ?? null, conflict, managed,
|
||||
attributes: input.operation === 'attributes' ? normalizeAttributes(input.attributes) : null,
|
||||
archiveFormat: ['archive', 'extract'].includes(input.operation) ? normalizeArchiveFormat(input.archiveFormat) : null,
|
||||
processedBytes: 0, totalBytes: 0, processedEntries: 0, totalEntries: 0, currentPath: null,
|
||||
published: [], staging: [], hidden: [], currentSource: null, currentDestination: null,
|
||||
error: null, errorCode: null, queuePosition: null, createdAt: now(), updatedAt: now(), cancelRequested: false,
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#schedule(task)
|
||||
return publicTask(task)
|
||||
}
|
||||
|
||||
startTransfer({ operation, path, totalBytes = null, managed = false }, executor) {
|
||||
if (!['upload', 'download'].includes(operation) || typeof executor !== 'function') {
|
||||
throw new FileManagerError('file transfer task is invalid')
|
||||
}
|
||||
const normalizedPath = normalizeAbsolutePath(path)
|
||||
if (managed && this.platformBusy()) throw new FileManagerError('a platform operation is already running', 409, 'FILE_TASK_CONFLICT')
|
||||
let resolveResult
|
||||
let rejectResult
|
||||
const result = new Promise((resolve, reject) => { resolveResult = resolve; rejectResult = reject })
|
||||
const task = {
|
||||
schema: 1, taskId: randomUUID(), operation,
|
||||
status: this.queue.length === 0 ? 'running' : 'queued', phase: this.queue.length === 0 ? 'transferring' : 'queued',
|
||||
path: normalizedPath, sources: [], destination: operation === 'upload' ? normalizedPath : null,
|
||||
destinationRevision: null, conflict: null, managed, attributes: null, archiveFormat: null,
|
||||
processedBytes: 0, totalBytes, processedEntries: 0, totalEntries: 1, currentPath: normalizedPath,
|
||||
published: [], staging: [], hidden: [], currentSource: null, currentDestination: null,
|
||||
error: null, errorCode: null, queuePosition: null, createdAt: now(), updatedAt: now(), cancelRequested: false,
|
||||
executor, resolveResult, rejectResult,
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#schedule(task)
|
||||
return { task: publicTask(task), result }
|
||||
}
|
||||
|
||||
list() {
|
||||
return [...this.search.list(), ...this.sizes.list(), ...[...this.tasks.values()].map(publicTask)].sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, 100)
|
||||
}
|
||||
|
||||
get(taskId, options = {}) {
|
||||
if (this.tasks.has(taskId)) return publicTask(this.tasks.get(taskId))
|
||||
try { return this.search.get(taskId, options) } catch (error) {
|
||||
if (error?.statusCode !== 404) throw error
|
||||
return this.sizes.get(taskId)
|
||||
}
|
||||
}
|
||||
|
||||
completion(taskId) {
|
||||
const task = this.tasks.get(taskId) ?? this.search.tasks.get(taskId) ?? this.sizes.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
return Promise.resolve(task.completion).then(() => this.get(taskId))
|
||||
}
|
||||
|
||||
cancel(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) {
|
||||
try { return this.search.cancel(taskId) } catch (error) {
|
||||
if (error?.statusCode !== 404) throw error
|
||||
return this.sizes.cancel(taskId)
|
||||
}
|
||||
}
|
||||
if (!['queued', 'running'].includes(task.status)) throw new FileManagerError('file task is already complete', 409, 'FILE_TASK_COMPLETE')
|
||||
if (task.status === 'queued') {
|
||||
task.cancelRequested = true
|
||||
task.status = 'cancelled'
|
||||
task.phase = 'cancelled'
|
||||
task.updatedAt = now()
|
||||
this.queue = this.queue.filter(value => value !== task)
|
||||
void Promise.resolve(task.persisted).then(() => this.#terminal(task, 'cancelled', 'cancelled'))
|
||||
task.rejectResult?.(Object.assign(new Error('file transfer cancelled'), { code: 'FILE_TASK_CANCELLED' }))
|
||||
this.#publishQueue()
|
||||
return publicTask(task)
|
||||
}
|
||||
if (['destination-committed', 'source-hidden', 'cleaning'].includes(task.phase)) throw new FileManagerError('file task passed its cancellation boundary', 409, 'FILE_TASK_COMMITTED')
|
||||
task.cancelRequested = true
|
||||
task.abortController?.abort()
|
||||
this.#publish(task)
|
||||
return publicTask(task)
|
||||
}
|
||||
|
||||
#schedule(task) {
|
||||
this.queue.push(task)
|
||||
this.#publishQueue()
|
||||
const previous = this.queueTail
|
||||
task.persisted = this.#persist(task)
|
||||
task.completion = Promise.all([previous.catch(() => {}), task.persisted]).then(async () => {
|
||||
if (task.cancelRequested || task.status === 'cancelled') return
|
||||
task.status = 'running'
|
||||
task.phase = 'validating'
|
||||
task.queuePosition = 0
|
||||
this.activeMutation = task
|
||||
await this.#phase(task, task.executor === undefined ? 'validating' : 'transferring')
|
||||
if (task.executor === undefined) await this.#run(task).catch(() => {})
|
||||
else await this.#runTransfer(task)
|
||||
}).finally(() => {
|
||||
if (this.activeMutation === task) this.activeMutation = undefined
|
||||
this.queue = this.queue.filter(value => value !== task)
|
||||
this.#publishQueue()
|
||||
})
|
||||
this.queueTail = task.completion
|
||||
}
|
||||
|
||||
async #runTransfer(task) {
|
||||
const controller = new AbortController()
|
||||
task.abortController = controller
|
||||
const controls = {
|
||||
signal: controller.signal,
|
||||
progress: ({ processedBytes, totalBytes, currentPath } = {}) => {
|
||||
if (Number.isSafeInteger(processedBytes) && processedBytes >= 0) task.processedBytes = processedBytes
|
||||
if (Number.isSafeInteger(totalBytes) && totalBytes >= 0) task.totalBytes = totalBytes
|
||||
if (typeof currentPath === 'string') task.currentPath = currentPath
|
||||
task.updatedAt = now()
|
||||
this.#publish(task)
|
||||
},
|
||||
}
|
||||
try {
|
||||
const result = await task.executor(controls)
|
||||
task.processedEntries = 1
|
||||
if (task.totalBytes !== null) task.processedBytes = task.totalBytes
|
||||
await this.#terminal(task, 'success', 'completed')
|
||||
await this.report(`file-task.${task.operation}.completed`, publicTask(task))
|
||||
task.resolveResult(result)
|
||||
} catch (error) {
|
||||
const status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#terminal(task, status, status)
|
||||
await this.report(`file-task.${task.operation}.${status}`, { ...publicTask(task), error })
|
||||
task.rejectResult(error)
|
||||
} finally {
|
||||
task.abortController = undefined
|
||||
await this.#prune()
|
||||
}
|
||||
}
|
||||
|
||||
#publishQueue() {
|
||||
let position = 0
|
||||
for (const task of this.queue) {
|
||||
if (task.status === 'queued') position += 1
|
||||
task.queuePosition = task.status === 'queued' ? position : 0
|
||||
this.#publish(task)
|
||||
}
|
||||
}
|
||||
|
||||
async #validate(task) {
|
||||
for (const source of task.sources) {
|
||||
const item = await this.inventory.stat(source.path)
|
||||
if (source.revision !== item.revision) throw new FileRevisionConflictError(`source changed: ${source.path}`)
|
||||
if (task.operation === 'attributes' && !['file', 'directory'].includes(item.type)) {
|
||||
throw new FileManagerError('attributes can be changed only for files and directories', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
}
|
||||
const metrics = task.operation === 'attributes' && task.attributes.recursive === false
|
||||
? { entries: 1, bytes: item.type === 'file' ? item.size : 0 }
|
||||
: await treeMetrics(source.path)
|
||||
task.totalEntries += metrics.entries
|
||||
task.totalBytes += metrics.bytes
|
||||
}
|
||||
if (task.destinationRevision !== null) {
|
||||
const destinationRoot = task.operation === 'rename' || task.operation === 'mkdir' || task.operation === 'touch'
|
||||
? dirname(task.destination) : task.destination
|
||||
const listed = await this.inventory.list(destinationRoot, { limit: 1 })
|
||||
if (listed.revision !== task.destinationRevision) throw new FileRevisionConflictError('destination directory changed')
|
||||
}
|
||||
}
|
||||
|
||||
async #run(task) {
|
||||
try {
|
||||
await this.#validate(task)
|
||||
await this.#phase(task, 'mutating')
|
||||
if (task.operation === 'mkdir') await mkdir(task.destination)
|
||||
else if (task.operation === 'touch') {
|
||||
const handle = await open(task.destination, 'wx', 0o644)
|
||||
await handle.sync(); await handle.close(); await syncDirectory(dirname(task.destination))
|
||||
} else if (task.operation === 'rename') await this.#rename(task)
|
||||
else if (task.operation === 'copy') await this.#copy(task, false)
|
||||
else if (task.operation === 'move') await this.#copy(task, true)
|
||||
else if (task.operation === 'delete') await this.#delete(task)
|
||||
else if (task.operation === 'attributes') await this.#attributes(task)
|
||||
else if (task.operation === 'archive') await this.#archive(task)
|
||||
else if (task.operation === 'extract') await this.#extract(task)
|
||||
await this.#terminal(task, 'success', 'completed')
|
||||
await this.report(`file-task.${task.operation}.completed`, publicTask(task))
|
||||
} catch (error) {
|
||||
const status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#cleanupStaging(task)
|
||||
await this.#terminal(task, status, status)
|
||||
await this.report(`file-task.${task.operation}.${status}`, { ...publicTask(task), error })
|
||||
throw error
|
||||
} finally {
|
||||
await this.#prune()
|
||||
}
|
||||
}
|
||||
|
||||
async #rename(task) {
|
||||
if (task.sources.length !== 1) throw new FileManagerError('rename requires one source')
|
||||
const source = task.sources[0].path
|
||||
if (dirname(source) !== dirname(task.destination)) throw new FileManagerError('rename destination must use the same directory')
|
||||
const destination = await uniqueDestination(task.destination, task.conflict)
|
||||
task.currentSource = source
|
||||
task.currentDestination = destination
|
||||
await this.#phase(task, 'move-prepared')
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(source, destination)
|
||||
task.published.push(destination)
|
||||
await this.#phase(task, 'destination-committed')
|
||||
await syncDirectory(dirname(destination))
|
||||
task.currentSource = null
|
||||
task.currentDestination = null
|
||||
await this.#phase(task, 'mutating')
|
||||
}
|
||||
|
||||
async #copy(task, move) {
|
||||
for (const source of task.sources) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const destination = await uniqueDestination(join(task.destination, cleanName(source.path)), task.conflict)
|
||||
if (source.path === destination || destination.startsWith(`${source.path}/`)) throw new FileManagerError('directory cannot be copied into itself')
|
||||
if (move) {
|
||||
task.currentSource = source.path
|
||||
task.currentDestination = destination
|
||||
await this.#phase(task, 'move-prepared')
|
||||
try {
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(source.path, destination)
|
||||
task.published.push(destination)
|
||||
await this.#phase(task, 'destination-committed')
|
||||
task.processedEntries += 1
|
||||
task.currentSource = null
|
||||
task.currentDestination = null
|
||||
await this.#phase(task, 'mutating')
|
||||
continue
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EXDEV') throw error
|
||||
}
|
||||
}
|
||||
const staging = join(dirname(destination), `.dsh-file-task-${task.taskId}-${randomUUID()}.tmp`)
|
||||
task.staging.push(staging)
|
||||
task.currentPath = source.path
|
||||
task.currentSource = source.path
|
||||
task.currentDestination = destination
|
||||
await this.#persist(task)
|
||||
await copyEntry(source.path, staging)
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(staging, destination)
|
||||
task.staging = task.staging.filter(path => path !== staging)
|
||||
task.published.push(destination)
|
||||
await this.#phase(task, move ? 'destination-committed' : 'mutating')
|
||||
if (move) await rm(source.path, { recursive: true })
|
||||
task.processedEntries += 1
|
||||
task.processedBytes = Math.min(task.totalBytes, task.processedBytes + (await treeMetrics(destination)).bytes)
|
||||
task.currentSource = null
|
||||
task.currentDestination = null
|
||||
await this.#phase(task, 'mutating')
|
||||
}
|
||||
}
|
||||
|
||||
async #delete(task) {
|
||||
for (const source of task.sources) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const hidden = join(dirname(source.path), `.dsh-delete-${task.taskId}-${randomUUID()}.tmp`)
|
||||
await rename(source.path, hidden)
|
||||
task.hidden.push(hidden)
|
||||
await this.#phase(task, 'source-hidden')
|
||||
const removed = await treeMetrics(hidden)
|
||||
await rm(hidden, { recursive: true })
|
||||
task.hidden = task.hidden.filter(path => path !== hidden)
|
||||
task.processedEntries += removed.entries
|
||||
task.processedBytes = Math.min(task.totalBytes, task.processedBytes + removed.bytes)
|
||||
await this.#phase(task, 'cleaning')
|
||||
}
|
||||
}
|
||||
|
||||
async #attributes(task) {
|
||||
const [uid, gid] = await Promise.all([
|
||||
this.identity.userId(task.attributes.user),
|
||||
this.identity.groupId(task.attributes.group),
|
||||
])
|
||||
const mode = Number.parseInt(task.attributes.mode, 8)
|
||||
const pending = [task.sources[0].path]
|
||||
task.resolvedAttributes = { uid, gid, mode }
|
||||
while (pending.length > 0) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const path = pending.pop()
|
||||
const details = await lstat(path)
|
||||
if (task.attributes.recursive && details.isDirectory() && !details.isSymbolicLink()) {
|
||||
for (const name of await readdir(path)) pending.push(join(path, name))
|
||||
}
|
||||
await lchown(path, uid, gid)
|
||||
if (!details.isSymbolicLink()) await chmod(path, mode)
|
||||
assertAttributesApplied(path, await lstat(path), { uid, gid, mode }, details.isSymbolicLink())
|
||||
task.currentPath = path
|
||||
task.processedEntries += 1
|
||||
if (details.isFile()) task.processedBytes += details.size
|
||||
if (task.processedEntries % 100 === 0) await this.#phase(task, 'mutating')
|
||||
}
|
||||
task.currentPath = null
|
||||
}
|
||||
|
||||
async #archive(task) {
|
||||
const output = await uniqueDestination(task.destination, task.conflict)
|
||||
const sourceRoot = join(dirname(output), `.dsh-archive-input-${task.taskId}`)
|
||||
const staging = join(dirname(output), `.dsh-archive-${task.taskId}.tmp`)
|
||||
task.staging.push(sourceRoot, staging)
|
||||
await mkdir(sourceRoot)
|
||||
const names = new Set()
|
||||
for (const source of task.sources) {
|
||||
const name = cleanName(source.path)
|
||||
if (names.has(name)) throw new FileManagerError('archive source names must be unique', 409, 'FILE_EXISTS')
|
||||
names.add(name)
|
||||
task.currentPath = source.path
|
||||
await copyEntry(source.path, join(sourceRoot, name))
|
||||
task.processedEntries += 1
|
||||
await this.#phase(task, 'staging')
|
||||
}
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const controller = new AbortController()
|
||||
task.abortController = controller
|
||||
try {
|
||||
await createArchive({ format: task.archiveFormat, sourceRoot, output: staging, signal: controller.signal })
|
||||
} finally {
|
||||
task.abortController = undefined
|
||||
}
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
if (task.conflict === 'overwrite' && await exists(output)) await rm(output, { recursive: true })
|
||||
await rename(staging, output)
|
||||
task.staging = task.staging.filter(path => path !== staging)
|
||||
task.published.push(output)
|
||||
task.processedBytes = task.totalBytes
|
||||
await rm(sourceRoot, { recursive: true, force: true })
|
||||
task.staging = task.staging.filter(path => path !== sourceRoot)
|
||||
task.currentPath = null
|
||||
await this.#phase(task, 'destination-committed')
|
||||
}
|
||||
|
||||
async #extract(task) {
|
||||
const archive = task.sources[0].path
|
||||
const staging = join(task.destination, `.dsh-extract-${task.taskId}.tmp`)
|
||||
task.staging.push(staging)
|
||||
task.currentPath = archive
|
||||
await this.#persist(task)
|
||||
const controller = new AbortController()
|
||||
task.abortController = controller
|
||||
let metrics
|
||||
try {
|
||||
metrics = await extractArchive({ format: task.archiveFormat, archive, output: staging, signal: controller.signal })
|
||||
} finally {
|
||||
task.abortController = undefined
|
||||
}
|
||||
task.totalEntries = metrics.entries
|
||||
task.totalBytes = metrics.bytes
|
||||
await this.#publishExtraction(task, staging)
|
||||
}
|
||||
|
||||
async #publishExtraction(task, staging) {
|
||||
const plan = []
|
||||
for (const name of await readdir(staging)) {
|
||||
plan.push({ source: join(staging, name), destination: await uniqueDestination(join(task.destination, name), task.conflict) })
|
||||
}
|
||||
for (const { source, destination } of plan) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(source, destination)
|
||||
task.published.push(destination)
|
||||
const item = await treeMetrics(destination)
|
||||
task.processedEntries += item.entries
|
||||
task.processedBytes += item.bytes
|
||||
await this.#phase(task, 'publishing')
|
||||
}
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
task.staging = task.staging.filter(path => path !== staging)
|
||||
task.currentPath = null
|
||||
}
|
||||
|
||||
async #cleanupStaging(task) {
|
||||
for (const path of task.staging) await rm(path, { recursive: true, force: true }).catch(() => {})
|
||||
task.staging = []
|
||||
}
|
||||
|
||||
async #recover(task) {
|
||||
if (task.operation === 'attributes') {
|
||||
try {
|
||||
task.cancelRequested = false
|
||||
task.processedEntries = 0
|
||||
task.processedBytes = 0
|
||||
await this.#phase(task, 'mutating')
|
||||
await this.#attributes(task)
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
await this.report('file-task.attributes.recovered', publicTask(task))
|
||||
} catch (error) {
|
||||
task.status = 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#phase(task, 'failed')
|
||||
await this.report('file-task.attributes.recovery-failed', { ...publicTask(task), error })
|
||||
}
|
||||
} else if (task.operation === 'archive' && task.phase === 'destination-committed') {
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else if (task.operation === 'extract' && task.phase === 'publishing' && task.staging?.length === 1 && await exists(task.staging[0])) {
|
||||
try {
|
||||
task.cancelRequested = false
|
||||
await this.#publishExtraction(task, task.staging[0])
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
await this.report('file-task.extract.recovered', publicTask(task))
|
||||
} catch (error) {
|
||||
task.status = 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#phase(task, 'failed')
|
||||
await this.report('file-task.extract.recovery-failed', { ...publicTask(task), error })
|
||||
}
|
||||
} else if (task.operation === 'delete' || task.phase === 'source-hidden' || task.phase === 'cleaning') {
|
||||
for (const path of task.hidden ?? []) await rm(path, { recursive: true, force: true })
|
||||
task.hidden = []
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else if (['move', 'rename'].includes(task.operation) && task.phase === 'destination-committed') {
|
||||
if (task.currentSource !== null && await exists(task.currentSource)) await rm(task.currentSource, { recursive: true })
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else if (['move', 'rename'].includes(task.operation) && task.phase === 'move-prepared'
|
||||
&& task.currentSource !== null && task.currentDestination !== null
|
||||
&& !await exists(task.currentSource) && await exists(task.currentDestination)) {
|
||||
task.published ??= []
|
||||
if (!task.published.includes(task.currentDestination)) task.published.push(task.currentDestination)
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else {
|
||||
await this.#cleanupStaging(task)
|
||||
task.status = 'interrupted'
|
||||
task.error = 'file operation was interrupted before a recoverable commit'
|
||||
await this.#phase(task, 'interrupted')
|
||||
}
|
||||
}
|
||||
|
||||
async #phase(task, phase) {
|
||||
task.phase = phase
|
||||
task.updatedAt = now()
|
||||
await this.#persist(task)
|
||||
this.#publish(task)
|
||||
}
|
||||
|
||||
async #terminal(task, status, phase) {
|
||||
task.status = status
|
||||
task.phase = phase
|
||||
task.updatedAt = now()
|
||||
await this.#persist(task)
|
||||
if (this.activeMutation === task) this.activeMutation = undefined
|
||||
this.queue = this.queue.filter(value => value !== task)
|
||||
this.#publishQueue()
|
||||
this.#publish(task)
|
||||
}
|
||||
|
||||
async #persist(task) {
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const value = publicTask(task)
|
||||
await durableReplace(join(this.root, `${task.taskId}.json`), Buffer.from(`${JSON.stringify(value)}\n`))
|
||||
}
|
||||
|
||||
#publish(task) { this.onState(publicTask(task)) }
|
||||
|
||||
async #prune() {
|
||||
const completed = [...this.tasks.values()].filter(task => !['queued', 'running'].includes(task.status)).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
|
||||
for (const task of completed.slice(100)) {
|
||||
this.tasks.delete(task.taskId)
|
||||
await rm(join(this.root, `${task.taskId}.json`), { force: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const fileTaskInternals = Object.freeze({ protectedDeleteRoots: DEFAULT_PROTECTED_DELETE_ROOTS, uniqueDestination, treeMetrics, normalizeAttributes, assertAttributesApplied })
|
||||
@@ -0,0 +1,205 @@
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { link, lstat, mkdir, mkdtemp, open, readlink, rename, rm, stat } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { basename, dirname, extname, join } from 'node:path'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import { Readable } from 'node:stream'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
FileManagerError, FileRevisionConflictError, fileError, fileManagerInternals,
|
||||
isManagedPath, normalizeAbsolutePath,
|
||||
} from './index.mjs'
|
||||
import { createArchive } from './archives.mjs'
|
||||
|
||||
function parseRange(value, size) {
|
||||
if (value === undefined) return { start: 0, end: Math.max(0, size - 1), partial: false }
|
||||
const match = /^bytes=(\d*)-(\d*)$/.exec(value)
|
||||
if (match === null || (match[1] === '' && match[2] === '')) throw new FileManagerError('range is invalid', 416, 'RANGE_INVALID')
|
||||
let start
|
||||
let end
|
||||
if (match[1] === '') {
|
||||
const suffix = Number(match[2])
|
||||
if (!Number.isSafeInteger(suffix) || suffix < 1) throw new FileManagerError('range is invalid', 416, 'RANGE_INVALID')
|
||||
start = Math.max(0, size - suffix)
|
||||
end = size - 1
|
||||
} else {
|
||||
start = Number(match[1])
|
||||
end = match[2] === '' ? size - 1 : Number(match[2])
|
||||
}
|
||||
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= size || end < start) {
|
||||
throw new FileManagerError('range is not satisfiable', 416, 'RANGE_INVALID')
|
||||
}
|
||||
return { start, end: Math.min(end, size - 1), partial: true }
|
||||
}
|
||||
|
||||
function disposition(name) {
|
||||
const fallback = name.replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_') || 'download'
|
||||
return `attachment; filename="${fallback}"; filename*=UTF-8''${encodeURIComponent(name)}`
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
async function existingMetadata(path, { requireFile = false } = {}) {
|
||||
try {
|
||||
const value = await stat(path, { bigint: true })
|
||||
if (requireFile && !value.isFile()) throw new FileManagerError('upload target is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
return value
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function renamedPath(path, index) {
|
||||
const extension = extname(path)
|
||||
const stem = extension === '' ? path : path.slice(0, -extension.length)
|
||||
return `${stem} (${String(index)})${extension}`
|
||||
}
|
||||
|
||||
export class FileTransferManager {
|
||||
constructor({ isManaged = isManagedPath, stagingRoot = tmpdir() } = {}) {
|
||||
this.isManaged = isManaged
|
||||
this.stagingRoot = stagingRoot
|
||||
}
|
||||
|
||||
async openDownload(value, { revision, range, signal } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
let handle
|
||||
let cleanup
|
||||
try {
|
||||
const pathStats = await lstat(path, { bigint: true })
|
||||
if (pathStats.isDirectory()) {
|
||||
if (range !== undefined) throw new FileManagerError('directory downloads do not support ranges', 416, 'RANGE_INVALID')
|
||||
const actualRevision = fileManagerInternals.revisionFor(path, pathStats)
|
||||
if (revision !== undefined && revision !== '' && revision !== actualRevision) throw new FileRevisionConflictError('download revision changed')
|
||||
await mkdir(this.stagingRoot, { recursive: true })
|
||||
const staging = await mkdtemp(join(this.stagingRoot, '.dsh-directory-download-'))
|
||||
cleanup = () => rm(staging, { recursive: true, force: true })
|
||||
const name = basename(path) || 'root'
|
||||
const archive = join(staging, `${name}.zip`)
|
||||
await createArchive({ format: 'zip', sourceRoot: path === '/' ? path : dirname(path), output: archive, entries: [path === '/' ? '.' : name], signal })
|
||||
const download = await this.openDownload(archive, { signal })
|
||||
download.path = path
|
||||
download.revision = actualRevision
|
||||
download.headers['content-disposition'] = disposition(`${name}.zip`)
|
||||
download.cleanup = cleanup
|
||||
cleanup = undefined
|
||||
return download
|
||||
}
|
||||
handle = await open(path, 'r')
|
||||
const before = await handle.stat({ bigint: true })
|
||||
if (!before.isFile()) throw new FileManagerError('path is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const actualRevision = fileManagerInternals.revisionFor(path, before, await readlink(path).catch(() => ''))
|
||||
if (revision !== undefined && revision !== '' && revision !== actualRevision) throw new FileRevisionConflictError('download revision changed')
|
||||
const size = Number(before.size)
|
||||
if (!Number.isSafeInteger(size)) throw new FileManagerError('file is too large', 413, 'FILE_TOO_LARGE')
|
||||
if (size === 0 && range !== undefined) throw new FileManagerError('range is not satisfiable', 416, 'RANGE_INVALID')
|
||||
const selected = size === 0 ? { start: 0, end: -1, partial: false } : parseRange(range, size)
|
||||
const length = selected.end < selected.start ? 0 : selected.end - selected.start + 1
|
||||
const stream = length === 0 ? Readable.from([]) : createReadStream(path, {
|
||||
fd: handle.fd, autoClose: false, start: selected.start, end: selected.end,
|
||||
})
|
||||
return {
|
||||
path, handle, stream, revision: actualRevision,
|
||||
status: selected.partial ? 206 : 200,
|
||||
headers: {
|
||||
'accept-ranges': 'bytes',
|
||||
'cache-control': 'no-store',
|
||||
'content-disposition': disposition(basename(path)),
|
||||
'content-length': String(length),
|
||||
'content-type': 'application/octet-stream',
|
||||
etag: `"${actualRevision}"`,
|
||||
...(selected.partial ? { 'content-range': `bytes ${String(selected.start)}-${String(selected.end)}/${String(size)}` } : {}),
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {})
|
||||
await cleanup?.().catch(() => {})
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async sendDownload(response, download, { onProgress = () => {}, signal } = {}) {
|
||||
response.writeHead(download.status, download.headers)
|
||||
let sent = 0
|
||||
download.stream.on('data', chunk => {
|
||||
sent += chunk.byteLength
|
||||
onProgress(sent, Number(download.headers['content-length']))
|
||||
})
|
||||
try {
|
||||
await pipeline(download.stream, response, { signal })
|
||||
} finally {
|
||||
await download.handle.close().catch(() => {})
|
||||
await download.cleanup?.().catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
async upload(input, value, { conflict = 'reject', contentLength, onProgress = () => {}, signal } = {}) {
|
||||
const requestedPath = normalizeAbsolutePath(value)
|
||||
if (!['reject', 'overwrite', 'rename'].includes(conflict)) throw new FileManagerError('upload conflict mode is invalid')
|
||||
const parent = dirname(requestedPath)
|
||||
const parentStats = await stat(parent).catch(error => { throw fileError(error) })
|
||||
if (!parentStats.isDirectory()) throw new FileManagerError('upload parent is not a directory', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const staging = join(parent, `.dsh-upload-${randomUUID()}.tmp`)
|
||||
let handle
|
||||
let received = 0
|
||||
try {
|
||||
const original = await existingMetadata(requestedPath, { requireFile: conflict === 'overwrite' })
|
||||
if (original !== undefined && conflict === 'reject') throw new FileManagerError('upload target already exists', 409, 'FILE_EXISTS')
|
||||
handle = await open(staging, 'wx', 0o600)
|
||||
for await (const chunk of input) {
|
||||
signal?.throwIfAborted()
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
received += bytes.byteLength
|
||||
await handle.write(bytes)
|
||||
onProgress(received, contentLength)
|
||||
}
|
||||
if (contentLength !== undefined && received !== contentLength) throw new FileManagerError('upload size does not match Content-Length')
|
||||
await handle.sync()
|
||||
if (original === undefined) await handle.chmod(0o644)
|
||||
else {
|
||||
await handle.chmod(Number(original.mode & 0o7777n))
|
||||
const current = await handle.stat({ bigint: true })
|
||||
if (current.uid !== original.uid || current.gid !== original.gid) await handle.chown(Number(original.uid), Number(original.gid))
|
||||
}
|
||||
await handle.close()
|
||||
handle = undefined
|
||||
let path = requestedPath
|
||||
if (conflict === 'overwrite') await rename(staging, path)
|
||||
else if (conflict === 'reject') {
|
||||
await link(staging, path)
|
||||
await rm(staging)
|
||||
} else {
|
||||
let index = 0
|
||||
for (;;) {
|
||||
path = index === 0 ? requestedPath : renamedPath(requestedPath, index)
|
||||
try {
|
||||
await link(staging, path)
|
||||
await rm(staging)
|
||||
break
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST') throw error
|
||||
index += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
await syncDirectory(parent)
|
||||
const result = await stat(path, { bigint: true })
|
||||
return {
|
||||
path,
|
||||
size: received,
|
||||
revision: fileManagerInternals.revisionFor(path, result),
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {})
|
||||
await rm(staging, { force: true }).catch(() => {})
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const fileTransferInternals = Object.freeze({ parseRange, disposition, renamedPath })
|
||||
@@ -0,0 +1,473 @@
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { appendFile, chmod, chown, mkdir, readdir, readFile, rename, rm, stat } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
|
||||
const SOURCE_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/
|
||||
const MAX_ENTRY_BYTES = 64 * 1024
|
||||
const CONSOLE_MARKER = 'dsh-platform-log-v1'
|
||||
const LOG_LEVELS = new Set(['debug', 'info', 'warning', 'error'])
|
||||
const MAX_DIAGNOSTIC_STRING = 16 * 1024
|
||||
const MAX_ERROR_DEPTH = 3
|
||||
const MAX_AGGREGATE_ERRORS = 8
|
||||
const SUPPRESSION_KEY_PATTERN = /^[a-z0-9][a-z0-9._:-]{0,255}$/
|
||||
|
||||
function boundedString(value) {
|
||||
const text = String(value)
|
||||
return text.length <= MAX_DIAGNOSTIC_STRING ? text : `${text.slice(0, MAX_DIAGNOSTIC_STRING)}...[truncated]`
|
||||
}
|
||||
|
||||
export function errorDetails(error, depth = 0) {
|
||||
if (error === undefined || error === null) return {}
|
||||
if (!(error instanceof Error)) return { error: boundedString(error), errorType: typeof error }
|
||||
const details = {
|
||||
error: boundedString(error.message || error.name),
|
||||
errorName: boundedString(error.name),
|
||||
}
|
||||
if (typeof error.code === 'string' || typeof error.code === 'number') details.errorCode = error.code
|
||||
if (typeof error.stack === 'string') details.errorStack = boundedString(error.stack)
|
||||
if (depth >= MAX_ERROR_DEPTH) return details
|
||||
if (error.cause !== undefined) details.errorCause = errorDetails(error.cause, depth + 1)
|
||||
if (error instanceof AggregateError) {
|
||||
details.errors = [...error.errors].slice(0, MAX_AGGREGATE_ERRORS).map(value => errorDetails(value, depth + 1))
|
||||
if (error.errors.length > MAX_AGGREGATE_ERRORS) details.errorsTruncated = error.errors.length - MAX_AGGREGATE_ERRORS
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
function validateSource(source) {
|
||||
if (typeof source !== 'string' || !SOURCE_PATTERN.test(source)) throw new Error('log source is invalid')
|
||||
return source
|
||||
}
|
||||
|
||||
function validateFilter(value, label) {
|
||||
if (value === undefined) return undefined
|
||||
if (typeof value !== 'string' || value.length === 0 || value.length > 128 || /[\0\r\n]/u.test(value)) {
|
||||
throw new Error(`${label} filter is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function matches(entry, filters) {
|
||||
return (filters.taskId === undefined || entry.taskId === filters.taskId)
|
||||
&& (filters.operation === undefined || entry.operation === filters.operation)
|
||||
&& (filters.phase === undefined || entry.phase === filters.phase)
|
||||
}
|
||||
|
||||
function parseLine(line) {
|
||||
try { return JSON.parse(line) } catch { return undefined }
|
||||
}
|
||||
|
||||
function parseForwardedLine(line) {
|
||||
const entry = parseLine(line)
|
||||
if (
|
||||
entry?.platformLog !== CONSOLE_MARKER
|
||||
|| typeof entry.timestamp !== 'string'
|
||||
|| typeof entry.message !== 'string'
|
||||
|| !['stdout', 'stderr', 'audit', 'platform'].includes(entry.stream)
|
||||
) return undefined
|
||||
try { validateSource(entry.source) } catch { return undefined }
|
||||
return entry
|
||||
}
|
||||
|
||||
export function capturedLogLevel(stream, message) {
|
||||
if (stream !== 'stderr') return 'info'
|
||||
const text = String(message).trim()
|
||||
const lines = text.split('\n')
|
||||
if (lines.some(line => /^(?:\s*at\s+|.*\b(?:error|fatal|failed|failure|exception|panic|unhandled)\b)|错误|失败|异常|致命/iu.test(line))) {
|
||||
return 'error'
|
||||
}
|
||||
if (lines.some(line => /\b(?:warn|warning|deprecated|deprecation)\b|警告|已弃用/iu.test(line))) return 'warning'
|
||||
return 'info'
|
||||
}
|
||||
|
||||
const NODE_ERROR_LINE = /^\s*(?:AggregateError|Error|EvalError|RangeError|ReferenceError|SyntaxError|TypeError|URIError):/u
|
||||
const NODE_LOCATION_LINE = /^(?:file:\/\/|\/).*:\d+(?::\d+)?$/u
|
||||
|
||||
function isNodeErrorContinuation(line) {
|
||||
return line === ''
|
||||
|| /^\s/u.test(line)
|
||||
|| NODE_ERROR_LINE.test(line)
|
||||
|| /^Node\.js v\d/u.test(line)
|
||||
|| /^\s*(?:\[cause\]|\[errors\]|Caused by:)/u.test(line)
|
||||
|| /^\s*[{}\[\]},]+$/u.test(line)
|
||||
}
|
||||
|
||||
function capturedMessages(source, stream, lines) {
|
||||
if (source !== 'dsh-runtime' || stream !== 'stderr') return lines
|
||||
const messages = []
|
||||
let cursor = 0
|
||||
while (cursor < lines.length) {
|
||||
const relativeError = lines.slice(cursor).findIndex(line => NODE_ERROR_LINE.test(line))
|
||||
if (relativeError < 0) {
|
||||
messages.push(...lines.slice(cursor))
|
||||
break
|
||||
}
|
||||
const errorIndex = cursor + relativeError
|
||||
let start = errorIndex
|
||||
for (let index = errorIndex - 1; index >= cursor && index >= errorIndex - 6; index -= 1) {
|
||||
if (NODE_LOCATION_LINE.test(lines[index])) {
|
||||
start = index
|
||||
break
|
||||
}
|
||||
}
|
||||
messages.push(...lines.slice(cursor, start))
|
||||
let end = errorIndex + 1
|
||||
while (end < lines.length && isNodeErrorContinuation(lines[end])) {
|
||||
const terminal = /^Node\.js v\d/u.test(lines[end])
|
||||
end += 1
|
||||
if (terminal) break
|
||||
}
|
||||
messages.push(boundedString(lines.slice(start, end).join('\n')))
|
||||
cursor = end
|
||||
}
|
||||
return messages
|
||||
}
|
||||
|
||||
export class JsonlLogManager extends EventEmitter {
|
||||
constructor({
|
||||
root,
|
||||
maxBytes = 100 * 1024 * 1024,
|
||||
retentionDays = 14,
|
||||
rotateBytes = 10 * 1024 * 1024,
|
||||
now = () => new Date(),
|
||||
output,
|
||||
fileMode = 0o660,
|
||||
fileUid,
|
||||
fileGid,
|
||||
}) {
|
||||
super()
|
||||
if (!Number.isSafeInteger(maxBytes) || maxBytes < MAX_ENTRY_BYTES) throw new Error('log maxBytes is invalid')
|
||||
if (!Number.isSafeInteger(retentionDays) || retentionDays < 1) throw new Error('log retentionDays is invalid')
|
||||
if (!Number.isSafeInteger(rotateBytes) || rotateBytes < 1_024) throw new Error('log rotateBytes is invalid')
|
||||
if (!Number.isSafeInteger(fileMode) || fileMode < 0 || fileMode > 0o777) throw new Error('log fileMode is invalid')
|
||||
this.root = root
|
||||
this.maxBytes = maxBytes
|
||||
this.retentionMs = retentionDays * 86_400_000
|
||||
this.rotateBytes = rotateBytes
|
||||
this.now = now
|
||||
this.output = output
|
||||
this.fileMode = fileMode
|
||||
this.fileUid = fileUid
|
||||
this.fileGid = fileGid
|
||||
this.queue = Promise.resolve()
|
||||
this.suppressedDiagnostics = new Map()
|
||||
}
|
||||
|
||||
prepare() {
|
||||
return this.serialized(async () => {
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const names = (await readdir(this.root)).filter(name => name.endsWith('.jsonl'))
|
||||
for (const name of names) {
|
||||
const path = join(this.root, name)
|
||||
if (this.fileGid !== undefined) await chown(path, -1, this.fileGid)
|
||||
await chmod(path, this.fileMode)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
writeOutput(stream, line) {
|
||||
const destination = stream === 'stderr' ? this.output?.stderr : this.output?.stdout
|
||||
destination?.write(`${line}\n`)
|
||||
}
|
||||
|
||||
mirror(entry) {
|
||||
if (this.output === undefined) return
|
||||
this.writeOutput(entry.stream, JSON.stringify({ ...entry, platformLog: CONSOLE_MARKER }))
|
||||
}
|
||||
|
||||
serialized(operation) {
|
||||
const result = this.queue.then(operation, operation)
|
||||
this.queue = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
currentPath(source) {
|
||||
return join(this.root, `${validateSource(source)}.jsonl`)
|
||||
}
|
||||
|
||||
append(source, stream, message, fields = {}) {
|
||||
return this.serialized(async () => {
|
||||
validateSource(source)
|
||||
if (!['stdout', 'stderr', 'audit', 'platform'].includes(stream)) throw new Error('log stream is invalid')
|
||||
if (typeof message !== 'string') throw new Error('log message must be a string')
|
||||
const level = fields.level ?? (stream === 'stderr' ? 'error' : 'info')
|
||||
if (!LOG_LEVELS.has(level)) throw new Error('log level is invalid')
|
||||
const entry = { ...fields, timestamp: this.now().toISOString(), source, stream, level, message }
|
||||
const line = Buffer.from(`${JSON.stringify(entry)}\n`)
|
||||
if (line.byteLength > MAX_ENTRY_BYTES) throw new Error('log entry exceeds 64 KiB')
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const path = this.currentPath(source)
|
||||
let exists = true
|
||||
const size = await stat(path).then(value => value.size, error => {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
exists = false
|
||||
return 0
|
||||
})
|
||||
if (size > 0 && size + line.byteLength > this.rotateBytes) {
|
||||
await rename(path, join(this.root, `${source}.${Date.now()}.${randomUUID()}.jsonl`))
|
||||
exists = false
|
||||
}
|
||||
await appendFile(path, line, { mode: this.fileMode })
|
||||
if (!exists) {
|
||||
if (this.fileUid !== undefined || this.fileGid !== undefined) await chown(path, this.fileUid ?? -1, this.fileGid ?? -1)
|
||||
await chmod(path, this.fileMode)
|
||||
}
|
||||
this.emit('entry', Object.freeze(entry))
|
||||
this.mirror(entry)
|
||||
await this.pruneUnlocked()
|
||||
return Object.freeze(entry)
|
||||
})
|
||||
}
|
||||
|
||||
audit(action, fields = {}) {
|
||||
return this.append('audit', 'audit', action, fields)
|
||||
}
|
||||
|
||||
async diagnostic(source, message, { error, level = error === undefined || error === null ? 'info' : 'error', stream = 'platform', ...fields } = {}) {
|
||||
const details = { ...fields, ...errorDetails(error), level }
|
||||
try {
|
||||
return await this.append(source, stream, message, details)
|
||||
} catch (loggingError) {
|
||||
try {
|
||||
this.writeOutput('stderr', JSON.stringify({
|
||||
timestamp: this.now().toISOString(),
|
||||
source: 'log-manager',
|
||||
stream: 'stderr',
|
||||
level: 'error',
|
||||
message: 'diagnostic.write.failed',
|
||||
diagnostic: { source, stream, message, ...details },
|
||||
loggingError: errorDetails(loggingError),
|
||||
platformLog: CONSOLE_MARKER,
|
||||
}))
|
||||
} catch {}
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
diagnosticRateLimited(key, source, message, fields = {}, { intervalMs = 30_000 } = {}) {
|
||||
if (typeof key !== 'string' || !SUPPRESSION_KEY_PATTERN.test(key)) {
|
||||
throw new Error('diagnostic suppression key is invalid')
|
||||
}
|
||||
if (!Number.isSafeInteger(intervalMs) || intervalMs < 1_000 || intervalMs > 86_400_000) {
|
||||
throw new Error('diagnostic suppression interval is invalid')
|
||||
}
|
||||
const timestamp = this.now().getTime()
|
||||
const previous = this.suppressedDiagnostics.get(key)
|
||||
if (previous !== undefined && timestamp - previous.lastReportedAt < intervalMs) {
|
||||
previous.suppressedCount += 1
|
||||
return Promise.resolve(undefined)
|
||||
}
|
||||
const suppressedCount = previous?.suppressedCount ?? 0
|
||||
this.suppressedDiagnostics.set(key, { lastReportedAt: timestamp, suppressedCount: 0 })
|
||||
return this.diagnostic(source, message, {
|
||||
...fields,
|
||||
...(suppressedCount === 0 ? {} : { suppressedCount }),
|
||||
})
|
||||
}
|
||||
|
||||
async files() {
|
||||
try {
|
||||
const names = (await readdir(this.root)).filter(name => name.endsWith('.jsonl'))
|
||||
return Promise.all(names.map(async name => ({ name, path: join(this.root, name), details: await stat(join(this.root, name)) })))
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return []
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async pruneUnlocked() {
|
||||
const cutoff = this.now().getTime() - this.retentionMs
|
||||
let files = await this.files()
|
||||
for (const file of files) {
|
||||
if (file.details.mtimeMs < cutoff) await rm(file.path, { force: true })
|
||||
}
|
||||
files = (await this.files()).sort((left, right) => left.details.mtimeMs - right.details.mtimeMs)
|
||||
let total = files.reduce((sum, file) => sum + file.details.size, 0)
|
||||
for (const file of files) {
|
||||
if (total <= this.maxBytes) break
|
||||
await rm(file.path, { force: true })
|
||||
total -= file.details.size
|
||||
}
|
||||
}
|
||||
|
||||
prune() {
|
||||
return this.serialized(() => this.pruneUnlocked())
|
||||
}
|
||||
|
||||
async query({ sources, since, taskId, operation, phase, limit = 200 } = {}) {
|
||||
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 5_000) throw new Error('log query limit is invalid')
|
||||
const selected = sources === undefined ? undefined : new Set(sources.map(validateSource))
|
||||
const threshold = since === undefined ? undefined : new Date(since).toISOString()
|
||||
const filters = {
|
||||
taskId: validateFilter(taskId, 'taskId'),
|
||||
operation: validateFilter(operation, 'operation'),
|
||||
phase: validateFilter(phase, 'phase'),
|
||||
}
|
||||
const entries = []
|
||||
for (const file of await this.files()) {
|
||||
const lines = (await readFile(file.path, 'utf8')).split('\n')
|
||||
for (const line of lines) {
|
||||
if (line === '') continue
|
||||
const entry = parseLine(line)
|
||||
if (
|
||||
entry !== undefined
|
||||
&& (selected === undefined || selected.has(entry.source))
|
||||
&& (threshold === undefined || entry.timestamp >= threshold)
|
||||
&& matches(entry, filters)
|
||||
) entries.push(entry)
|
||||
}
|
||||
}
|
||||
entries.sort((left, right) => left.timestamp.localeCompare(right.timestamp))
|
||||
return Object.freeze(entries.slice(-limit).map(Object.freeze))
|
||||
}
|
||||
|
||||
async sources() {
|
||||
const result = new Set()
|
||||
for (const file of await this.files()) {
|
||||
for (const line of (await readFile(file.path, 'utf8')).split('\n')) {
|
||||
const entry = parseLine(line)
|
||||
if (entry?.source !== undefined) {
|
||||
try { result.add(validateSource(entry.source)) } catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...result].sort()
|
||||
}
|
||||
|
||||
async follow({ sources, since, taskId, operation, phase } = {}, listener, { intervalMs = 250, onError = () => {} } = {}) {
|
||||
if (typeof listener !== 'function') throw new Error('log follow listener is required')
|
||||
if (!Number.isSafeInteger(intervalMs) || intervalMs < 50) throw new Error('log follow interval is invalid')
|
||||
const selected = sources === undefined ? undefined : new Set(sources.map(validateSource))
|
||||
const threshold = since === undefined ? undefined : new Date(since).toISOString()
|
||||
const filters = {
|
||||
taskId: validateFilter(taskId, 'taskId'),
|
||||
operation: validateFilter(operation, 'operation'),
|
||||
phase: validateFilter(phase, 'phase'),
|
||||
}
|
||||
await mkdir(this.root, { recursive: true })
|
||||
let states = new Map()
|
||||
for (const file of await this.files()) {
|
||||
states.set(file.name, { identity: `${String(file.details.dev)}:${String(file.details.ino)}`, offset: file.details.size, pending: Buffer.alloc(0) })
|
||||
}
|
||||
let closed = false
|
||||
let scanning = false
|
||||
let scanAgain = false
|
||||
const scan = async () => {
|
||||
if (closed) return
|
||||
if (scanning) {
|
||||
scanAgain = true
|
||||
return
|
||||
}
|
||||
scanning = true
|
||||
try {
|
||||
const files = await this.files()
|
||||
const byIdentity = new Map([...states.values()].map(state => [state.identity, state]))
|
||||
const nextStates = new Map()
|
||||
const entries = []
|
||||
for (const file of files) {
|
||||
const identity = `${String(file.details.dev)}:${String(file.details.ino)}`
|
||||
let state = states.get(file.name)
|
||||
if (state?.identity !== identity) state = byIdentity.get(identity)
|
||||
if (state === undefined || file.details.size < state.offset) state = { identity, offset: 0, pending: Buffer.alloc(0) }
|
||||
if (file.details.size > state.offset) {
|
||||
const chunks = []
|
||||
const end = file.details.size - 1
|
||||
for await (const chunk of createReadStream(file.path, { start: state.offset, end })) chunks.push(chunk)
|
||||
const content = Buffer.concat([state.pending, ...chunks])
|
||||
let lineStart = 0
|
||||
for (let index = 0; index < content.length; index += 1) {
|
||||
if (content[index] !== 0x0a) continue
|
||||
const line = content.subarray(lineStart, index).toString('utf8')
|
||||
lineStart = index + 1
|
||||
if (line === '') continue
|
||||
const entry = parseLine(line)
|
||||
if (entry !== undefined
|
||||
&& (selected === undefined || selected.has(entry.source))
|
||||
&& (threshold === undefined || entry.timestamp >= threshold)
|
||||
&& matches(entry, filters)) entries.push(entry)
|
||||
}
|
||||
state = { ...state, identity, offset: file.details.size, pending: content.subarray(lineStart) }
|
||||
}
|
||||
nextStates.set(file.name, state)
|
||||
}
|
||||
states = nextStates
|
||||
entries.sort((left, right) => left.timestamp.localeCompare(right.timestamp))
|
||||
for (const entry of entries) listener(Object.freeze(entry))
|
||||
} catch (error) {
|
||||
onError(error)
|
||||
} finally {
|
||||
scanning = false
|
||||
if (scanAgain) {
|
||||
scanAgain = false
|
||||
void scan()
|
||||
}
|
||||
}
|
||||
}
|
||||
const timer = setInterval(() => { void scan() }, intervalMs)
|
||||
timer.unref()
|
||||
return Object.freeze({
|
||||
close: () => {
|
||||
closed = true
|
||||
clearInterval(timer)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
capture(child, source, declaration = { stdout: true, stderr: true }, { acceptForwarded = false } = {}) {
|
||||
for (const streamName of ['stdout', 'stderr']) {
|
||||
if (!declaration[streamName]) continue
|
||||
const stream = child[streamName]
|
||||
if (stream === undefined || stream === null) continue
|
||||
let pending = ''
|
||||
let completeLines = []
|
||||
let flushScheduled = false
|
||||
const appendLines = lines => {
|
||||
if (lines.length === 0) return
|
||||
const messages = capturedMessages(source, streamName, lines)
|
||||
for (const message of messages) {
|
||||
void this.append(source, streamName, message, { level: capturedLogLevel(streamName, message) })
|
||||
.catch(error => this.emit('error', error))
|
||||
}
|
||||
}
|
||||
const flush = () => {
|
||||
flushScheduled = false
|
||||
const lines = completeLines
|
||||
completeLines = []
|
||||
let unforwarded = []
|
||||
const flushUnforwarded = () => {
|
||||
appendLines(unforwarded)
|
||||
unforwarded = []
|
||||
}
|
||||
for (const line of lines) {
|
||||
const forwarded = acceptForwarded && this.output !== undefined ? parseForwardedLine(line) : undefined
|
||||
if (forwarded === undefined) unforwarded.push(line)
|
||||
else {
|
||||
flushUnforwarded()
|
||||
this.writeOutput(forwarded.stream, line)
|
||||
}
|
||||
}
|
||||
flushUnforwarded()
|
||||
}
|
||||
const scheduleFlush = () => {
|
||||
if (flushScheduled) return
|
||||
flushScheduled = true
|
||||
setImmediate(flush)
|
||||
}
|
||||
stream.setEncoding?.('utf8')
|
||||
stream.on('data', chunk => {
|
||||
pending += String(chunk)
|
||||
const lines = pending.split('\n')
|
||||
pending = lines.pop()
|
||||
completeLines.push(...lines)
|
||||
scheduleFlush()
|
||||
})
|
||||
stream.once('end', () => {
|
||||
if (pending !== '') completeLines.push(pending)
|
||||
pending = ''
|
||||
flush()
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { cp, lstat, mkdir, readFile, readdir, readlink, rename, rm, symlink } from 'node:fs/promises'
|
||||
import { basename, dirname, join, resolve } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { TrustError } from '../../../platform/lib/validation.mjs'
|
||||
import { artifactForReference, parseEnvironmentManifest } from '../../../platform/lib/contracts.mjs'
|
||||
|
||||
async function patchModule(path) {
|
||||
return import(`${pathToFileURL(resolve(path)).href}?runtime=${randomUUID()}`)
|
||||
}
|
||||
|
||||
export async function verifyNpmIntegrity(path, integrity) {
|
||||
if (typeof integrity !== 'string' || !integrity.startsWith('sha512-')) {
|
||||
throw new TrustError('npm integrity must use SHA-512')
|
||||
}
|
||||
const expected = integrity.slice('sha512-'.length)
|
||||
const bytes = await readFile(path)
|
||||
const actual = createHash('sha512').update(bytes).digest('base64')
|
||||
if (actual !== expected) throw new TrustError('npm tarball integrity mismatch', 'TRUST_ARTIFACT_MISMATCH')
|
||||
return bytes.byteLength
|
||||
}
|
||||
|
||||
export async function applyPatchSet(dshRoot, patchPaths) {
|
||||
for (const path of patchPaths) {
|
||||
const module = await patchModule(path)
|
||||
if (typeof module.applyPatch !== 'function') {
|
||||
throw new Error(`Patch ${path} does not export applyPatch(dshRoot)`)
|
||||
}
|
||||
await module.applyPatch(dshRoot)
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyPatchSet(dshRoot, patchPaths) {
|
||||
for (const path of patchPaths) {
|
||||
const module = await patchModule(path)
|
||||
if (typeof module.verifyPatch !== 'function') {
|
||||
throw new Error(`Patch ${path} does not export verifyPatch(dshRoot)`)
|
||||
}
|
||||
await module.verifyPatch(resolve(dshRoot))
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyRuntimePatches({ runtimeRoot, environmentRoot }) {
|
||||
const patchSet = await loadEnvironmentPatchSet(environmentRoot)
|
||||
await verifyPatchSet(join(resolve(runtimeRoot), 'package'), patchSet.paths)
|
||||
return patchSet.ids
|
||||
}
|
||||
|
||||
export async function loadEnvironmentPatchSet(environmentRoot) {
|
||||
const environment = resolve(environmentRoot)
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(environment, 'environment.manifest.json')))
|
||||
const paths = []
|
||||
for (const reference of manifest.patches) {
|
||||
const descriptor = artifactForReference(manifest, reference)
|
||||
const path = join(environment, 'artifacts', descriptor.id)
|
||||
const bytes = await readFile(path)
|
||||
if (bytes.byteLength !== descriptor.size
|
||||
|| createHash('sha256').update(bytes).digest('hex') !== descriptor.sha256) {
|
||||
throw new TrustError(`Patch Artifact ${descriptor.id} differs from the Environment Manifest`)
|
||||
}
|
||||
paths.push(path)
|
||||
}
|
||||
return Object.freeze({
|
||||
ids: Object.freeze(manifest.patches.map(reference => reference.id)),
|
||||
paths: Object.freeze(paths),
|
||||
})
|
||||
}
|
||||
|
||||
async function measureTree(path) {
|
||||
const details = await lstat(path)
|
||||
if (details.isSymbolicLink()) return { bytes: 0, items: 1 }
|
||||
if (!details.isDirectory()) return { bytes: details.size, items: 1 }
|
||||
let bytes = 0
|
||||
let items = 0
|
||||
for (const entry of await readdir(path)) {
|
||||
const measured = await measureTree(join(path, entry))
|
||||
bytes += measured.bytes
|
||||
items += measured.items
|
||||
}
|
||||
return { bytes, items }
|
||||
}
|
||||
|
||||
async function copyTree(source, destination, onProgress = async () => {}) {
|
||||
const details = await lstat(source)
|
||||
if (details.isSymbolicLink()) {
|
||||
await symlink(await readlink(source), destination)
|
||||
await onProgress({ processedBytes: 0, processedItems: 1 })
|
||||
return
|
||||
}
|
||||
if (!details.isDirectory()) {
|
||||
await cp(source, destination, { preserveTimestamps: true, verbatimSymlinks: true })
|
||||
await onProgress({ processedBytes: details.size, processedItems: 1 })
|
||||
return
|
||||
}
|
||||
await mkdir(destination, { recursive: true })
|
||||
for (const entry of await readdir(source)) {
|
||||
await copyTree(join(source, entry), join(destination, entry), onProgress)
|
||||
}
|
||||
}
|
||||
|
||||
export async function buildRuntime({ pristineRoot, versionsRoot, runtimeId, patchPaths, onProgress = async () => {} }) {
|
||||
if (typeof runtimeId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(runtimeId)) {
|
||||
throw new Error('Runtime ID is invalid')
|
||||
}
|
||||
const versions = resolve(versionsRoot)
|
||||
const destination = join(versions, runtimeId)
|
||||
try {
|
||||
await lstat(destination)
|
||||
throw new Error(`Runtime ${runtimeId} already exists`)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
await mkdir(versions, { recursive: true })
|
||||
const staging = join(versions, `.${runtimeId}.${randomUUID()}.tmp`)
|
||||
try {
|
||||
const packageRoot = join(staging, 'package')
|
||||
await mkdir(staging)
|
||||
const total = await measureTree(pristineRoot)
|
||||
let processedBytes = 0
|
||||
let processedItems = 0
|
||||
await copyTree(pristineRoot, packageRoot, async update => {
|
||||
processedBytes += update.processedBytes
|
||||
processedItems += update.processedItems
|
||||
await onProgress({ processedBytes, totalBytes: total.bytes, processedItems, totalItems: total.items })
|
||||
})
|
||||
await applyPatchSet(packageRoot, patchPaths)
|
||||
const binTarget = join(packageRoot, 'lib', 'bin.js')
|
||||
const details = await lstat(binTarget)
|
||||
if (!details.isFile()) throw new Error('DSH Runtime has no lib/bin.js entrypoint')
|
||||
await mkdir(join(staging, 'bin'))
|
||||
await symlink('../package/lib/bin.js', join(staging, 'bin', 'dsh'))
|
||||
await rename(staging, destination)
|
||||
return destination
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function optionalLink(path) {
|
||||
try {
|
||||
return basename(await import('node:fs/promises').then(({ readlink }) => readlink(path)))
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function replaceLink(root, name, runtimeId) {
|
||||
const temporary = join(root, `.${name}.${randomUUID()}.tmp`)
|
||||
await symlink(join('versions', runtimeId), temporary)
|
||||
await rename(temporary, join(root, name))
|
||||
}
|
||||
|
||||
export function cloneTree(source, destination) {
|
||||
return new Promise((resolveClone, reject) => {
|
||||
const child = spawn('cp', ['-a', '--reflink=auto', '--', resolve(source), resolve(destination)], {
|
||||
stdio: ['ignore', 'ignore', 'pipe'],
|
||||
})
|
||||
const stderr = []
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolveClone(destination)
|
||||
: reject(new Error(`Platform tree copy failed: ${Buffer.concat(stderr).toString('utf8').trim()}`)))
|
||||
})
|
||||
}
|
||||
|
||||
export class RuntimeSlots {
|
||||
constructor(root) {
|
||||
this.root = resolve(root)
|
||||
}
|
||||
|
||||
async state() {
|
||||
return Object.freeze({
|
||||
current: await optionalLink(join(this.root, 'current')),
|
||||
previous: await optionalLink(join(this.root, 'previous')),
|
||||
})
|
||||
}
|
||||
|
||||
async promote(runtimeId) {
|
||||
await lstat(join(this.root, 'versions', runtimeId))
|
||||
const state = await this.state()
|
||||
if (state.current !== undefined && state.current !== runtimeId) await replaceLink(this.root, 'previous', state.current)
|
||||
await replaceLink(this.root, 'current', runtimeId)
|
||||
return this.state()
|
||||
}
|
||||
|
||||
async rollback() {
|
||||
const state = await this.state()
|
||||
if (state.previous === undefined) throw new Error('no previous Runtime exists')
|
||||
await replaceLink(this.root, 'current', state.previous)
|
||||
if (state.current !== undefined) await replaceLink(this.root, 'previous', state.current)
|
||||
return this.state()
|
||||
}
|
||||
|
||||
async prune() {
|
||||
const state = await this.state()
|
||||
const retained = new Set([state.current, state.previous].filter(Boolean))
|
||||
let versions
|
||||
try { versions = await readdir(join(this.root, 'versions')) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return Object.freeze([])
|
||||
throw error
|
||||
}
|
||||
const removed = []
|
||||
for (const version of versions) {
|
||||
if (!version.startsWith('.') && !retained.has(version)) {
|
||||
await rm(join(this.root, 'versions', version), { recursive: true, force: true })
|
||||
removed.push(version)
|
||||
}
|
||||
}
|
||||
return Object.freeze(removed.sort())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,441 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { lstat, mkdir, readdir, readFile, readlink, rename, rm, symlink, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { canonicalJson } from '../../../platform/lib/canonical-json.mjs'
|
||||
import { artifactForReference, parseEnvironmentManifest } from '../../../platform/lib/contracts.mjs'
|
||||
import { hashTree } from '../../../platform/lib/tree-hash.mjs'
|
||||
|
||||
const PLUGIN_ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/
|
||||
const SELECTION_SCHEMA = 1
|
||||
export const PROTECTED_SYSTEM_PLUGIN_IDS = Object.freeze(['platform-management'])
|
||||
|
||||
function assertPluginId(pluginId) {
|
||||
if (typeof pluginId !== 'string' || !PLUGIN_ID_PATTERN.test(pluginId)) {
|
||||
throw new Error('System Plugin ID is invalid')
|
||||
}
|
||||
return pluginId
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => {
|
||||
if (code === 0) resolveRun(Buffer.concat(stdout).toString('utf8'))
|
||||
else reject(new Error(`${command} failed: ${Buffer.concat(stderr).toString('utf8').trim()}`))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async function extractPackage(archive, destination) {
|
||||
const listing = (await run('tar', ['-tzf', archive])).split('\n').filter(Boolean)
|
||||
if (listing.length === 0) throw new Error('System Plugin archive is empty')
|
||||
for (const name of listing) {
|
||||
if (!name.startsWith('package/') || name.startsWith('/') || name.split('/').includes('..')) {
|
||||
throw new Error(`System Plugin archive path is unsafe: ${name}`)
|
||||
}
|
||||
}
|
||||
await mkdir(destination, { recursive: true })
|
||||
await run('tar', ['-xzf', archive, '--strip-components=1', '--no-same-owner', '--no-same-permissions', '-C', destination])
|
||||
}
|
||||
|
||||
function validatePackage(value, pluginId) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`System Plugin ${pluginId} package.json must be an object`)
|
||||
}
|
||||
const expectedName = `@dsh-docker/${pluginId}`
|
||||
if (value.name !== expectedName) throw new Error(`System Plugin ${pluginId} package name must be ${expectedName}`)
|
||||
if (typeof value.main !== 'string' || value.main.length === 0
|
||||
|| value.main.startsWith('/') || value.main.includes('\\') || value.main.split('/').includes('..')) {
|
||||
throw new Error(`System Plugin ${pluginId} package main must be a safe relative path`)
|
||||
}
|
||||
return expectedName
|
||||
}
|
||||
|
||||
function validatePatch(value, pluginId, packageName) {
|
||||
if (!Array.isArray(value)) throw new Error(`System Plugin ${pluginId} cordis.patch.json must be an array`)
|
||||
return value.map((entry, index) => {
|
||||
if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) {
|
||||
throw new Error(`System Plugin ${pluginId} patch ${String(index)} must be an object`)
|
||||
}
|
||||
if (Object.keys(entry).length !== 1 || !Array.isArray(entry.insert) || entry.insert.length === 0) {
|
||||
throw new Error(`System Plugin ${pluginId} patches must contain only a non-empty insert list`)
|
||||
}
|
||||
for (const row of entry.insert) {
|
||||
if (row === null || typeof row !== 'object' || Array.isArray(row)) {
|
||||
throw new Error(`System Plugin ${pluginId} inserted rows must be objects`)
|
||||
}
|
||||
if (typeof row.id !== 'string' || !row.id.startsWith(`dsh-docker.${pluginId}.`)) {
|
||||
throw new Error(`System Plugin ${pluginId} patch IDs must use its dsh-docker namespace`)
|
||||
}
|
||||
if (typeof row.name !== 'string' || row.name.length === 0) {
|
||||
throw new Error(`System Plugin ${pluginId} inserted rows must name a package`)
|
||||
}
|
||||
if (row.name !== packageName) {
|
||||
throw new Error(`System Plugin ${pluginId} inserted rows must load its own package`)
|
||||
}
|
||||
}
|
||||
return {
|
||||
insert: entry.insert.map(row => ({ ...row })),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function replaceLink(root, name, version) {
|
||||
const temporary = join(root, `.${name}.${randomUUID()}.tmp`)
|
||||
await symlink(join('versions', version), temporary)
|
||||
await rename(temporary, join(root, name))
|
||||
}
|
||||
|
||||
export async function reconcileSystemPlugins({ root, environmentVersion, plugins, artifactPath }) {
|
||||
const managedRoot = resolve(root)
|
||||
const versions = join(managedRoot, 'versions')
|
||||
const destination = join(versions, environmentVersion)
|
||||
const staging = join(versions, `.${environmentVersion}.${randomUUID()}.tmp`)
|
||||
await mkdir(staging, { recursive: true })
|
||||
const patches = []
|
||||
try {
|
||||
for (const plugin of plugins) {
|
||||
const packageRoot = join(staging, 'packages', plugin.id)
|
||||
await extractPackage(artifactPath(plugin), packageRoot)
|
||||
const metadata = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8'))
|
||||
const packageName = validatePackage(metadata, plugin.id)
|
||||
const patch = JSON.parse(await readFile(join(packageRoot, 'cordis.patch.json'), 'utf8'))
|
||||
patches.push(...validatePatch(patch, plugin.id, packageName))
|
||||
}
|
||||
await writeFile(join(staging, 'cordis.patch.yml'), canonicalJson(patches), { flag: 'wx' })
|
||||
await rename(staging, destination)
|
||||
const current = await import('node:fs/promises').then(({ readlink }) => readlink(join(managedRoot, 'current')).catch(error => error.code === 'ENOENT' ? undefined : Promise.reject(error)))
|
||||
if (current !== undefined) await replaceLink(managedRoot, 'previous', current.split('/').at(-1))
|
||||
await replaceLink(managedRoot, 'current', environmentVersion)
|
||||
return destination
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function readOverlay(root) {
|
||||
const patch = JSON.parse(await readFile(join(root, 'cordis.patch.yml'), 'utf8'))
|
||||
if (!Array.isArray(patch)) throw new Error('System Plugin overlay patch must be an array')
|
||||
return patch
|
||||
}
|
||||
|
||||
function defaultSelection(pluginIds, protectedIds) {
|
||||
return Object.fromEntries(pluginIds.map(id => [id, {
|
||||
installed: true,
|
||||
enabled: true,
|
||||
protected: protectedIds.has(id),
|
||||
}]))
|
||||
}
|
||||
|
||||
function validateSelectionDocument(value, pluginIds, protectedIds) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value) || value.schema !== SELECTION_SCHEMA
|
||||
|| value.plugins === null || typeof value.plugins !== 'object' || Array.isArray(value.plugins)) {
|
||||
throw new Error('System Plugin selection state is invalid')
|
||||
}
|
||||
const selection = defaultSelection(pluginIds, protectedIds)
|
||||
for (const id of pluginIds) {
|
||||
const saved = value.plugins[id]
|
||||
if (saved === undefined) continue
|
||||
if (saved === null || typeof saved !== 'object' || Array.isArray(saved)
|
||||
|| typeof saved.installed !== 'boolean' || typeof saved.enabled !== 'boolean') {
|
||||
throw new Error(`System Plugin ${id} selection state is invalid`)
|
||||
}
|
||||
if (saved.enabled && !saved.installed) {
|
||||
throw new Error(`System Plugin ${id} cannot be enabled while it is not installed`)
|
||||
}
|
||||
selection[id] = { installed: saved.installed, enabled: saved.enabled, protected: protectedIds.has(id) }
|
||||
}
|
||||
return selection
|
||||
}
|
||||
|
||||
export class SystemPluginSelectionStore {
|
||||
constructor(path, { protectedIds = PROTECTED_SYSTEM_PLUGIN_IDS } = {}) {
|
||||
this.path = resolve(path)
|
||||
this.protectedIds = new Set(protectedIds)
|
||||
}
|
||||
|
||||
async read(pluginIds) {
|
||||
const ids = pluginIds.map(assertPluginId)
|
||||
let value
|
||||
try {
|
||||
value = JSON.parse(await readFile(this.path, 'utf8'))
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return defaultSelection(ids, this.protectedIds)
|
||||
throw error
|
||||
}
|
||||
return validateSelectionDocument(value, ids, this.protectedIds)
|
||||
}
|
||||
|
||||
async write(pluginIds, selection) {
|
||||
const ids = pluginIds.map(assertPluginId)
|
||||
const normalized = validateSelectionDocument({ schema: SELECTION_SCHEMA, plugins: selection }, ids, this.protectedIds)
|
||||
const document = {
|
||||
schema: SELECTION_SCHEMA,
|
||||
plugins: Object.fromEntries(ids.map(id => [id, {
|
||||
installed: normalized[id].installed,
|
||||
enabled: normalized[id].enabled,
|
||||
}])),
|
||||
}
|
||||
await mkdir(dirname(this.path), { recursive: true })
|
||||
const temporary = `${this.path}.${randomUUID()}.tmp`
|
||||
try {
|
||||
await writeFile(temporary, `${canonicalJson(document)}\n`, { flag: 'wx', mode: 0o600 })
|
||||
await rename(temporary, this.path)
|
||||
} finally {
|
||||
await rm(temporary, { force: true })
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
async configure(pluginIds, pluginId, action) {
|
||||
const ids = pluginIds.map(assertPluginId)
|
||||
assertPluginId(pluginId)
|
||||
if (!ids.includes(pluginId)) throw new Error(`System Plugin ${pluginId} is not provided by the current Environment`)
|
||||
if (this.protectedIds.has(pluginId)) throw new Error(`System Plugin ${pluginId} is managed by the platform and cannot be changed`)
|
||||
return this.configureSelection(ids, pluginId, action)
|
||||
}
|
||||
|
||||
async recover(pluginIds, pluginId, action) {
|
||||
const ids = pluginIds.map(assertPluginId)
|
||||
assertPluginId(pluginId)
|
||||
if (!ids.includes(pluginId)) throw new Error(`System Plugin ${pluginId} is not provided by the current Environment`)
|
||||
if (!this.protectedIds.has(pluginId)) throw new Error(`System Plugin ${pluginId} is not a platform recovery target`)
|
||||
return this.configureSelection(ids, pluginId, action)
|
||||
}
|
||||
|
||||
async configureSelection(ids, pluginId, action) {
|
||||
if (!['install', 'uninstall', 'enable', 'disable'].includes(action)) {
|
||||
throw new Error('System Plugin action is invalid')
|
||||
}
|
||||
const selection = await this.read(ids)
|
||||
const current = selection[pluginId]
|
||||
if (action === 'install') selection[pluginId] = { ...current, installed: true, enabled: true }
|
||||
else if (action === 'uninstall') selection[pluginId] = { ...current, installed: false, enabled: false }
|
||||
else if (action === 'enable') {
|
||||
if (!current.installed) throw new Error(`System Plugin ${pluginId} must be installed before it can be enabled`)
|
||||
selection[pluginId] = { ...current, enabled: true }
|
||||
} else selection[pluginId] = { ...current, enabled: false }
|
||||
return this.write(ids, selection)
|
||||
}
|
||||
}
|
||||
|
||||
async function pluginCatalog(environmentRoot) {
|
||||
const root = resolve(environmentRoot)
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(root, 'environment.manifest.json')))
|
||||
return {
|
||||
root,
|
||||
plugins: manifest.systemPlugins.map(reference => ({
|
||||
id: reference.id,
|
||||
descriptor: artifactForReference(manifest, reference),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async function readActiveSelection(root, plugins) {
|
||||
if (root === undefined) return undefined
|
||||
const overlay = await readOverlay(resolve(root))
|
||||
const enabledPackages = new Set(overlay.flatMap(entry => (
|
||||
Array.isArray(entry?.insert) ? entry.insert.map(row => row?.name).filter(Boolean) : []
|
||||
)))
|
||||
return Object.fromEntries(await Promise.all(plugins.map(async ({ id }) => {
|
||||
const installed = await lstat(join(resolve(root), 'packages', id)).then(
|
||||
() => true,
|
||||
error => error?.code === 'ENOENT' ? false : Promise.reject(error),
|
||||
)
|
||||
return [id, { installed, enabled: installed && enabledPackages.has(`@dsh-docker/${id}`) }]
|
||||
})))
|
||||
}
|
||||
|
||||
export async function listManagedSystemPlugins({ environmentRoot, sourceRoot, selectionStore, activeRoot }) {
|
||||
const catalog = await pluginCatalog(environmentRoot)
|
||||
const selection = await selectionStore.read(catalog.plugins.map(plugin => plugin.id))
|
||||
const active = await readActiveSelection(activeRoot, catalog.plugins)
|
||||
return Object.freeze(await Promise.all(catalog.plugins.map(async ({ id, descriptor }) => {
|
||||
const metadata = JSON.parse(await readFile(join(resolve(sourceRoot), 'packages', id, 'package.json'), 'utf8'))
|
||||
validatePackage(metadata, id)
|
||||
let description = null
|
||||
const value = metadata.dshDocker?.description
|
||||
if (value !== undefined) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| typeof value.zh !== 'string' || typeof value.en !== 'string'
|
||||
|| value.zh.length === 0 || value.en.length === 0 || value.zh.length > 240 || value.en.length > 240) {
|
||||
throw new Error(`System Plugin ${id} description is invalid`)
|
||||
}
|
||||
description = Object.freeze({ zh: value.zh, en: value.en })
|
||||
}
|
||||
const activeInstalled = active?.[id].installed ?? selection[id].installed
|
||||
const activeEnabled = active?.[id].enabled ?? selection[id].enabled
|
||||
return Object.freeze({
|
||||
id,
|
||||
artifactId: descriptor.id,
|
||||
sha256: descriptor.sha256,
|
||||
description,
|
||||
installed: selection[id].installed,
|
||||
enabled: selection[id].enabled,
|
||||
activeInstalled,
|
||||
activeEnabled,
|
||||
pendingRestart: activeInstalled !== selection[id].installed || activeEnabled !== selection[id].enabled,
|
||||
protected: selection[id].protected,
|
||||
reason: null,
|
||||
})
|
||||
})))
|
||||
}
|
||||
|
||||
export async function discardSystemPluginSelection({ environmentRoot, sourceRoot, selectionStore, activeRoot }) {
|
||||
const catalog = await pluginCatalog(environmentRoot)
|
||||
const pluginIds = catalog.plugins.map(plugin => plugin.id)
|
||||
const active = await readActiveSelection(activeRoot, catalog.plugins)
|
||||
if (active === undefined) throw new Error('active System Plugin view is unavailable')
|
||||
await selectionStore.write(pluginIds, active)
|
||||
return listManagedSystemPlugins({ environmentRoot, sourceRoot, selectionStore, activeRoot })
|
||||
}
|
||||
|
||||
export async function materializeSystemPluginSelection({
|
||||
environmentRoot,
|
||||
sourceRoot,
|
||||
outputRoot,
|
||||
selectionStore,
|
||||
}) {
|
||||
const catalog = await pluginCatalog(environmentRoot)
|
||||
const pluginIds = catalog.plugins.map(plugin => plugin.id)
|
||||
const selection = await selectionStore.read(pluginIds)
|
||||
const root = resolve(outputRoot)
|
||||
const staging = join(root, `.selection.${randomUUID()}.tmp`)
|
||||
const destination = join(root, `selection-${randomUUID()}`)
|
||||
const patches = []
|
||||
await mkdir(join(staging, 'packages'), { recursive: true })
|
||||
try {
|
||||
for (const { id } of catalog.plugins) {
|
||||
if (!selection[id].installed) continue
|
||||
const packageRoot = join(resolve(sourceRoot), 'packages', id)
|
||||
const metadata = JSON.parse(await readFile(join(packageRoot, 'package.json'), 'utf8'))
|
||||
const packageName = validatePackage(metadata, id)
|
||||
const patch = JSON.parse(await readFile(join(packageRoot, 'cordis.patch.json'), 'utf8'))
|
||||
const validatedPatch = validatePatch(patch, id, packageName)
|
||||
await symlink(packageRoot, join(staging, 'packages', id), 'dir')
|
||||
if (selection[id].enabled) patches.push(...validatedPatch)
|
||||
}
|
||||
await writeFile(join(staging, 'cordis.patch.yml'), canonicalJson(patches), { flag: 'wx' })
|
||||
await rename(staging, destination)
|
||||
return Object.freeze({ path: destination, plugins: await listManagedSystemPlugins({ environmentRoot, sourceRoot, selectionStore }) })
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export async function pruneSystemPluginSelectionViews({ outputRoot, keepPath }) {
|
||||
const root = resolve(outputRoot)
|
||||
const keep = resolve(keepPath)
|
||||
for (const entry of await readdir(root, { withFileTypes: true })) {
|
||||
if (!entry.name.startsWith('selection-')) continue
|
||||
const candidate = join(root, entry.name)
|
||||
if (candidate !== keep) await rm(candidate, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export async function listBundledSystemPlugins({ environmentRoot, viewRoot }) {
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(resolve(environmentRoot), 'environment.manifest.json')))
|
||||
const overlay = await readOverlay(resolve(viewRoot)).catch(() => [])
|
||||
const moduleNames = new Set(overlay.flatMap(entry => (
|
||||
Array.isArray(entry?.insert) ? entry.insert.map(row => row?.name).filter(Boolean) : []
|
||||
)))
|
||||
const result = []
|
||||
for (const reference of manifest.systemPlugins) {
|
||||
const descriptor = artifactForReference(manifest, reference)
|
||||
let installed = false
|
||||
let reason = null
|
||||
try {
|
||||
const metadata = JSON.parse(await readFile(join(resolve(viewRoot), 'packages', reference.id, 'package.json'), 'utf8'))
|
||||
const packageName = validatePackage(metadata, reference.id)
|
||||
installed = moduleNames.has(packageName)
|
||||
if (!installed) reason = 'plugin loader entry is absent'
|
||||
} catch (error) {
|
||||
reason = error instanceof Error ? error.message : 'plugin package is unavailable'
|
||||
}
|
||||
result.push(Object.freeze({
|
||||
id: reference.id,
|
||||
artifactId: descriptor.id,
|
||||
sha256: descriptor.sha256,
|
||||
installed,
|
||||
reason,
|
||||
}))
|
||||
}
|
||||
return Object.freeze(result)
|
||||
}
|
||||
|
||||
export async function rebuildBundledSystemPluginView({
|
||||
environmentRoot,
|
||||
outputRoot,
|
||||
expectedSha256,
|
||||
requestedPluginId,
|
||||
}) {
|
||||
const source = resolve(environmentRoot)
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(source, 'environment.manifest.json')))
|
||||
if (!manifest.systemPlugins.some(plugin => plugin.id === requestedPluginId)) {
|
||||
throw new Error(`System Plugin ${requestedPluginId} is not bundled by the current Environment`)
|
||||
}
|
||||
const root = resolve(outputRoot)
|
||||
const destination = join(root, expectedSha256)
|
||||
try {
|
||||
const details = await lstat(destination)
|
||||
if (!details.isDirectory() || details.isSymbolicLink() || await hashTree(destination) !== expectedSha256) {
|
||||
await rm(destination, { recursive: true, force: true })
|
||||
} else {
|
||||
return destination
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
|
||||
const buildRoot = join(root, `.build.${randomUUID()}.tmp`)
|
||||
await mkdir(root, { recursive: true })
|
||||
try {
|
||||
const built = await reconcileSystemPlugins({
|
||||
root: buildRoot,
|
||||
environmentVersion: expectedSha256,
|
||||
plugins: manifest.systemPlugins,
|
||||
artifactPath: reference => join(source, 'artifacts', artifactForReference(manifest, reference).id),
|
||||
})
|
||||
if (await hashTree(built) !== expectedSha256) {
|
||||
throw new Error('rebuilt System Plugin view differs from the current Deployment Record')
|
||||
}
|
||||
await rename(built, destination)
|
||||
return destination
|
||||
} finally {
|
||||
await rm(buildRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export async function linkSystemPluginScope({
|
||||
dshHome,
|
||||
viewRoot = '/run/dsh-platform/views/system-plugins',
|
||||
}) {
|
||||
const modulesRoot = join(resolve(dshHome), 'profiles', 'node_modules')
|
||||
const scopeLink = join(modulesRoot, '@dsh-docker')
|
||||
const target = join(resolve(viewRoot), 'packages')
|
||||
await mkdir(modulesRoot, { recursive: true })
|
||||
const existing = await lstat(scopeLink).catch(error => error?.code === 'ENOENT' ? undefined : Promise.reject(error))
|
||||
if (existing !== undefined) {
|
||||
if (!existing.isSymbolicLink()) {
|
||||
throw new Error(`System Plugin scope ${scopeLink} exists and is not a symlink`)
|
||||
}
|
||||
if (await readlink(scopeLink) === target) return scopeLink
|
||||
await rm(scopeLink)
|
||||
}
|
||||
const temporary = join(modulesRoot, `.@dsh-docker.${randomUUID()}.tmp`)
|
||||
try {
|
||||
await symlink(target, temporary, 'dir')
|
||||
await rename(temporary, scopeLink)
|
||||
} finally {
|
||||
await rm(temporary, { force: true })
|
||||
}
|
||||
return scopeLink
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
|
||||
const PACKAGE_NAME_PATTERN = /^(?:@[a-z0-9][a-z0-9._-]{0,126}\/[a-z0-9][a-z0-9._-]{0,126}|[a-z0-9][a-z0-9._-]{0,213})$/i
|
||||
const SELECTION_SCHEMA = 1
|
||||
|
||||
function packagePath(root, packageName) {
|
||||
if (!PACKAGE_NAME_PATTERN.test(packageName)) return undefined
|
||||
return join(root, 'node_modules', ...packageName.split('/'), 'package.json')
|
||||
}
|
||||
|
||||
async function optionalBytes(path) {
|
||||
return readFile(path).then(
|
||||
bytes => ({ present: true, bytes }),
|
||||
error => error?.code === 'ENOENT' ? { present: false, bytes: Buffer.alloc(0) } : Promise.reject(error),
|
||||
)
|
||||
}
|
||||
|
||||
function parseObject(bytes, label) {
|
||||
let value
|
||||
try {
|
||||
value = JSON.parse(bytes.toString('utf8'))
|
||||
} catch (error) {
|
||||
throw new Error(`${label} is not valid JSON`, { cause: error })
|
||||
}
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new Error(`${label} must contain a JSON object`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function profileFields(manifest) {
|
||||
const dependencies = manifest.dependencies ?? {}
|
||||
if (dependencies === null || typeof dependencies !== 'object' || Array.isArray(dependencies)) {
|
||||
throw new Error('Web Profile dependencies must be an object')
|
||||
}
|
||||
for (const [name, spec] of Object.entries(dependencies)) {
|
||||
if (typeof spec !== 'string' || spec.length === 0) {
|
||||
throw new Error(`Web Profile dependency ${name} must have a non-empty string spec`)
|
||||
}
|
||||
}
|
||||
const bundles = manifest.dsh?.profile?.bundles ?? []
|
||||
if (!Array.isArray(bundles) || bundles.some(name => typeof name !== 'string')) {
|
||||
throw new Error('Web Profile dsh.profile.bundles must be a string array')
|
||||
}
|
||||
return { dependencies, bundles }
|
||||
}
|
||||
|
||||
function disabledPositions(value) {
|
||||
if (value === undefined) return new Map()
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| value.schema !== SELECTION_SCHEMA || !Array.isArray(value.disabled)) {
|
||||
throw new Error('User Plugin selection state is invalid')
|
||||
}
|
||||
const positions = new Map()
|
||||
for (const entry of value.disabled) {
|
||||
if (entry === null || typeof entry !== 'object' || Array.isArray(entry)
|
||||
|| !PACKAGE_NAME_PATTERN.test(entry.name) || !Number.isSafeInteger(entry.index) || entry.index < 0) {
|
||||
throw new Error('User Plugin disabled-order entry is invalid')
|
||||
}
|
||||
if (positions.has(entry.name)) throw new Error(`User Plugin ${entry.name} has duplicate disabled-order entries`)
|
||||
positions.set(entry.name, entry.index)
|
||||
}
|
||||
return positions
|
||||
}
|
||||
|
||||
function sourceForSpec(spec) {
|
||||
if (/^(?:file|link):/i.test(spec) || /^(?:\.{1,2}\/|\/)/.test(spec)) return 'file'
|
||||
if (/^(?:git(?:\+[^:]+)?|github|gitlab|bitbucket):/i.test(spec) || /\.git(?:#|$)/i.test(spec)) return 'git'
|
||||
if (/^https?:/i.test(spec)) return 'url'
|
||||
if (/^npm:/i.test(spec) || /^(?:[~^<>=*]|\d|v\d)/i.test(spec) || spec === 'latest' || spec === 'next') return 'registry'
|
||||
return 'other'
|
||||
}
|
||||
|
||||
function updateRevision(hash, label, file) {
|
||||
hash.update(`${label}:${file.present ? '1' : '0'}:${String(file.bytes.length)}:`)
|
||||
hash.update(file.bytes)
|
||||
hash.update('\0')
|
||||
}
|
||||
|
||||
function metadataFailure(error) {
|
||||
if (error?.code === 'ENOENT') return 'installed package metadata is missing'
|
||||
if (error instanceof SyntaxError) return 'installed package metadata is not valid JSON'
|
||||
return error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
async function inspectDependency(profileRoot, name, spec, enabledBundles, disabled, reservedNames) {
|
||||
const path = packagePath(profileRoot, name)
|
||||
let metadata
|
||||
let metadataError = null
|
||||
if (path === undefined) {
|
||||
metadataError = 'dependency name is invalid'
|
||||
} else {
|
||||
try {
|
||||
metadata = JSON.parse(await readFile(path, 'utf8'))
|
||||
if (metadata === null || typeof metadata !== 'object' || Array.isArray(metadata)) {
|
||||
throw new Error('installed package metadata must be an object')
|
||||
}
|
||||
if (metadata.name !== name) throw new Error(`installed package name is ${String(metadata.name)}`)
|
||||
} catch (error) {
|
||||
metadataError = metadataFailure(error)
|
||||
}
|
||||
}
|
||||
if (metadataError === null && metadata.version !== undefined
|
||||
&& (typeof metadata.version !== 'string' || metadata.version.length === 0)) {
|
||||
metadataError = 'installed package version is invalid'
|
||||
}
|
||||
const bundleDeclaration = metadata?.dsh?.bundle
|
||||
const patch = bundleDeclaration?.patch
|
||||
const isBundle = typeof patch === 'string' && patch.length > 0
|
||||
const reservedNameConflict = reservedNames.has(name)
|
||||
if (metadataError === null && !isBundle
|
||||
&& (bundleDeclaration !== undefined || enabledBundles.has(name))) {
|
||||
metadataError = 'installed package Bundle metadata is invalid'
|
||||
}
|
||||
if (!isBundle && metadataError === null && !reservedNameConflict) return undefined
|
||||
return Object.freeze({
|
||||
name,
|
||||
spec,
|
||||
source: sourceForSpec(spec),
|
||||
version: typeof metadata?.version === 'string' ? metadata.version : null,
|
||||
description: typeof metadata?.description === 'string' && metadata.description.trim() !== '' ? metadata.description.trim() : null,
|
||||
enabled: !reservedNameConflict && enabledBundles.has(name),
|
||||
previousIndex: disabled.get(name) ?? null,
|
||||
damaged: metadataError !== null,
|
||||
metadataError,
|
||||
reservedNameConflict,
|
||||
})
|
||||
}
|
||||
|
||||
function loadedIdentity(plugin) {
|
||||
return JSON.stringify([
|
||||
plugin.name, plugin.spec, plugin.version, plugin.enabled, plugin.damaged,
|
||||
plugin.metadataError, plugin.reservedNameConflict,
|
||||
])
|
||||
}
|
||||
|
||||
export function markUserPluginRestartState(current, loaded) {
|
||||
if (loaded === undefined) return current
|
||||
const loadedPlugins = new Map(loaded.plugins.map(plugin => [plugin.name, loadedIdentity(plugin)]))
|
||||
const plugins = current.plugins.map(plugin => Object.freeze({
|
||||
...plugin,
|
||||
pendingRestart: loadedPlugins.get(plugin.name) !== loadedIdentity(plugin),
|
||||
}))
|
||||
const currentNames = new Set(current.plugins.map(plugin => plugin.name))
|
||||
const removedLoadedPlugin = loaded.plugins.some(plugin => !currentNames.has(plugin.name))
|
||||
return Object.freeze({
|
||||
...current,
|
||||
plugins: Object.freeze(plugins),
|
||||
restartRequired: removedLoadedPlugin || plugins.some(plugin => plugin.pendingRestart),
|
||||
})
|
||||
}
|
||||
|
||||
export class UserPluginInventory {
|
||||
constructor({ dshHome = '/data/dsh', profile = 'web', selectionPath, systemPluginNames = async () => [] } = {}) {
|
||||
if (profile !== 'web') throw new Error('Only the web DSH Profile is supported')
|
||||
this.profile = profile
|
||||
this.profileRoot = join(resolve(dshHome), 'profiles', profile)
|
||||
this.selectionPath = selectionPath === undefined
|
||||
? join(resolve(dshHome), '..', 'platform', 'state', 'management', 'user-plugins.json')
|
||||
: resolve(selectionPath)
|
||||
this.systemPluginNames = systemPluginNames
|
||||
}
|
||||
|
||||
async read() {
|
||||
const manifestPath = join(this.profileRoot, 'package.json')
|
||||
const lockfilePath = join(this.profileRoot, 'pnpm-lock.yaml')
|
||||
const [manifestBytes, lockfile, selectionFile, names] = await Promise.all([
|
||||
readFile(manifestPath),
|
||||
optionalBytes(lockfilePath),
|
||||
optionalBytes(this.selectionPath),
|
||||
this.systemPluginNames(),
|
||||
])
|
||||
const manifest = parseObject(manifestBytes, 'Web Profile package.json')
|
||||
const { dependencies, bundles } = profileFields(manifest)
|
||||
const selection = !selectionFile.present
|
||||
? undefined
|
||||
: parseObject(selectionFile.bytes, 'User Plugin selection state')
|
||||
const disabled = disabledPositions(selection)
|
||||
const reservedNames = new Set(names)
|
||||
if ([...reservedNames].some(name => typeof name !== 'string' || !PACKAGE_NAME_PATTERN.test(name))) {
|
||||
throw new Error('Verified System Plugin names are invalid')
|
||||
}
|
||||
const enabledBundles = new Set(bundles)
|
||||
const inspected = await Promise.all(Object.entries(dependencies).map(([name, spec]) => (
|
||||
inspectDependency(this.profileRoot, name, spec, enabledBundles, disabled, reservedNames)
|
||||
)))
|
||||
const revisionHash = createHash('sha256')
|
||||
updateRevision(revisionHash, 'manifest', { present: true, bytes: manifestBytes })
|
||||
updateRevision(revisionHash, 'lockfile', lockfile)
|
||||
updateRevision(revisionHash, 'selection', selectionFile)
|
||||
const revision = revisionHash.digest('hex')
|
||||
return Object.freeze({
|
||||
schema: 1,
|
||||
profile: this.profile,
|
||||
revision: `sha256:${revision}`,
|
||||
plugins: Object.freeze(inspected.filter(Boolean)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const userPluginInternals = Object.freeze({ PACKAGE_NAME_PATTERN, sourceForSpec, loadedIdentity })
|
||||
@@ -0,0 +1,116 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { durableReplace } from '../../../platform/lib/atomic.mjs'
|
||||
import { userPluginInternals } from './user-inventory.mjs'
|
||||
|
||||
const PHASES = new Set([
|
||||
'validated', 'paused', 'snapshotted', 'mutating', 'committed', 'restarting',
|
||||
'restoring', 'completed', 'failed',
|
||||
])
|
||||
const TERMINAL = new Set(['completed', 'failed'])
|
||||
const TRANSITIONS = new Map([
|
||||
['validated', new Set(['paused', 'snapshotted', 'failed'])],
|
||||
['paused', new Set(['snapshotted', 'restoring', 'failed'])],
|
||||
['snapshotted', new Set(['mutating', 'restoring'])],
|
||||
['mutating', new Set(['committed', 'restoring'])],
|
||||
['committed', new Set(['restarting', 'completed', 'failed'])],
|
||||
['restarting', new Set(['completed', 'failed'])],
|
||||
['restoring', new Set(['failed'])],
|
||||
['completed', new Set()],
|
||||
['failed', new Set()],
|
||||
])
|
||||
|
||||
function parseAction(value) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).sort().join(',') !== 'action,name'
|
||||
|| typeof value.name !== 'string' || !userPluginInternals.PACKAGE_NAME_PATTERN.test(value.name)
|
||||
|| !['enable', 'disable', 'uninstall'].includes(value.action)) {
|
||||
throw new Error('User Plugin journal action is invalid')
|
||||
}
|
||||
return Object.freeze({ name: value.name, action: value.action })
|
||||
}
|
||||
|
||||
function parse(value) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).sort().join(',') !== 'actions,error,phase,previousDisabled,recoveryResult,revision,schema,selectionPresent,snapshotId,taskId,updatedAt'
|
||||
|| value.schema !== 1 || typeof value.taskId !== 'string' || !/^[A-Za-z0-9-]{1,128}$/.test(value.taskId)
|
||||
|| !PHASES.has(value.phase) || typeof value.revision !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(value.revision)
|
||||
|| !Array.isArray(value.actions) || value.actions.length === 0
|
||||
|| typeof value.selectionPresent !== 'boolean' || !Array.isArray(value.previousDisabled)
|
||||
|| (value.snapshotId !== null && typeof value.snapshotId !== 'string')
|
||||
|| (value.error !== null && typeof value.error !== 'string')
|
||||
|| ![null, 'success', 'failed'].includes(value.recoveryResult)
|
||||
|| typeof value.updatedAt !== 'string' || new Date(value.updatedAt).toISOString() !== value.updatedAt) {
|
||||
throw new Error('User Plugin transaction journal is invalid')
|
||||
}
|
||||
const names = new Set()
|
||||
const previousDisabled = value.previousDisabled.map(entry => {
|
||||
if (entry === null || typeof entry !== 'object' || Array.isArray(entry)
|
||||
|| Object.keys(entry).sort().join(',') !== 'index,name'
|
||||
|| typeof entry.name !== 'string' || !userPluginInternals.PACKAGE_NAME_PATTERN.test(entry.name)
|
||||
|| !Number.isSafeInteger(entry.index) || entry.index < 0
|
||||
|| names.has(entry.name)) throw new Error('User Plugin journal disabled-order state is invalid')
|
||||
names.add(entry.name)
|
||||
return Object.freeze({ name: entry.name, index: entry.index })
|
||||
})
|
||||
return Object.freeze({
|
||||
...value,
|
||||
actions: Object.freeze(value.actions.map(parseAction)),
|
||||
previousDisabled: Object.freeze(previousDisabled),
|
||||
})
|
||||
}
|
||||
|
||||
export class UserPluginJournal {
|
||||
constructor(path, now = () => new Date()) {
|
||||
this.path = path
|
||||
this.now = now
|
||||
this.queue = Promise.resolve()
|
||||
}
|
||||
|
||||
exclusive(operation) {
|
||||
const result = this.queue.then(operation, operation)
|
||||
this.queue = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
async read() {
|
||||
try { return parse(JSON.parse(await readFile(this.path, 'utf8'))) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
begin({ taskId, revision, actions, selection }) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
if (current !== undefined && !TERMINAL.has(current.phase)) {
|
||||
throw new Error('a User Plugin transaction is already active')
|
||||
}
|
||||
const next = parse({
|
||||
schema: 1, taskId, revision, actions,
|
||||
selectionPresent: selection.present,
|
||||
previousDisabled: selection.state.disabled,
|
||||
phase: 'validated', snapshotId: null,
|
||||
error: null, recoveryResult: null, updatedAt: this.now().toISOString(),
|
||||
})
|
||||
await durableReplace(this.path, `${JSON.stringify(next)}\n`)
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
transition(phase, fields = {}) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
if (current === undefined || !TRANSITIONS.get(current.phase)?.has(phase)) {
|
||||
throw new Error(`User Plugin journal cannot transition to ${phase}`)
|
||||
}
|
||||
const next = parse({ ...current, ...fields, phase, updatedAt: this.now().toISOString() })
|
||||
if (next.taskId !== current.taskId || next.revision !== current.revision) {
|
||||
throw new Error('User Plugin journal identity cannot change')
|
||||
}
|
||||
await durableReplace(this.path, `${JSON.stringify(next)}\n`)
|
||||
return next
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const userPluginJournalInternals = Object.freeze({ parse, TERMINAL })
|
||||
@@ -0,0 +1,143 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { lstat, mkdir, open, readFile, readdir, rename, rm } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { canonicalJson } from '../../../platform/lib/canonical-json.mjs'
|
||||
import { durableReplace } from '../../../platform/lib/atomic.mjs'
|
||||
|
||||
const ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/
|
||||
|
||||
function validId(id) {
|
||||
if (typeof id !== 'string' || !ID_PATTERN.test(id)) throw new Error('User Plugin snapshot ID is invalid')
|
||||
return id
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolveRun(Buffer.concat(stdout).toString('utf8'))
|
||||
: reject(new Error(`${command} failed: ${Buffer.concat(stderr).toString('utf8').trim()}`)))
|
||||
})
|
||||
}
|
||||
|
||||
async function measure(path) {
|
||||
const handle = await open(path, 'r')
|
||||
const hash = createHash('sha256')
|
||||
let size = 0
|
||||
try {
|
||||
for await (const chunk of handle.createReadStream({ autoClose: false })) {
|
||||
hash.update(chunk)
|
||||
size += chunk.byteLength
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return { sha256: hash.digest('hex'), size }
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
function parseManifest(value) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).sort().join(',') !== 'archiveSha256,archiveSize,createdAt,id,schema,source'
|
||||
|| value.schema !== 1 || !ID_PATTERN.test(value.id)
|
||||
|| value.source !== '/data/dsh/profiles/web'
|
||||
|| typeof value.createdAt !== 'string' || new Date(value.createdAt).toISOString() !== value.createdAt
|
||||
|| typeof value.archiveSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.archiveSha256)
|
||||
|| !Number.isSafeInteger(value.archiveSize) || value.archiveSize < 0) {
|
||||
throw new Error('User Plugin snapshot manifest is invalid')
|
||||
}
|
||||
return Object.freeze({ ...value })
|
||||
}
|
||||
|
||||
export class UserPluginSnapshots {
|
||||
constructor({ root, profileRoot = '/data/dsh/profiles/web', now = () => new Date(), runImpl = run }) {
|
||||
this.root = resolve(root)
|
||||
this.profileRoot = resolve(profileRoot)
|
||||
this.now = now
|
||||
this.run = runImpl
|
||||
}
|
||||
|
||||
path(id) { return join(this.root, validId(id)) }
|
||||
|
||||
async create(id = randomUUID()) {
|
||||
validId(id)
|
||||
const details = await lstat(this.profileRoot)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) throw new Error('Web Profile must be a real directory')
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const destination = this.path(id)
|
||||
await lstat(destination).then(
|
||||
() => { throw new Error(`User Plugin snapshot ${id} already exists`) },
|
||||
error => { if (error?.code !== 'ENOENT') throw error },
|
||||
)
|
||||
const staging = join(this.root, `.${id}.${randomUUID()}.tmp`)
|
||||
await mkdir(staging)
|
||||
const archive = join(staging, 'profile.tar')
|
||||
try {
|
||||
await this.run('tar', ['-cpf', archive, '-C', this.profileRoot, '.'])
|
||||
const measured = await measure(archive)
|
||||
const manifest = parseManifest({
|
||||
schema: 1,
|
||||
id,
|
||||
source: '/data/dsh/profiles/web',
|
||||
createdAt: this.now().toISOString(),
|
||||
archiveSha256: measured.sha256,
|
||||
archiveSize: measured.size,
|
||||
})
|
||||
await durableReplace(join(staging, 'manifest.json'), canonicalJson(manifest))
|
||||
await rename(staging, destination)
|
||||
await syncDirectory(this.root)
|
||||
return Object.freeze({ ...manifest, path: destination })
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(id) {
|
||||
const root = this.path(id)
|
||||
const manifest = parseManifest(JSON.parse(await readFile(join(root, 'manifest.json'), 'utf8')))
|
||||
if (manifest.id !== id) throw new Error('User Plugin snapshot ID does not match its directory')
|
||||
const archive = await lstat(join(root, 'profile.tar')).then(
|
||||
() => join(root, 'profile.tar'),
|
||||
error => error?.code === 'ENOENT' ? join(root, 'profile.tar.gz') : Promise.reject(error),
|
||||
)
|
||||
const measured = await measure(archive)
|
||||
if (measured.sha256 !== manifest.archiveSha256 || measured.size !== manifest.archiveSize) {
|
||||
throw new Error('User Plugin snapshot archive does not match its manifest')
|
||||
}
|
||||
const entries = (await this.run('tar', [archive.endsWith('.gz') ? '-tzf' : '-tf', archive])).split('\n').filter(Boolean)
|
||||
if (entries.length === 0 || entries.some(entry => entry.startsWith('/') || entry.split('/').includes('..'))) {
|
||||
throw new Error('User Plugin snapshot archive contains unsafe paths')
|
||||
}
|
||||
return Object.freeze({ ...manifest, archive })
|
||||
}
|
||||
|
||||
async restore(id) {
|
||||
const snapshot = await this.inspect(id)
|
||||
await mkdir(this.profileRoot, { recursive: true })
|
||||
for (const entry of await readdir(this.profileRoot)) {
|
||||
await rm(join(this.profileRoot, entry), { recursive: true, force: true })
|
||||
}
|
||||
const ownership = process.getuid?.() === 0 ? [] : ['--no-same-owner']
|
||||
await this.run('tar', [snapshot.archive.endsWith('.gz') ? '-xzpf' : '-xpf', snapshot.archive, ...ownership, '-C', this.profileRoot])
|
||||
await syncDirectory(this.profileRoot)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
async remove(id) {
|
||||
validId(id)
|
||||
await rm(this.path(id), { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export const userPluginSnapshotInternals = Object.freeze({ parseManifest })
|
||||
@@ -0,0 +1,67 @@
|
||||
import { open, readFile, rm } from 'node:fs/promises'
|
||||
import { dirname } from 'node:path'
|
||||
import { canonicalJson } from '../../../platform/lib/canonical-json.mjs'
|
||||
import { durableReplace } from '../../../platform/lib/atomic.mjs'
|
||||
import { userPluginInternals } from './user-inventory.mjs'
|
||||
|
||||
function parse(value) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).sort().join(',') !== 'disabled,schema'
|
||||
|| value.schema !== 1 || !Array.isArray(value.disabled)) {
|
||||
throw new Error('User Plugin selection state is invalid')
|
||||
}
|
||||
const names = new Set()
|
||||
const disabled = value.disabled.map(entry => {
|
||||
if (entry === null || typeof entry !== 'object' || Array.isArray(entry)
|
||||
|| Object.keys(entry).some(key => !['index', 'name'].includes(key))
|
||||
|| !userPluginInternals.PACKAGE_NAME_PATTERN.test(entry.name)
|
||||
|| !Number.isSafeInteger(entry.index) || entry.index < 0 || names.has(entry.name)) {
|
||||
throw new Error('User Plugin disabled-order entry is invalid')
|
||||
}
|
||||
names.add(entry.name)
|
||||
return Object.freeze({ name: entry.name, index: entry.index })
|
||||
})
|
||||
return Object.freeze({ schema: 1, disabled: Object.freeze(disabled) })
|
||||
}
|
||||
|
||||
export class UserPluginSelectionStore {
|
||||
constructor(path) {
|
||||
this.path = path
|
||||
}
|
||||
|
||||
async read() {
|
||||
return (await this.snapshot()).state
|
||||
}
|
||||
|
||||
async snapshot() {
|
||||
try {
|
||||
return Object.freeze({ present: true, state: parse(JSON.parse(await readFile(this.path, 'utf8'))) })
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') {
|
||||
return Object.freeze({ present: false, state: parse({ schema: 1, disabled: [] }) })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async write(value) {
|
||||
const state = parse(value)
|
||||
await durableReplace(this.path, `${canonicalJson(state).toString('utf8')}\n`)
|
||||
return state
|
||||
}
|
||||
|
||||
async restore(snapshot) {
|
||||
if (snapshot === null || typeof snapshot !== 'object' || typeof snapshot.present !== 'boolean') {
|
||||
throw new Error('User Plugin selection snapshot is invalid')
|
||||
}
|
||||
if (snapshot.present) return this.write(snapshot.state)
|
||||
await rm(this.path, { force: true })
|
||||
const directory = await open(dirname(this.path), 'r').catch(error => error?.code === 'ENOENT' ? undefined : Promise.reject(error))
|
||||
if (directory !== undefined) {
|
||||
try { await directory.sync() } finally { await directory.close() }
|
||||
}
|
||||
return parse({ schema: 1, disabled: [] })
|
||||
}
|
||||
}
|
||||
|
||||
export const userPluginStateInternals = Object.freeze({ parse })
|
||||
@@ -0,0 +1,414 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { join } from 'node:path'
|
||||
import { canonicalJson } from '../../../platform/lib/canonical-json.mjs'
|
||||
import { durableReplace } from '../../../platform/lib/atomic.mjs'
|
||||
import { userPluginInternals } from './user-inventory.mjs'
|
||||
|
||||
const PACKAGE_PATH_ENVIRONMENT = new Set([
|
||||
'npm_config_cache',
|
||||
'npm_config_prefix',
|
||||
'npm_config_store_dir',
|
||||
'npm_config_userconfig',
|
||||
'pnpm_home',
|
||||
])
|
||||
|
||||
function pluginEnvironment(dshHome, home = '/home/node') {
|
||||
const environment = {
|
||||
...process.env,
|
||||
HOME: home,
|
||||
XDG_CACHE_HOME: join(home, '.cache'),
|
||||
XDG_CONFIG_HOME: join(home, '.config'),
|
||||
XDG_DATA_HOME: join(home, '.local', 'share'),
|
||||
XDG_STATE_HOME: join(home, '.local', 'state'),
|
||||
DSH_HOME: dshHome,
|
||||
}
|
||||
for (const name of Object.keys(environment)) {
|
||||
if (PACKAGE_PATH_ENVIRONMENT.has(name.toLowerCase())) delete environment[name]
|
||||
}
|
||||
return environment
|
||||
}
|
||||
|
||||
function runDshPlugin({ dshHome, profile, name }, timeoutMs = 300_000) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn('/usr/local/bin/dsh', ['plugin', '--profile', profile, 'remove', name], {
|
||||
env: pluginEnvironment(dshHome),
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
})
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
let size = 0
|
||||
let settled = false
|
||||
let timeoutError
|
||||
let killTimer
|
||||
const finish = (error) => {
|
||||
if (settled) return
|
||||
settled = true
|
||||
clearTimeout(timer)
|
||||
clearTimeout(killTimer)
|
||||
if (error === undefined) resolveRun()
|
||||
else reject(error)
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
timeoutError = new Error(`dsh plugin remove timed out for ${name}`)
|
||||
child.kill('SIGTERM')
|
||||
killTimer = setTimeout(() => child.kill('SIGKILL'), 5_000)
|
||||
killTimer.unref()
|
||||
}, timeoutMs)
|
||||
timer.unref()
|
||||
for (const stream of [child.stdout, child.stderr]) stream.on('data', chunk => {
|
||||
size += chunk.byteLength
|
||||
if (size <= 256 * 1024) (stream === child.stdout ? stdout : stderr).push(chunk)
|
||||
})
|
||||
child.once('error', finish)
|
||||
child.once('exit', code => timeoutError !== undefined
|
||||
? finish(timeoutError)
|
||||
: code === 0
|
||||
? finish()
|
||||
: finish(new Error(`dsh plugin remove failed for ${name}: ${[
|
||||
['stdout', Buffer.concat(stdout).toString('utf8').trim()],
|
||||
['stderr', Buffer.concat(stderr).toString('utf8').trim()],
|
||||
].filter(([, value]) => value !== '').map(([stream, value]) => `${stream}: ${value}`).join('\\n')}`)))
|
||||
})
|
||||
}
|
||||
|
||||
function message(error) {
|
||||
return error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
|
||||
function validateActions(inventory, actions) {
|
||||
if (!Array.isArray(actions) || actions.length === 0 || actions.length > 128) {
|
||||
throw new Error('User Plugin actions must be a non-empty bounded array')
|
||||
}
|
||||
const plugins = new Map(inventory.plugins.map(plugin => [plugin.name, plugin]))
|
||||
const names = new Set()
|
||||
return Object.freeze(actions.map(value => {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).sort().join(',') !== 'action,name'
|
||||
|| !userPluginInternals.PACKAGE_NAME_PATTERN.test(value.name)
|
||||
|| !['enable', 'disable', 'uninstall'].includes(value.action)
|
||||
|| names.has(value.name)) {
|
||||
throw new Error('User Plugin action is invalid')
|
||||
}
|
||||
names.add(value.name)
|
||||
const plugin = plugins.get(value.name)
|
||||
if (plugin === undefined) throw new Error(`User Plugin ${value.name} is not managed by the Web Profile`)
|
||||
if (value.action === 'enable' && (plugin.damaged || plugin.reservedNameConflict)) {
|
||||
throw new Error(`User Plugin ${value.name} cannot be enabled`)
|
||||
}
|
||||
return Object.freeze({ name: value.name, action: value.action })
|
||||
}))
|
||||
}
|
||||
|
||||
async function mutateProfile({ inventory, selectionStore, actions, runPlugin }) {
|
||||
const manifestPath = join(inventory.profileRoot, 'package.json')
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'))
|
||||
const bundles = [...(manifest.dsh?.profile?.bundles ?? [])]
|
||||
const originalBundles = [...bundles]
|
||||
const selection = await selectionStore.read()
|
||||
const disabled = new Map(selection.disabled.map(entry => [entry.name, entry.index]))
|
||||
for (const { name, action } of actions) {
|
||||
if (action === 'disable') {
|
||||
const index = bundles.indexOf(name)
|
||||
if (index !== -1) {
|
||||
disabled.set(name, originalBundles.indexOf(name))
|
||||
for (let found = bundles.indexOf(name); found !== -1; found = bundles.indexOf(name)) bundles.splice(found, 1)
|
||||
}
|
||||
} else if (action === 'enable') {
|
||||
if (!bundles.includes(name)) bundles.splice(Math.min(disabled.get(name) ?? bundles.length, bundles.length), 0, name)
|
||||
disabled.delete(name)
|
||||
} else if (action === 'uninstall') {
|
||||
for (let found = bundles.indexOf(name); found !== -1; found = bundles.indexOf(name)) bundles.splice(found, 1)
|
||||
}
|
||||
}
|
||||
manifest.dsh = { ...manifest.dsh, profile: { ...manifest.dsh?.profile, bundles } }
|
||||
await durableReplace(manifestPath, `${canonicalJson(manifest).toString('utf8')}\n`)
|
||||
for (const { name, action } of actions) {
|
||||
if (action === 'uninstall') {
|
||||
await runPlugin({ dshHome: join(inventory.profileRoot, '..', '..'), profile: inventory.profile, name })
|
||||
disabled.delete(name)
|
||||
}
|
||||
}
|
||||
if (actions.some(({ action }) => action === 'uninstall')) {
|
||||
const installedManifest = JSON.parse(await readFile(manifestPath, 'utf8'))
|
||||
installedManifest.dsh = {
|
||||
...installedManifest.dsh,
|
||||
profile: { ...installedManifest.dsh?.profile, bundles },
|
||||
}
|
||||
await durableReplace(manifestPath, `${canonicalJson(installedManifest).toString('utf8')}\n`)
|
||||
}
|
||||
await selectionStore.write({
|
||||
schema: 1,
|
||||
disabled: [...disabled].map(([name, index]) => ({ name, index })),
|
||||
})
|
||||
const resulting = await inventory.read()
|
||||
const result = new Map(resulting.plugins.map(plugin => [plugin.name, plugin]))
|
||||
for (const { name, action } of actions) {
|
||||
if (action === 'uninstall' && result.has(name)) throw new Error(`User Plugin ${name} remains installed`)
|
||||
if (action === 'enable' && result.get(name)?.enabled !== true) throw new Error(`User Plugin ${name} was not enabled`)
|
||||
if (action === 'disable' && result.get(name)?.enabled !== false) throw new Error(`User Plugin ${name} was not disabled`)
|
||||
}
|
||||
return resulting
|
||||
}
|
||||
|
||||
export class UserPluginTransactionManager {
|
||||
constructor({ inventory, selectionStore, snapshots, journal, pauseDsh, restartDsh, runPlugin = runDshPlugin, report = async () => {} }) {
|
||||
this.inventory = inventory
|
||||
this.selectionStore = selectionStore
|
||||
this.snapshots = snapshots
|
||||
this.journal = journal
|
||||
this.pauseDsh = pauseDsh
|
||||
this.restartDsh = restartDsh
|
||||
this.runPlugin = runPlugin
|
||||
this.report = report
|
||||
}
|
||||
|
||||
record(messageValue, fields = {}) {
|
||||
return Promise.resolve().then(() => this.report(messageValue, fields)).catch(() => {})
|
||||
}
|
||||
|
||||
async validate({ revision, actions }) {
|
||||
const current = await this.inventory.read()
|
||||
if (revision !== current.revision) {
|
||||
const error = new Error('User Plugin inventory revision has changed')
|
||||
error.code = 'REVISION_CONFLICT'
|
||||
throw error
|
||||
}
|
||||
return Object.freeze({ revision, actions: validateActions(current, actions) })
|
||||
}
|
||||
|
||||
async apply({ taskId = randomUUID(), revision, actions, strategy = 'restart', onProgress = () => {} }) {
|
||||
if (!['restart', 'next-start'].includes(strategy)) throw new Error('User Plugin activation strategy is invalid')
|
||||
const validated = await this.validate({ revision, actions })
|
||||
const normalized = validated.actions
|
||||
const publishProgress = state => { try { onProgress(state) } catch {} }
|
||||
const selectionBefore = await this.selectionStore.snapshot()
|
||||
let state = await this.journal.begin({ taskId, revision, actions: normalized, selection: selectionBefore })
|
||||
publishProgress(state)
|
||||
let paused = false
|
||||
let snapshotCreated = false
|
||||
await this.record('user-plugin.transaction.started', { taskId, actions: normalized })
|
||||
try {
|
||||
if (strategy === 'restart') {
|
||||
await this.pauseDsh()
|
||||
paused = true
|
||||
state = await this.journal.transition('paused')
|
||||
publishProgress(state)
|
||||
}
|
||||
await this.snapshots.create(taskId)
|
||||
snapshotCreated = true
|
||||
state = await this.journal.transition('snapshotted', { snapshotId: taskId })
|
||||
publishProgress(state)
|
||||
state = await this.journal.transition('mutating')
|
||||
publishProgress(state)
|
||||
const result = await mutateProfile({
|
||||
inventory: this.inventory,
|
||||
selectionStore: this.selectionStore,
|
||||
actions: normalized,
|
||||
runPlugin: this.runPlugin,
|
||||
})
|
||||
state = await this.journal.transition('committed')
|
||||
publishProgress(state)
|
||||
let completedStrategy = strategy
|
||||
let activationError = null
|
||||
if (strategy === 'restart') {
|
||||
state = await this.journal.transition('restarting')
|
||||
publishProgress(state)
|
||||
try {
|
||||
await this.restartDsh()
|
||||
} catch (error) {
|
||||
completedStrategy = 'next-start'
|
||||
activationError = message(error)
|
||||
await this.record('user-plugin.activation.deferred', { taskId, error })
|
||||
}
|
||||
}
|
||||
state = await this.journal.transition('completed')
|
||||
publishProgress(state)
|
||||
await this.snapshots.remove(taskId).catch(error => this.record('user-plugin.snapshot.cleanup.failed', { taskId, error }))
|
||||
await this.record('user-plugin.transaction.completed', { taskId })
|
||||
return Object.freeze({
|
||||
taskId, status: 'success', strategy: completedStrategy, activationError, inventory: result,
|
||||
})
|
||||
} catch (error) {
|
||||
const committed = ['committed', 'restarting'].includes(state.phase)
|
||||
if (committed) {
|
||||
await this.journal.transition('failed', { error: message(error) })
|
||||
await this.snapshots.remove(taskId).catch(cleanupError => this.record('user-plugin.snapshot.cleanup.failed', { taskId, error: cleanupError }))
|
||||
} else if (snapshotCreated || state.snapshotId !== null) {
|
||||
const snapshotId = state.snapshotId ?? taskId
|
||||
try {
|
||||
if (state.phase !== 'restoring') state = await this.journal.transition('restoring', {
|
||||
error: message(error),
|
||||
snapshotId,
|
||||
})
|
||||
await this.snapshots.restore(snapshotId)
|
||||
await this.selectionStore.restore(selectionBefore)
|
||||
if (strategy === 'restart') await this.restartDsh()
|
||||
await this.journal.transition('failed', { recoveryResult: 'success' })
|
||||
await this.snapshots.remove(snapshotId).catch(cleanupError => this.record('user-plugin.snapshot.cleanup.failed', {
|
||||
taskId,
|
||||
error: cleanupError,
|
||||
}))
|
||||
} catch (recoveryError) {
|
||||
const aggregate = new AggregateError([error, recoveryError], 'User Plugin transaction and recovery failed')
|
||||
aggregate.journal = await this.journal.transition('failed', {
|
||||
error: `${message(error)}; recovery failed: ${message(recoveryError)}`,
|
||||
recoveryResult: 'failed',
|
||||
}).catch(() => {})
|
||||
throw aggregate
|
||||
}
|
||||
} else if (paused) {
|
||||
try {
|
||||
await this.restartDsh()
|
||||
await this.journal.transition('failed', { error: message(error), recoveryResult: 'success' })
|
||||
} catch (recoveryError) {
|
||||
const aggregate = new AggregateError([error, recoveryError], 'User Plugin transaction and recovery failed')
|
||||
aggregate.journal = await this.journal.transition('failed', {
|
||||
error: `${message(error)}; recovery failed: ${message(recoveryError)}`,
|
||||
recoveryResult: 'failed',
|
||||
}).catch(() => {})
|
||||
throw aggregate
|
||||
}
|
||||
} else {
|
||||
await this.journal.transition('failed', { error: message(error) }).catch(() => {})
|
||||
}
|
||||
await this.record('user-plugin.transaction.failed', { taskId, error, committed })
|
||||
if (error !== null && typeof error === 'object') error.journal = await this.journal.read().catch(() => undefined)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async recover() {
|
||||
const state = await this.journal.read()
|
||||
if (state === undefined) return undefined
|
||||
if (['completed', 'failed'].includes(state.phase)) {
|
||||
if (state.snapshotId !== null) await this.snapshots.remove(state.snapshotId).catch(error => this.record('user-plugin.snapshot.cleanup.failed', {
|
||||
taskId: state.taskId,
|
||||
error,
|
||||
}))
|
||||
return state
|
||||
}
|
||||
if (['committed', 'restarting'].includes(state.phase)) {
|
||||
try {
|
||||
if (state.phase === 'committed') await this.journal.transition('restarting')
|
||||
await this.restartDsh()
|
||||
const completed = await this.journal.transition('completed')
|
||||
if (state.snapshotId !== null) await this.snapshots.remove(state.snapshotId).catch(error => this.record('user-plugin.snapshot.cleanup.failed', {
|
||||
taskId: state.taskId,
|
||||
error,
|
||||
}))
|
||||
return completed
|
||||
} catch (error) {
|
||||
const failed = await this.journal.transition('failed', { error: message(error) })
|
||||
if (state.snapshotId !== null) {
|
||||
await this.snapshots.remove(state.snapshotId).catch(cleanupError => this.record('user-plugin.snapshot.cleanup.failed', {
|
||||
taskId: state.taskId,
|
||||
error: cleanupError,
|
||||
}))
|
||||
}
|
||||
return failed
|
||||
}
|
||||
}
|
||||
if (state.phase === 'validated') return this.journal.transition('failed', { error: 'transaction interrupted before DSH pause' })
|
||||
await this.pauseDsh()
|
||||
if (state.phase === 'paused') {
|
||||
try {
|
||||
await this.restartDsh()
|
||||
return this.journal.transition('failed', { error: 'transaction interrupted before snapshot', recoveryResult: 'success' })
|
||||
} catch (error) {
|
||||
return this.journal.transition('failed', { error: message(error), recoveryResult: 'failed' })
|
||||
}
|
||||
}
|
||||
try {
|
||||
const restoring = state.phase === 'restoring' ? state : await this.journal.transition('restoring', {
|
||||
error: state.error ?? 'transaction interrupted before commit',
|
||||
})
|
||||
await this.snapshots.restore(restoring.snapshotId)
|
||||
await this.selectionStore.restore({
|
||||
present: restoring.selectionPresent,
|
||||
state: { schema: 1, disabled: restoring.previousDisabled },
|
||||
})
|
||||
await this.restartDsh()
|
||||
const failed = await this.journal.transition('failed', { recoveryResult: 'success' })
|
||||
await this.snapshots.remove(restoring.snapshotId).catch(error => this.record('user-plugin.snapshot.cleanup.failed', {
|
||||
taskId: state.taskId,
|
||||
error,
|
||||
}))
|
||||
return failed
|
||||
} catch (error) {
|
||||
return this.journal.transition('failed', { error: message(error), recoveryResult: 'failed' })
|
||||
}
|
||||
}
|
||||
|
||||
async recoverBeforeDshStart() {
|
||||
return recoverUserPluginBeforeDshStart({
|
||||
journal: this.journal,
|
||||
report: (messageValue, fields) => this.record(messageValue, fields),
|
||||
selectionStore: this.selectionStore,
|
||||
snapshots: this.snapshots,
|
||||
})
|
||||
}
|
||||
|
||||
async completeDshStartup({ healthy, error = null }) {
|
||||
const state = await this.journal.read()
|
||||
if (state === undefined || state.phase !== 'restarting') return state
|
||||
const next = healthy
|
||||
? await this.journal.transition('completed')
|
||||
: await this.journal.transition('failed', { error: error ?? 'DSH failed to start after the committed User Plugin change' })
|
||||
if (state.snapshotId !== null) {
|
||||
await this.snapshots.remove(state.snapshotId).catch(cleanupError => this.record('user-plugin.snapshot.cleanup.failed', {
|
||||
taskId: state.taskId,
|
||||
error: cleanupError,
|
||||
}))
|
||||
}
|
||||
return next
|
||||
}
|
||||
}
|
||||
|
||||
export async function recoverUserPluginBeforeDshStart({ journal, selectionStore, snapshots, report = async () => {} }) {
|
||||
const record = (messageValue, fields) => Promise.resolve()
|
||||
.then(() => report(messageValue, fields))
|
||||
.catch(() => {})
|
||||
let state = await journal.read()
|
||||
if (state === undefined) return undefined
|
||||
if (['completed', 'failed'].includes(state.phase)) {
|
||||
if (state.snapshotId !== null) await snapshots.remove(state.snapshotId).catch(error => record(
|
||||
'user-plugin.snapshot.cleanup.failed', { taskId: state.taskId, error },
|
||||
))
|
||||
return state
|
||||
}
|
||||
if (state.phase === 'validated') {
|
||||
return journal.transition('failed', {
|
||||
error: 'transaction interrupted before DSH pause', recoveryResult: 'success',
|
||||
})
|
||||
}
|
||||
if (state.phase === 'paused') {
|
||||
return journal.transition('failed', {
|
||||
error: 'transaction interrupted before snapshot', recoveryResult: 'success',
|
||||
})
|
||||
}
|
||||
if (['committed', 'restarting'].includes(state.phase)) {
|
||||
if (state.phase === 'committed') state = await journal.transition('restarting')
|
||||
return state
|
||||
}
|
||||
try {
|
||||
if (state.phase !== 'restoring') state = await journal.transition('restoring', {
|
||||
error: state.error ?? 'transaction interrupted before commit',
|
||||
})
|
||||
await snapshots.restore(state.snapshotId)
|
||||
await selectionStore.restore({
|
||||
present: state.selectionPresent,
|
||||
state: { schema: 1, disabled: state.previousDisabled },
|
||||
})
|
||||
const failed = await journal.transition('failed', { recoveryResult: 'success' })
|
||||
await snapshots.remove(state.snapshotId).catch(error => record(
|
||||
'user-plugin.snapshot.cleanup.failed', { taskId: state.taskId, error },
|
||||
))
|
||||
return failed
|
||||
} catch (error) {
|
||||
return journal.transition('failed', { error: message(error), recoveryResult: 'failed' })
|
||||
}
|
||||
}
|
||||
|
||||
export const userPluginTransactionInternals = Object.freeze({ mutateProfile, pluginEnvironment, runDshPlugin, validateActions })
|
||||
@@ -0,0 +1,297 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { spawn } from 'node:child_process'
|
||||
import { cp, lstat, mkdir, readFile, readdir, readlink, rename, rm, symlink, writeFile } from 'node:fs/promises'
|
||||
import { dirname, join, resolve } from 'node:path'
|
||||
import { canonicalJson } from '../../../platform/lib/canonical-json.mjs'
|
||||
import { parseEnvironmentManifest } from '../../../platform/lib/contracts.mjs'
|
||||
import { hashTree } from '../../../platform/lib/tree-hash.mjs'
|
||||
|
||||
const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/
|
||||
const STATE_SCHEMA = 1
|
||||
export const SYSTEM_SKILL_CATALOG_ARTIFACT_ID = 'system-skill-catalog'
|
||||
export const SYSTEM_SKILL_CATALOG_MEDIA_TYPE = 'application/vnd.dsh-platform.system-skill-catalog.v1+tar+gzip'
|
||||
|
||||
function runTar(args, { capture = false } = {}) {
|
||||
return new Promise((resolveTar, reject) => {
|
||||
const child = spawn('tar', args, { stdio: ['ignore', capture ? 'pipe' : 'ignore', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout?.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolveTar(Buffer.concat(stdout).toString('utf8'))
|
||||
: reject(new Error(`System Skill archive failed: ${Buffer.concat(stderr).toString('utf8')}`)))
|
||||
})
|
||||
}
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
export async function retainLegacySystemSkillCatalog({ sourceRoot, outputRoot }) {
|
||||
const source = resolve(sourceRoot)
|
||||
const destination = join(resolve(outputRoot), 'legacy-bootstrap')
|
||||
if (await exists(join(destination, 'catalog.json'))) {
|
||||
await readSystemSkillCatalog(destination)
|
||||
return destination
|
||||
}
|
||||
await readSystemSkillCatalog(source)
|
||||
const staging = `${destination}.${randomUUID()}.tmp`
|
||||
try {
|
||||
await cp(source, staging, { recursive: true, verbatimSymlinks: true })
|
||||
try {
|
||||
await rename(staging, destination)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
|
||||
}
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
await readSystemSkillCatalog(destination)
|
||||
return destination
|
||||
}
|
||||
|
||||
export async function materializeSystemSkillCatalog({ environmentRoot, outputRoot }) {
|
||||
const root = resolve(environmentRoot)
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(root, 'environment.manifest.json')))
|
||||
const artifact = manifest.artifacts.find(entry => entry.id === SYSTEM_SKILL_CATALOG_ARTIFACT_ID)
|
||||
if (artifact === undefined) return undefined
|
||||
if (artifact.mediaType !== SYSTEM_SKILL_CATALOG_MEDIA_TYPE) {
|
||||
throw new Error('System Skill catalog Artifact media type is invalid')
|
||||
}
|
||||
const archive = join(root, 'artifacts', artifact.id)
|
||||
const bytes = await readFile(archive)
|
||||
if (bytes.byteLength !== artifact.size || createHash('sha256').update(bytes).digest('hex') !== artifact.sha256) {
|
||||
throw new Error('System Skill catalog Artifact differs from its Environment manifest')
|
||||
}
|
||||
const destination = join(resolve(outputRoot), artifact.sha256)
|
||||
if (!await exists(destination)) {
|
||||
const staging = `${destination}.${randomUUID()}.tmp`
|
||||
await mkdir(staging, { recursive: true })
|
||||
try {
|
||||
const entries = (await runTar(['-tzf', archive], { capture: true })).split('\n').filter(Boolean)
|
||||
if (entries.length === 0 || entries.some(entry => (
|
||||
entry.startsWith('/') || !entry.startsWith('skills/') || entry.split('/').includes('..')
|
||||
))) throw new Error('System Skill catalog archive contains an unsafe path')
|
||||
await runTar(['-xzf', archive, '--no-same-owner', '--no-same-permissions', '--strip-components=1', '-C', staging])
|
||||
await readSystemSkillCatalog(staging)
|
||||
try {
|
||||
await rename(staging, destination)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
|
||||
}
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
await readSystemSkillCatalog(destination)
|
||||
return destination
|
||||
}
|
||||
|
||||
function exactKeys(value, keys, label) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) {
|
||||
throw new Error(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function skillId(value) {
|
||||
if (typeof value !== 'string' || !ID_PATTERN.test(value) || value.length > 64) {
|
||||
throw new Error('System Skill ID is invalid')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function description(value, label) {
|
||||
exactKeys(value, ['en', 'zh'], label)
|
||||
for (const language of ['en', 'zh']) {
|
||||
if (typeof value[language] !== 'string' || value[language].length === 0 || value[language].length > 240) {
|
||||
throw new Error(`${label}.${language} is invalid`)
|
||||
}
|
||||
}
|
||||
return Object.freeze({ en: value.en, zh: value.zh })
|
||||
}
|
||||
|
||||
export async function readSystemSkillCatalog(root) {
|
||||
const sourceRoot = resolve(root)
|
||||
const document = JSON.parse(await readFile(join(sourceRoot, 'catalog.json'), 'utf8'))
|
||||
exactKeys(document, ['schema', 'skills'], 'System Skill catalog')
|
||||
if (document.schema !== 1 || !Array.isArray(document.skills)) throw new Error('System Skill catalog is invalid')
|
||||
const ids = new Set()
|
||||
const skills = []
|
||||
for (const entry of document.skills) {
|
||||
exactKeys(entry, ['description', 'id', 'source'], 'System Skill catalog entry')
|
||||
const id = skillId(entry.id)
|
||||
if (ids.has(id)) throw new Error(`duplicate System Skill ID ${id}`)
|
||||
ids.add(id)
|
||||
if (typeof entry.source !== 'string' || entry.source !== id) throw new Error(`System Skill ${id} source is invalid`)
|
||||
const source = resolve(sourceRoot, entry.source)
|
||||
if (!source.startsWith(`${sourceRoot}/`)) throw new Error(`System Skill ${id} source escapes its catalog`)
|
||||
const metadata = await lstat(join(source, 'SKILL.md'))
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink()) throw new Error(`System Skill ${id} SKILL.md is invalid`)
|
||||
const bytes = await readFile(join(source, 'SKILL.md'), 'utf8')
|
||||
if (!bytes.startsWith('---\n') || !bytes.includes(`\nname: ${id}\n`)) {
|
||||
throw new Error(`System Skill ${id} frontmatter does not match its catalog`)
|
||||
}
|
||||
skills.push(Object.freeze({
|
||||
id,
|
||||
source,
|
||||
sha256: await hashTree(source),
|
||||
description: description(entry.description, `System Skill ${id} description`),
|
||||
}))
|
||||
}
|
||||
return Object.freeze(skills)
|
||||
}
|
||||
|
||||
function defaults(catalog) {
|
||||
return Object.fromEntries(catalog.map(skill => [skill.id, { installed: true, enabled: true }]))
|
||||
}
|
||||
|
||||
function normalizeState(value, catalog) {
|
||||
const fallback = defaults(catalog)
|
||||
if (value === undefined) return fallback
|
||||
exactKeys(value, ['schema', 'skills'], 'System Skill selection state')
|
||||
if (value.schema !== STATE_SCHEMA || value.skills === null || typeof value.skills !== 'object' || Array.isArray(value.skills)) {
|
||||
throw new Error('System Skill selection state is invalid')
|
||||
}
|
||||
for (const skill of catalog) {
|
||||
const saved = value.skills[skill.id]
|
||||
if (saved === undefined) continue
|
||||
exactKeys(saved, ['enabled', 'installed'], `System Skill ${skill.id} selection`)
|
||||
if (typeof saved.installed !== 'boolean' || typeof saved.enabled !== 'boolean' || (saved.enabled && !saved.installed)) {
|
||||
throw new Error(`System Skill ${skill.id} selection is invalid`)
|
||||
}
|
||||
fallback[skill.id] = { installed: saved.installed, enabled: saved.enabled }
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
export class SystemSkillSelectionStore {
|
||||
constructor(path) {
|
||||
this.path = resolve(path)
|
||||
}
|
||||
|
||||
async read(catalog) {
|
||||
let value
|
||||
try {
|
||||
value = JSON.parse(await readFile(this.path, 'utf8'))
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw error
|
||||
}
|
||||
return normalizeState(value, catalog)
|
||||
}
|
||||
|
||||
async write(catalog, state) {
|
||||
const normalized = normalizeState({ schema: STATE_SCHEMA, skills: state }, catalog)
|
||||
const document = {
|
||||
schema: STATE_SCHEMA,
|
||||
skills: Object.fromEntries(catalog.map(skill => [skill.id, normalized[skill.id]])),
|
||||
}
|
||||
await mkdir(dirname(this.path), { recursive: true })
|
||||
const temporary = `${this.path}.${randomUUID()}.tmp`
|
||||
try {
|
||||
await writeFile(temporary, `${canonicalJson(document)}\n`, { flag: 'wx', mode: 0o600 })
|
||||
await rename(temporary, this.path)
|
||||
} finally {
|
||||
await rm(temporary, { force: true })
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
}
|
||||
|
||||
async function replaceSkillLink(viewRoot, skill) {
|
||||
const path = join(viewRoot, skill.id)
|
||||
const existing = await lstat(path).catch(error => error?.code === 'ENOENT' ? undefined : Promise.reject(error))
|
||||
if (existing?.isSymbolicLink() && await readlink(path) === skill.source) return
|
||||
const temporary = join(viewRoot, `.${skill.id}.${randomUUID()}.tmp`)
|
||||
try {
|
||||
await symlink(skill.source, temporary, 'dir')
|
||||
await rename(temporary, path)
|
||||
} finally {
|
||||
await rm(temporary, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export async function synchronizeSystemSkillView({ catalog, selection, viewRoot }) {
|
||||
const root = resolve(viewRoot)
|
||||
await mkdir(root, { recursive: true })
|
||||
const expected = new Set(catalog.filter(skill => selection[skill.id].enabled).map(skill => skill.id))
|
||||
for (const entry of await readdir(root, { withFileTypes: true })) {
|
||||
if (!expected.has(entry.name)) await rm(join(root, entry.name), { recursive: true, force: true })
|
||||
}
|
||||
for (const skill of catalog) {
|
||||
if (selection[skill.id].enabled) await replaceSkillLink(root, skill)
|
||||
}
|
||||
}
|
||||
|
||||
export class SystemSkillManager {
|
||||
constructor({ sourceRoot, viewRoot, statePath }) {
|
||||
this.sourceRoot = resolve(sourceRoot)
|
||||
this.viewRoot = resolve(viewRoot)
|
||||
this.store = new SystemSkillSelectionStore(statePath)
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
const catalog = await readSystemSkillCatalog(this.sourceRoot)
|
||||
const selection = await this.store.read(catalog)
|
||||
await this.store.write(catalog, selection)
|
||||
await synchronizeSystemSkillView({ catalog, selection, viewRoot: this.viewRoot })
|
||||
return this.listFrom(catalog, selection)
|
||||
}
|
||||
|
||||
async useSourceRoot(sourceRoot) {
|
||||
const nextRoot = resolve(sourceRoot)
|
||||
const catalog = await readSystemSkillCatalog(nextRoot)
|
||||
const selection = await this.store.read(catalog)
|
||||
await this.store.write(catalog, selection)
|
||||
await synchronizeSystemSkillView({ catalog, selection, viewRoot: this.viewRoot })
|
||||
this.sourceRoot = nextRoot
|
||||
return this.listFrom(catalog, selection)
|
||||
}
|
||||
|
||||
listFrom(catalog, selection) {
|
||||
return Object.freeze(catalog.map(skill => Object.freeze({
|
||||
id: skill.id,
|
||||
sha256: skill.sha256,
|
||||
description: skill.description,
|
||||
installed: selection[skill.id].installed,
|
||||
enabled: selection[skill.id].enabled,
|
||||
})))
|
||||
}
|
||||
|
||||
async list() {
|
||||
const catalog = await readSystemSkillCatalog(this.sourceRoot)
|
||||
return this.listFrom(catalog, await this.store.read(catalog))
|
||||
}
|
||||
|
||||
async configure(idValue, action) {
|
||||
const id = skillId(idValue)
|
||||
if (!['install', 'uninstall', 'enable', 'disable'].includes(action)) throw new Error('System Skill action is invalid')
|
||||
const catalog = await readSystemSkillCatalog(this.sourceRoot)
|
||||
if (!catalog.some(skill => skill.id === id)) throw new Error(`System Skill ${id} is not provided by the current Environment`)
|
||||
const before = await this.store.read(catalog)
|
||||
const next = structuredClone(before)
|
||||
const current = next[id]
|
||||
if (action === 'install') next[id] = { installed: true, enabled: true }
|
||||
else if (action === 'uninstall') next[id] = { installed: false, enabled: false }
|
||||
else if (action === 'enable') {
|
||||
if (!current.installed) throw new Error(`System Skill ${id} must be installed before it can be enabled`)
|
||||
next[id] = { ...current, enabled: true }
|
||||
} else next[id] = { ...current, enabled: false }
|
||||
try {
|
||||
await this.store.write(catalog, next)
|
||||
await synchronizeSystemSkillView({ catalog, selection: next, viewRoot: this.viewRoot })
|
||||
return this.listFrom(catalog, next)
|
||||
} catch (error) {
|
||||
try {
|
||||
await this.store.write(catalog, before)
|
||||
await synchronizeSystemSkillView({ catalog, selection: before, viewRoot: this.viewRoot })
|
||||
} catch (rollbackError) {
|
||||
throw new AggregateError([error, rollbackError], 'System Skill operation and rollback failed')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,229 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { cp, lstat, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { RuntimeSlots } from '../../patch-manager/index.mjs'
|
||||
import { PlatformPaths } from '../../../../platform/lib/paths.mjs'
|
||||
import { LocalApiClient } from './client.mjs'
|
||||
import { ManagedDeploymentBuilder } from './managed-store.mjs'
|
||||
import { deriveRecordId, parseDeploymentRecord } from '../../../../platform/lib/deployment-contracts.mjs'
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolve(Buffer.concat(stdout))
|
||||
: reject(new Error(Buffer.concat(stderr).toString('utf8'))))
|
||||
})
|
||||
}
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
function matchesRecoveryJournal(record, deployment) {
|
||||
if (record?.id === deployment.runtime) return true
|
||||
if (record === null || record === undefined) return false
|
||||
if (!Array.isArray(record.receiptTokens) || !Array.isArray(deployment.receiptTokens)) return false
|
||||
const sameReceipts = [...record.receiptTokens].sort().join('\0')
|
||||
=== [...deployment.receiptTokens].sort().join('\0')
|
||||
return record.dshVersion === deployment.dsh
|
||||
&& record.environmentVersion === deployment.environment
|
||||
&& record.snapshotId === deployment.dataSnapshot
|
||||
&& sameReceipts
|
||||
}
|
||||
|
||||
export class PlatformActivator {
|
||||
constructor({
|
||||
dataRoot,
|
||||
runRoot = process.env.DSH_PLATFORM_RUN ?? '/run/dsh-platform',
|
||||
bootstrap,
|
||||
stage0,
|
||||
builder,
|
||||
sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)),
|
||||
}) {
|
||||
this.dataRoot = dataRoot
|
||||
this.paths = new PlatformPaths(dataRoot, runRoot)
|
||||
this.bootstrap = bootstrap ?? new LocalApiClient(this.paths.bootstrapSocket)
|
||||
this.stage0 = stage0 ?? new LocalApiClient(this.paths.trustSocket)
|
||||
this.builder = builder ?? new ManagedDeploymentBuilder({ paths: this.paths })
|
||||
this.sleep = sleep
|
||||
this.runtimeSlots = new RuntimeSlots(this.paths.runtimesRoot)
|
||||
this.environmentSlots = new RuntimeSlots(this.paths.environmentsRoot)
|
||||
this.systemPluginSlots = new RuntimeSlots(this.paths.systemPluginsRoot)
|
||||
this.experimentalCandidates = new Map()
|
||||
}
|
||||
|
||||
prepareManaged(prepared, options) {
|
||||
return this.builder.buildStable(prepared, options)
|
||||
}
|
||||
|
||||
async pristine(version, receipt) {
|
||||
const root = join(this.paths.pristineRoot, version)
|
||||
if (!await exists(root)) {
|
||||
const staging = `${root}.${randomUUID()}.tmp`
|
||||
await mkdir(staging, { recursive: true })
|
||||
try {
|
||||
const entries = (await run('tar', ['-tzf', receipt.path])).toString('utf8').split('\n').filter(Boolean)
|
||||
if (!entries.includes('package/package.json')) throw new Error('official DSH has no package metadata')
|
||||
if (entries.some(entry => entry.startsWith('/') || !entry.startsWith('package/') || entry.split('/').includes('..'))) {
|
||||
throw new Error('official DSH archive contains an unsafe path')
|
||||
}
|
||||
await run('tar', ['-xzf', receipt.path, '-C', staging])
|
||||
await rename(staging, root)
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
const packageMetadata = JSON.parse(await readFile(join(root, 'package', 'package.json'), 'utf8'))
|
||||
if (packageMetadata.name !== '@deepseek-ai/dsh' || packageMetadata.version !== version) {
|
||||
throw new Error('official DSH package metadata differs from its requested version')
|
||||
}
|
||||
return join(root, 'package')
|
||||
}
|
||||
|
||||
async stageEnvironment(prepared) {
|
||||
const version = prepared.environment.manifest.version
|
||||
const versions = join(this.paths.environmentsRoot, 'versions')
|
||||
const destination = join(versions, version)
|
||||
if (await exists(destination)) return version
|
||||
const staging = join(versions, `.${version}.${randomUUID()}.tmp`)
|
||||
await mkdir(join(staging, 'artifacts'), { recursive: true })
|
||||
try {
|
||||
await cp(prepared.paths.get(prepared.stable.desired.environment.manifestArtifactId), join(staging, 'environment.manifest.json'))
|
||||
for (const descriptor of prepared.environment.manifest.artifacts) {
|
||||
await cp(prepared.paths.get(descriptor.id), join(staging, 'artifacts', descriptor.id))
|
||||
}
|
||||
await rename(staging, destination)
|
||||
return version
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async activate(prepared, { onProgress = async () => {}, onSwitching = async () => {} } = {}) {
|
||||
const desiredBootstrap = prepared.stable.desired.bootstrap.version
|
||||
const artifacts = prepared.bootstrap.manifest.artifacts
|
||||
if (artifacts.length !== 1) throw new Error('Bootstrap manifest must contain exactly one package')
|
||||
const receipt = prepared.receipts.get(artifacts[0].id)
|
||||
if (receipt === undefined) throw new Error('Bootstrap package receipt is required')
|
||||
const staged = await this.stage0.stageBootstrap(receipt.token, desiredBootstrap)
|
||||
// Older Stage-0 responses have only status=switching. Preserve that handoff
|
||||
// contract while allowing content-identical targets to continue in place.
|
||||
if (staged.restartRequired === true || staged.status === 'switching') await new Promise(() => {})
|
||||
const managed = await this.prepareManaged(prepared, { onProgress })
|
||||
await onSwitching()
|
||||
await this.bootstrap.request('POST', '/v1/deployments/activate', { record: managed.record })
|
||||
return managed.record
|
||||
}
|
||||
|
||||
async rollback(recordId) {
|
||||
return this.bootstrap.request('POST', '/v1/deployments/rollback', recordId === undefined ? {} : { recordId })
|
||||
}
|
||||
|
||||
rollbackDeployments() {
|
||||
return this.bootstrap.request('GET', '/v1/deployments/rollback-plan')
|
||||
}
|
||||
|
||||
async currentDeployment({ required = true } = {}) {
|
||||
let record
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
({ record } = await this.bootstrap.request('GET', '/v1/deployments/current'))
|
||||
if (record !== null || !required) break
|
||||
const status = await this.bootstrap.request('GET', '/v1/status')
|
||||
if (status.startupComplete === true) break
|
||||
await this.sleep(100)
|
||||
}
|
||||
if (record === null) {
|
||||
if (required) throw new Error('current Deployment is required')
|
||||
return null
|
||||
}
|
||||
return Object.freeze({
|
||||
authority: record.authority,
|
||||
targetSequence: record.targetSequence,
|
||||
dsh: record.dshVersion,
|
||||
runtime: record.id,
|
||||
environment: record.environmentVersion,
|
||||
dataSnapshot: record.snapshotId,
|
||||
receiptTokens: Object.freeze([...record.receiptTokens]),
|
||||
})
|
||||
}
|
||||
|
||||
async prepareExperimental(prepared, { targetSequence }) {
|
||||
const { record: current } = await this.bootstrap.request('GET', '/v1/deployments/current')
|
||||
if (current === null) throw new Error('current Deployment is required')
|
||||
const receiptTokens = await this.experimentalActivationTokens(prepared.receiptTokens)
|
||||
const built = await this.builder.buildExperimental(prepared, current, receiptTokens, { targetSequence })
|
||||
this.experimentalCandidates.set(built.record.id, built.record)
|
||||
return Object.freeze({
|
||||
runtimeId: built.record.id,
|
||||
environmentVersion: built.record.environmentVersion,
|
||||
dshVersion: built.record.dshVersion,
|
||||
})
|
||||
}
|
||||
|
||||
suspendDsh() { return this.bootstrap.request('POST', '/v1/components/dsh-runtime/suspend') }
|
||||
resumeDsh() { return this.bootstrap.request('POST', '/v1/components/dsh-runtime/resume') }
|
||||
health() { return this.bootstrap.request('GET', '/v1/health') }
|
||||
|
||||
async experimentalActivationTokens(experimentalTokens) {
|
||||
const active = (await this.stage0.activeReceipts()).receipts
|
||||
return Object.freeze([
|
||||
...active.filter(receipt => receipt.authorityType === 'stable').map(receipt => receipt.token),
|
||||
...experimentalTokens,
|
||||
])
|
||||
}
|
||||
|
||||
async switchExperimental(runtimeId) {
|
||||
const record = this.experimentalCandidates.get(runtimeId)
|
||||
if (record === undefined) throw new Error('Experimental Deployment candidate is not prepared')
|
||||
return this.bootstrap.request('POST', '/v1/deployments/candidate', { record })
|
||||
}
|
||||
|
||||
async bindExperimentalSnapshot(runtimeId, snapshotId) {
|
||||
const record = this.experimentalCandidates.get(runtimeId)
|
||||
if (record === undefined) throw new Error('Experimental Deployment candidate is not prepared')
|
||||
const content = { ...record, snapshotId }
|
||||
delete content.id
|
||||
const bound = parseDeploymentRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
this.experimentalCandidates.delete(runtimeId)
|
||||
this.experimentalCandidates.set(bound.id, bound)
|
||||
return bound.id
|
||||
}
|
||||
|
||||
async commitExperimental(runtimeId) {
|
||||
const result = await this.bootstrap.request('POST', '/v1/deployments/candidate/commit', { recordId: runtimeId })
|
||||
this.experimentalCandidates.delete(runtimeId)
|
||||
return result
|
||||
}
|
||||
|
||||
async restoreDeployment(deployment, { resume = true } = {}) {
|
||||
await this.bootstrap.request('POST', '/v1/deployments/candidate/cancel')
|
||||
const { record } = await this.bootstrap.request('GET', '/v1/deployments/current')
|
||||
if (!matchesRecoveryJournal(record, deployment)) {
|
||||
const { previous } = await this.rollbackDeployments()
|
||||
if (previous === null) throw new Error('previous Deployment is unavailable')
|
||||
if (!matchesRecoveryJournal(previous, deployment)) {
|
||||
throw new Error('previous Deployment differs from the recovery journal')
|
||||
}
|
||||
await this.bootstrap.request('POST', '/v1/deployments/rollback', { recordId: previous.id })
|
||||
}
|
||||
if (resume) await this.resumeDsh()
|
||||
}
|
||||
|
||||
async cleanup() {
|
||||
const platform = await this.bootstrap.request('POST', '/v1/platform/gc')
|
||||
return Object.freeze({
|
||||
platform,
|
||||
bootstrap: await this.stage0.collectBootstrap(),
|
||||
trustedObjects: (await this.stage0.collectGarbage()).removed,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { durableReplace } from '../../../../platform/lib/atomic.mjs'
|
||||
import { compareDshVersions } from '../../../../platform/lib/supported-target.mjs'
|
||||
|
||||
export const AUTOMATIC_CHECK_INTERVALS = Object.freeze([3_600, 10_800, 21_600, 43_200, 86_400])
|
||||
|
||||
const DEFAULT_STATE = Object.freeze({
|
||||
schema: 1,
|
||||
automaticCheck: Object.freeze({
|
||||
enabled: true,
|
||||
intervalSeconds: 21_600,
|
||||
notificationsEnabled: true,
|
||||
}),
|
||||
latestAutomatic: Object.freeze({ stable: null, upstream: null }),
|
||||
})
|
||||
|
||||
function exactKeys(value, expected, label) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)) throw new Error(`${label} must be an object`)
|
||||
const keys = Object.keys(value).sort()
|
||||
const wanted = [...expected].sort()
|
||||
if (keys.length !== wanted.length || keys.some((key, index) => key !== wanted[index])) {
|
||||
throw new Error(`${label} fields are invalid`)
|
||||
}
|
||||
}
|
||||
|
||||
function parseConfig(value) {
|
||||
exactKeys(value, ['enabled', 'intervalSeconds', 'notificationsEnabled'], 'automatic check config')
|
||||
if (typeof value.enabled !== 'boolean') throw new Error('automatic check enabled must be boolean')
|
||||
if (!AUTOMATIC_CHECK_INTERVALS.includes(value.intervalSeconds)) throw new Error('automatic check interval is invalid')
|
||||
if (typeof value.notificationsEnabled !== 'boolean') throw new Error('update notifications enabled must be boolean')
|
||||
return Object.freeze({ ...value })
|
||||
}
|
||||
|
||||
function parseStable(value) {
|
||||
if (value === null) return null
|
||||
exactKeys(value, ['checkedAt', 'dsh', 'environment', 'targetSequence'], 'Stable notification')
|
||||
if (!Number.isSafeInteger(value.targetSequence) || value.targetSequence < 1) throw new Error('Stable notification sequence is invalid')
|
||||
compareDshVersions(value.dsh, value.dsh)
|
||||
if (typeof value.environment !== 'string' || value.environment.length === 0) throw new Error('Stable notification Environment is invalid')
|
||||
if (Number.isNaN(Date.parse(value.checkedAt))) throw new Error('Stable notification timestamp is invalid')
|
||||
return Object.freeze({ ...value })
|
||||
}
|
||||
|
||||
function parseUpstream(value) {
|
||||
if (value === null) return null
|
||||
exactKeys(value, ['checkedAt', 'version'], 'Upstream notification')
|
||||
compareDshVersions(value.version, value.version)
|
||||
if (Number.isNaN(Date.parse(value.checkedAt))) throw new Error('Upstream notification timestamp is invalid')
|
||||
return Object.freeze({ ...value })
|
||||
}
|
||||
|
||||
function parseState(value) {
|
||||
exactKeys(value, ['automaticCheck', 'latestAutomatic', 'schema'], 'automatic check state')
|
||||
if (value.schema !== 1) throw new Error('automatic check state schema must be 1')
|
||||
exactKeys(value.latestAutomatic, ['stable', 'upstream'], 'automatic notification state')
|
||||
return Object.freeze({
|
||||
schema: 1,
|
||||
automaticCheck: parseConfig(value.automaticCheck),
|
||||
latestAutomatic: Object.freeze({
|
||||
stable: parseStable(value.latestAutomatic.stable),
|
||||
upstream: parseUpstream(value.latestAutomatic.upstream),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
function isHeld(version, environment, holds) {
|
||||
return holds.some(hold => hold.dshVersion === version
|
||||
&& (hold.type !== 'combination' || hold.environmentVersion === environment))
|
||||
}
|
||||
|
||||
function stableSatisfied(candidate, current) {
|
||||
return current.environment === candidate.environment && compareDshVersions(current.dsh, candidate.dsh) >= 0
|
||||
}
|
||||
|
||||
export class AutomaticCheckStateStore {
|
||||
constructor(path, now = () => new Date()) {
|
||||
this.path = path
|
||||
this.now = now
|
||||
this.pending = Promise.resolve()
|
||||
}
|
||||
|
||||
async read() {
|
||||
try {
|
||||
return parseState(JSON.parse(await readFile(this.path, 'utf8')))
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return DEFAULT_STATE
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async replace(update) {
|
||||
const operation = this.pending.then(async () => {
|
||||
const next = parseState(await update(await this.read()))
|
||||
await durableReplace(this.path, `${JSON.stringify(next)}\n`)
|
||||
return next
|
||||
})
|
||||
this.pending = operation.catch(() => {})
|
||||
return operation
|
||||
}
|
||||
|
||||
configure(value) {
|
||||
const config = parseConfig(value)
|
||||
return this.replace(previous => ({
|
||||
...previous,
|
||||
automaticCheck: config,
|
||||
latestAutomatic: !config.enabled || !config.notificationsEnabled
|
||||
? { stable: null, upstream: null }
|
||||
: previous.latestAutomatic,
|
||||
}))
|
||||
}
|
||||
|
||||
record({ channel, current, target, upstream, stableAvailable, holds = [] }) {
|
||||
return this.replace(previous => {
|
||||
if (!previous.automaticCheck.enabled || !previous.automaticCheck.notificationsEnabled) return previous
|
||||
const checkedAt = this.now().toISOString()
|
||||
let stable = previous.latestAutomatic.stable
|
||||
let latestUpstream = previous.latestAutomatic.upstream
|
||||
|
||||
if (stable !== null && stableSatisfied(stable, current)) stable = null
|
||||
if (
|
||||
stableAvailable === true
|
||||
&& (stable === null || target.targetSequence > stable.targetSequence)
|
||||
) {
|
||||
stable = {
|
||||
targetSequence: target.targetSequence,
|
||||
dsh: target.desired.dsh.version,
|
||||
environment: target.desired.environment.version,
|
||||
checkedAt,
|
||||
}
|
||||
}
|
||||
|
||||
if (channel !== 'experimental') {
|
||||
latestUpstream = null
|
||||
} else {
|
||||
if (latestUpstream !== null && compareDshVersions(current.dsh, latestUpstream.version) >= 0) latestUpstream = null
|
||||
if (
|
||||
upstream !== null
|
||||
&& compareDshVersions(upstream.version, current.dsh) > 0
|
||||
&& !isHeld(upstream.version, target.desired.environment.version, holds)
|
||||
&& (latestUpstream === null || compareDshVersions(upstream.version, latestUpstream.version) > 0)
|
||||
) latestUpstream = { version: upstream.version, checkedAt }
|
||||
}
|
||||
|
||||
return { ...previous, latestAutomatic: { stable, upstream: latestUpstream } }
|
||||
})
|
||||
}
|
||||
|
||||
clearSatisfied(current) {
|
||||
return this.replace(previous => ({
|
||||
...previous,
|
||||
latestAutomatic: {
|
||||
stable: previous.latestAutomatic.stable !== null
|
||||
&& stableSatisfied(previous.latestAutomatic.stable, current)
|
||||
? null
|
||||
: previous.latestAutomatic.stable,
|
||||
upstream: previous.latestAutomatic.upstream !== null
|
||||
&& compareDshVersions(current.dsh, previous.latestAutomatic.upstream.version) >= 0
|
||||
? null
|
||||
: previous.latestAutomatic.upstream,
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
clearUpstream() {
|
||||
return this.replace(previous => ({
|
||||
...previous,
|
||||
latestAutomatic: { ...previous.latestAutomatic, upstream: null },
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { durableReplace } from '../../../../platform/lib/atomic.mjs'
|
||||
import { compareDshVersions } from '../../../../platform/lib/supported-target.mjs'
|
||||
import { isoTimestamp, plainObject, TrustError } from '../../../../platform/lib/validation.mjs'
|
||||
|
||||
const DEFAULT_STATE = Object.freeze({
|
||||
schema: 1,
|
||||
updateChannel: 'stable',
|
||||
holds: Object.freeze([]),
|
||||
experimentalBlocked: null,
|
||||
})
|
||||
|
||||
function text(value, label) {
|
||||
if (typeof value !== 'string' || value === '' || value.length > 512 || value.includes('\0')) {
|
||||
throw new TrustError(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function parseHold(value) {
|
||||
const hold = plainObject(value, 'update hold')
|
||||
const keys = Object.keys(hold).sort()
|
||||
const expected = ['createdAt', 'dshVersion', 'environmentVersion', 'id', 'reason', 'type']
|
||||
if (JSON.stringify(keys) !== JSON.stringify(expected)) throw new TrustError('update hold has unexpected fields')
|
||||
if (!['version', 'combination'].includes(hold.type)) throw new TrustError('update hold type is invalid')
|
||||
if (hold.type === 'version' && hold.environmentVersion !== null) throw new TrustError('version hold cannot name an Environment')
|
||||
if (hold.type === 'combination' && typeof hold.environmentVersion !== 'string') {
|
||||
throw new TrustError('combination hold requires an Environment')
|
||||
}
|
||||
isoTimestamp(hold.createdAt, 'update hold creation time')
|
||||
return Object.freeze({
|
||||
id: text(hold.id, 'update hold ID'),
|
||||
type: hold.type,
|
||||
dshVersion: text(hold.dshVersion, 'update hold DSH version'),
|
||||
environmentVersion: hold.environmentVersion === null ? null : text(hold.environmentVersion, 'update hold Environment'),
|
||||
reason: text(hold.reason, 'update hold reason'),
|
||||
createdAt: hold.createdAt,
|
||||
})
|
||||
}
|
||||
|
||||
function parseState(value) {
|
||||
const state = plainObject(value, 'channel state')
|
||||
const keys = Object.keys(state).sort()
|
||||
if (JSON.stringify(keys) !== JSON.stringify(['experimentalBlocked', 'holds', 'schema', 'updateChannel'])) {
|
||||
throw new TrustError('channel state has unexpected fields')
|
||||
}
|
||||
if (state.schema !== 1) throw new TrustError('channel state schema must be 1')
|
||||
if (!['stable', 'experimental'].includes(state.updateChannel)) throw new TrustError('update channel is invalid')
|
||||
if (!Array.isArray(state.holds)) throw new TrustError('channel holds must be an array')
|
||||
const holds = state.holds.map(parseHold)
|
||||
if (new Set(holds.map(hold => hold.id)).size !== holds.length) throw new TrustError('update hold IDs must be unique')
|
||||
const blocked = state.experimentalBlocked === null ? null : parseHold(state.experimentalBlocked)
|
||||
if (blocked !== null && blocked.type !== 'combination') throw new TrustError('Experimental block must identify a combination')
|
||||
return Object.freeze({ schema: 1, updateChannel: state.updateChannel, holds: Object.freeze(holds), experimentalBlocked: blocked })
|
||||
}
|
||||
|
||||
export class ChannelStateStore {
|
||||
constructor(path, now = () => new Date()) {
|
||||
this.path = path
|
||||
this.now = now
|
||||
this.queue = Promise.resolve()
|
||||
}
|
||||
|
||||
exclusive(operation) {
|
||||
const result = this.queue.then(operation, operation)
|
||||
this.queue = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
async read() {
|
||||
try { return parseState(JSON.parse(await readFile(this.path, 'utf8'))) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return DEFAULT_STATE
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
write(value) { return durableReplace(this.path, `${JSON.stringify(parseState(value))}\n`) }
|
||||
|
||||
setChannel(updateChannel) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
const next = parseState({ ...current, updateChannel })
|
||||
await this.write(next)
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
addHold({ type, dshVersion, environmentVersion = null, reason }) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
const identity = `${type}\0${dshVersion}\0${environmentVersion ?? ''}`
|
||||
const id = createHash('sha256').update(identity).digest('hex').slice(0, 24)
|
||||
const hold = parseHold({
|
||||
id, type, dshVersion, environmentVersion, reason,
|
||||
createdAt: this.now().toISOString(),
|
||||
})
|
||||
const holds = [...current.holds.filter(item => item.id !== id), hold]
|
||||
const next = parseState({ ...current, holds })
|
||||
await this.write(next)
|
||||
return hold
|
||||
})
|
||||
}
|
||||
|
||||
retry(id) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
const holds = current.holds.filter(hold => hold.id !== id)
|
||||
const clearsBlock = current.experimentalBlocked?.id === id
|
||||
if (holds.length === current.holds.length && !clearsBlock) throw new TrustError('update hold does not exist')
|
||||
const next = parseState({ ...current, holds, experimentalBlocked: clearsBlock ? null : current.experimentalBlocked })
|
||||
await this.write(next)
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
block({ dshVersion, environmentVersion, reason }) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
const blocked = parseHold({
|
||||
id: `blocked-${randomUUID()}`, type: 'combination', dshVersion, environmentVersion, reason,
|
||||
createdAt: this.now().toISOString(),
|
||||
})
|
||||
const next = parseState({ ...current, experimentalBlocked: blocked })
|
||||
await this.write(next)
|
||||
return next
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function held(local, dshVersion, environmentVersion) {
|
||||
return local.holds.find(hold => (
|
||||
hold.dshVersion === dshVersion
|
||||
&& (hold.type === 'version' || hold.environmentVersion === environmentVersion)
|
||||
))
|
||||
}
|
||||
|
||||
export function planDesiredState({ local, current, supported, stableTargetSequence, upstream = null }) {
|
||||
const aheadOfStable = compareDshVersions(current.dsh, supported.dsh) > 0
|
||||
const stableIdentityMismatch = Number.isSafeInteger(stableTargetSequence) && (
|
||||
current.targetSequence !== stableTargetSequence
|
||||
|| (local.updateChannel === 'stable'
|
||||
? current.authority !== 'stable'
|
||||
: !['stable', 'experimental'].includes(current.authority))
|
||||
)
|
||||
const base = {
|
||||
updateChannel: local.updateChannel,
|
||||
current,
|
||||
supported,
|
||||
upstream,
|
||||
aheadOfStable,
|
||||
experimentalBlocked: local.experimentalBlocked,
|
||||
holds: local.holds,
|
||||
}
|
||||
if (aheadOfStable) return Object.freeze({ ...base, action: 'frozen', reason: 'Stable has not caught up with the current DSH' })
|
||||
if (compareDshVersions(current.dsh, supported.dsh) < 0 || current.environment !== supported.environment || stableIdentityMismatch) {
|
||||
return Object.freeze({ ...base, action: 'stable', reason: 'converge the signed Stable target before Experimental DSH' })
|
||||
}
|
||||
if (local.updateChannel === 'stable' || upstream === null) {
|
||||
return Object.freeze({ ...base, action: 'none', reason: null })
|
||||
}
|
||||
const matchingHold = held(local, upstream.version, supported.environment)
|
||||
if (matchingHold !== undefined) return Object.freeze({ ...base, action: 'held', reason: matchingHold.reason, hold: matchingHold })
|
||||
if (
|
||||
local.experimentalBlocked?.dshVersion === upstream.version
|
||||
&& local.experimentalBlocked.environmentVersion === supported.environment
|
||||
) return Object.freeze({ ...base, action: 'blocked', reason: local.experimentalBlocked.reason })
|
||||
return Object.freeze({ ...base, action: 'experimental', reason: null })
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { request } from 'node:http'
|
||||
|
||||
export class LocalApiClient {
|
||||
constructor(socketPath) {
|
||||
this.socketPath = socketPath
|
||||
}
|
||||
|
||||
request(method, path, body, extraHeaders = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const bytes = body === undefined ? undefined : Buffer.from(JSON.stringify(body))
|
||||
const req = request({
|
||||
socketPath: this.socketPath,
|
||||
method,
|
||||
path,
|
||||
headers: {
|
||||
...extraHeaders,
|
||||
...(bytes === undefined ? {} : {
|
||||
'content-type': 'application/json',
|
||||
'content-length': bytes.byteLength,
|
||||
}),
|
||||
},
|
||||
}, response => {
|
||||
const chunks = []
|
||||
response.on('data', chunk => chunks.push(chunk))
|
||||
response.on('end', () => {
|
||||
let value
|
||||
try { value = JSON.parse(Buffer.concat(chunks).toString('utf8')) } catch { value = {} }
|
||||
if ((response.statusCode ?? 500) >= 400) {
|
||||
const error = new Error(value.error ?? `local API returned ${String(response.statusCode)}`)
|
||||
error.statusCode = response.statusCode
|
||||
if (typeof value.code === 'string') error.code = value.code
|
||||
if (Number.isInteger(value.retryAfterSeconds)) error.retryAfterSeconds = value.retryAfterSeconds
|
||||
error.localApiPath = path
|
||||
reject(error)
|
||||
} else resolve(value)
|
||||
})
|
||||
})
|
||||
req.once('error', reject)
|
||||
req.end(bytes)
|
||||
})
|
||||
}
|
||||
|
||||
status() { return this.request('GET', '/v1/status') }
|
||||
activeReceipts() { return this.request('GET', '/v1/receipts/active') }
|
||||
acceptKeyring(document, signature) {
|
||||
return this.request('POST', '/v1/keyring', { document: document.toString('base64'), signature })
|
||||
}
|
||||
acceptTarget(document, signature) {
|
||||
return this.request('POST', '/v1/target', { document: document.toString('base64'), signature })
|
||||
}
|
||||
importArtifact(artifactId, sourcePath, parentReceipt = null) {
|
||||
return this.request('POST', '/v1/artifacts/import', { artifactId, sourcePath, parentReceipt })
|
||||
}
|
||||
ensureOfficialDsh(version, metadataPath, tarballPath) {
|
||||
return this.request('POST', '/v1/dsh/ensure', { version, metadataPath, tarballPath })
|
||||
}
|
||||
acceptManifest(receipt, signatureReceipt) {
|
||||
return this.request('POST', '/v1/manifests/accept', { receipt, signatureReceipt })
|
||||
}
|
||||
activate(receipts) { return this.request('POST', '/v1/activate', { receipts }) }
|
||||
collectGarbage() { return this.request('POST', '/v1/objects/collect', {}) }
|
||||
stageBootstrap(receipt, version) {
|
||||
return this.request('POST', '/v1/bootstrap/stage', { receipt, version })
|
||||
}
|
||||
collectBootstrap() { return this.request('POST', '/v1/bootstrap/collect', {}) }
|
||||
}
|
||||
@@ -0,0 +1,894 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { recoverInterruptedUpdate } from './recovery.mjs'
|
||||
import { planDesiredState } from './channel-state.mjs'
|
||||
import { compareDshVersions } from '../../../../platform/lib/supported-target.mjs'
|
||||
|
||||
export class UpdateConflictError extends Error {}
|
||||
|
||||
function healthMetrics(health) {
|
||||
const components = Array.isArray(health?.components) ? health.components : []
|
||||
return {
|
||||
readyServices: components.filter(component => component?.healthy === true).length,
|
||||
totalServices: components.length,
|
||||
}
|
||||
}
|
||||
|
||||
function journalDeployment(deployment) {
|
||||
return Object.freeze({
|
||||
dsh: deployment.dsh,
|
||||
environment: deployment.environment,
|
||||
runtime: deployment.runtime,
|
||||
dataSnapshot: deployment.dataSnapshot,
|
||||
receiptTokens: deployment.receiptTokens,
|
||||
})
|
||||
}
|
||||
|
||||
function metricPercentage(processed, total) {
|
||||
return typeof processed === 'number' && typeof total === 'number' && total > 0
|
||||
? Math.floor(Math.max(0, Math.min(1, processed / total)) * 100)
|
||||
: null
|
||||
}
|
||||
|
||||
function holdReason(message) {
|
||||
const lines = message.split(/\r?\n/u).map(line => line.trim()).filter(Boolean)
|
||||
const npmNotarget = lines.find(line => /^npm error notarget\b/u.test(line))
|
||||
const selected = npmNotarget?.replace(/^npm error notarget\s*/u, '') ?? lines[0] ?? 'Experimental update failed'
|
||||
const normalized = selected.replace(/[\u0000-\u001f\u007f]/gu, ' ').replace(/\s+/gu, ' ').trim()
|
||||
return (normalized || 'Experimental update failed').slice(0, 512)
|
||||
}
|
||||
|
||||
export class UpdateCoordinator extends EventEmitter {
|
||||
constructor({
|
||||
metadata, preparer, activator, state, npm, journal, snapshots, channelState, completeRecovery, automaticChecks,
|
||||
allowUnavailableMetadata = false,
|
||||
probationSeconds = 120,
|
||||
report = async () => {},
|
||||
now = () => new Date(),
|
||||
sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)),
|
||||
}) {
|
||||
super()
|
||||
this.metadata = metadata
|
||||
this.preparer = preparer
|
||||
this.activator = activator
|
||||
this.state = state
|
||||
this.npm = npm
|
||||
this.journal = journal
|
||||
this.snapshots = snapshots
|
||||
this.channelState = channelState
|
||||
this.completeRecovery = completeRecovery
|
||||
this.automaticChecks = automaticChecks
|
||||
this.allowUnavailableMetadata = allowUnavailableMetadata
|
||||
this.probationSeconds = probationSeconds
|
||||
this.report = report
|
||||
this.now = now
|
||||
this.sleep = sleep
|
||||
this.task = undefined
|
||||
this.checkTask = undefined
|
||||
this.reportTimes = new Map()
|
||||
}
|
||||
|
||||
async transition(status, fields = {}) {
|
||||
const previous = await this.state.read()
|
||||
const phase = fields.phase ?? (
|
||||
status === 'idle'
|
||||
? null
|
||||
: fields.rollbackPhase ?? (
|
||||
['success', 'failed'].includes(status)
|
||||
? previous.phase ?? previous.rollbackPhase ?? (['idle', 'success', 'failed'].includes(previous.status) ? null : previous.status)
|
||||
: status
|
||||
)
|
||||
)
|
||||
const contextChanged = phase !== (previous.phase ?? null)
|
||||
|| (fields.operation !== undefined && fields.operation !== previous.operation)
|
||||
const phaseMetrics = contextChanged ? {
|
||||
processedBytes: null,
|
||||
totalBytes: null,
|
||||
processedItems: null,
|
||||
totalItems: null,
|
||||
readyServices: null,
|
||||
totalServices: null,
|
||||
detail: null,
|
||||
} : {}
|
||||
const value = await this.state.write(status, { ...phaseMetrics, ...fields, phase })
|
||||
this.emit('state', value)
|
||||
await this.record('update.phase.changed', {
|
||||
status,
|
||||
phase,
|
||||
operation: value.operation ?? null,
|
||||
rollbackPhase: value.rollbackPhase ?? null,
|
||||
taskId: value.taskId ?? null,
|
||||
progress: value.progress ?? null,
|
||||
processedBytes: value.processedBytes ?? null,
|
||||
totalBytes: value.totalBytes ?? null,
|
||||
processedItems: value.processedItems ?? null,
|
||||
totalItems: value.totalItems ?? null,
|
||||
readyServices: value.readyServices ?? null,
|
||||
totalServices: value.totalServices ?? null,
|
||||
detail: value.detail ?? null,
|
||||
targetSequence: value.targetSequence ?? value.available?.targetSequence ?? null,
|
||||
outcome: value.outcome ?? null,
|
||||
...(value.error === null || value.error === undefined ? {} : { error: value.error, level: 'error' }),
|
||||
})
|
||||
return value
|
||||
}
|
||||
|
||||
async record(message, fields = {}) {
|
||||
try {
|
||||
const current = await this.state.read()
|
||||
const context = {
|
||||
...(current.taskId === null || current.taskId === undefined ? {} : { taskId: current.taskId }),
|
||||
...(current.operation === null || current.operation === undefined ? {} : { operation: current.operation }),
|
||||
...(current.phase === null || current.phase === undefined
|
||||
? (current.status === null || current.status === undefined ? {} : { phase: current.status })
|
||||
: { phase: current.phase }),
|
||||
}
|
||||
return await this.report(message, { ...context, ...fields })
|
||||
} catch {}
|
||||
}
|
||||
|
||||
async bestEffort(message, operation, fallback, fields = {}, cooldownMs = 0) {
|
||||
try {
|
||||
return await operation()
|
||||
} catch (error) {
|
||||
const timestamp = this.now().valueOf()
|
||||
const previous = this.reportTimes.get(message)
|
||||
if (cooldownMs === 0 || previous === undefined || timestamp - previous >= cooldownMs) {
|
||||
this.reportTimes.set(message, timestamp)
|
||||
await this.record(message, { ...fields, error, level: 'warning' })
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
async reportHealth(status, fields) {
|
||||
if (this.activator.health === undefined) return undefined
|
||||
const health = await this.activator.health()
|
||||
await this.transition(status, { ...fields, ...healthMetrics(health) })
|
||||
return health
|
||||
}
|
||||
|
||||
hasActiveTask() {
|
||||
return this.task !== undefined
|
||||
}
|
||||
|
||||
check(source = 'manual') {
|
||||
if (this.task !== undefined && source === 'page-open') return Promise.resolve({ busy: true })
|
||||
if (this.task !== undefined) return Promise.reject(new UpdateConflictError('an update task is already running'))
|
||||
if (this.checkTask !== undefined) return this.checkTask
|
||||
this.checkTask = this.runCheck(source).finally(() => { this.checkTask = undefined })
|
||||
return this.checkTask
|
||||
}
|
||||
|
||||
startTask(operation) {
|
||||
if (this.task !== undefined) throw new UpdateConflictError('an update task is already running')
|
||||
const taskId = randomUUID()
|
||||
const precedingCheck = this.checkTask
|
||||
this.task = (async () => {
|
||||
if (precedingCheck !== undefined) {
|
||||
try { await precedingCheck } catch {}
|
||||
}
|
||||
return operation(taskId)
|
||||
})().finally(() => { this.task = undefined })
|
||||
return { taskId, completion: this.task }
|
||||
}
|
||||
|
||||
async runCheck(source) {
|
||||
await this.transition('checking', {
|
||||
checkSource: source, taskId: null, progress: 0, operation: 'check', rollbackPhase: null, error: null,
|
||||
})
|
||||
try {
|
||||
if (this.allowUnavailableMetadata) return await this.recordUnavailableMetadata(source)
|
||||
if (this.channelState !== undefined) {
|
||||
const plan = await this.desiredState()
|
||||
const rollbackPlan = await this.rollbackPlan()
|
||||
const updateAvailable = ['stable', 'experimental'].includes(plan.action)
|
||||
await this.transition('idle', {
|
||||
available: {
|
||||
targetSequence: plan.target.targetSequence,
|
||||
bootstrap: plan.target.desired.bootstrap.version,
|
||||
environment: plan.target.desired.environment.version,
|
||||
dsh: plan.target.desired.dsh.version,
|
||||
},
|
||||
supported: plan.supported,
|
||||
...(plan.upstream === null ? {} : { upstream: { version: plan.upstream.version } }),
|
||||
current: { dsh: plan.current.dsh, environment: plan.current.environment, runtime: plan.current.runtime },
|
||||
aheadOfStable: plan.aheadOfStable,
|
||||
experimentalBlocked: plan.experimentalBlocked,
|
||||
holds: plan.holds,
|
||||
rollbackPlan,
|
||||
updateAvailable,
|
||||
checkedAt: this.now().toISOString(),
|
||||
metadataUnavailable: false,
|
||||
remoteCheckError: null,
|
||||
remoteCheckFailedAt: null,
|
||||
remoteCheckSource: null,
|
||||
})
|
||||
if (source === 'automatic' && this.automaticChecks !== undefined) {
|
||||
await this.automaticChecks.record({
|
||||
channel: plan.updateChannel,
|
||||
current: plan.current,
|
||||
target: plan.target,
|
||||
upstream: plan.upstream,
|
||||
stableAvailable: plan.action === 'stable',
|
||||
holds: plan.holds,
|
||||
})
|
||||
this.emit('state', await this.state.read())
|
||||
}
|
||||
return { value: plan.target, updateAvailable }
|
||||
}
|
||||
const target = await this.metadata.check()
|
||||
await this.transition('idle', {
|
||||
available: {
|
||||
targetSequence: target.value.targetSequence,
|
||||
bootstrap: target.value.desired.bootstrap.version,
|
||||
environment: target.value.desired.environment.version,
|
||||
dsh: target.value.desired.dsh.version,
|
||||
},
|
||||
checkedAt: new Date().toISOString(),
|
||||
metadataUnavailable: false,
|
||||
remoteCheckError: null,
|
||||
remoteCheckFailedAt: null,
|
||||
remoteCheckSource: null,
|
||||
})
|
||||
return target
|
||||
} catch (error) {
|
||||
await this.record('update.check.failed', { error, checkSource: source })
|
||||
await this.transition('idle', {
|
||||
error: null,
|
||||
remoteCheckError: error instanceof Error ? error.message : 'update check failed',
|
||||
remoteCheckFailedAt: this.now().toISOString(),
|
||||
remoteCheckSource: error?.remoteCheckSource === 'upstream' ? 'upstream' : 'stable',
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async recordUnavailableMetadata(source) {
|
||||
const local = await this.channelState?.read()
|
||||
const upstream = local?.updateChannel === 'experimental' && this.npm !== undefined
|
||||
? await this.bestEffort('update.upstream-discovery.failed', () => this.npm.discover(), null, { checkSource: source })
|
||||
: null
|
||||
await this.transition('idle', {
|
||||
metadataUnavailable: true,
|
||||
...(upstream === null ? {} : { upstream }),
|
||||
updateAvailable: false,
|
||||
checkedAt: this.now().toISOString(),
|
||||
error: null,
|
||||
})
|
||||
return { unavailable: true, upstream }
|
||||
}
|
||||
|
||||
start() {
|
||||
return this.startTask(taskId => this.run(taskId))
|
||||
}
|
||||
|
||||
startExperimental() {
|
||||
if (this.npm === undefined || this.journal === undefined || this.snapshots === undefined) {
|
||||
throw new Error('Experimental updates are not configured')
|
||||
}
|
||||
return this.startTask(taskId => this.runExperimental(taskId))
|
||||
}
|
||||
|
||||
startReconcile() {
|
||||
return this.startTask(taskId => this.runReconcile(taskId))
|
||||
}
|
||||
|
||||
async setChannel(updateChannel) {
|
||||
if (this.task !== undefined) throw new UpdateConflictError('an update task is already running')
|
||||
if (this.channelState === undefined) throw new Error('update channels are not configured')
|
||||
const result = await this.channelState.setChannel(updateChannel)
|
||||
if (updateChannel === 'stable') await this.automaticChecks?.clearUpstream()
|
||||
await this.record('update.channel.changed', { updateChannel })
|
||||
return result
|
||||
}
|
||||
|
||||
async retryHold(id) {
|
||||
if (this.task !== undefined) throw new UpdateConflictError('an update task is already running')
|
||||
if (this.channelState === undefined) throw new Error('update channels are not configured')
|
||||
const result = await this.channelState.retry(id)
|
||||
await this.record('update.hold.retried', { holdId: id })
|
||||
return result
|
||||
}
|
||||
|
||||
rollbackPlan() {
|
||||
if (this.completeRecovery === undefined) return Promise.resolve(null)
|
||||
return this.completeRecovery.plan()
|
||||
}
|
||||
|
||||
startCompleteRollback(planId, options = {}) {
|
||||
if (this.completeRecovery === undefined) throw new Error('complete rollback is not configured')
|
||||
return this.startTask(taskId => this.runCompleteRollback(taskId, planId, options))
|
||||
}
|
||||
|
||||
async runCompleteRollback(taskId, planId, options) {
|
||||
const operation = options.requireConfirmation ? 'return-stable' : 'rollback'
|
||||
try {
|
||||
const rollbackPlan = await this.completeRecovery.plan()
|
||||
let verifiedStable = null
|
||||
await this.transition('restoring-data', {
|
||||
taskId,
|
||||
progress: 5,
|
||||
operation,
|
||||
rollbackPhase: 'preparing',
|
||||
rollbackIncludesSnapshot: rollbackPlan?.snapshot !== null && rollbackPlan?.snapshot !== undefined,
|
||||
error: null,
|
||||
})
|
||||
if (options.requireConfirmation) {
|
||||
const [plan, target] = await Promise.all([this.completeRecovery.plan(), this.metadata.check()])
|
||||
if (
|
||||
plan === null
|
||||
|| plan.planId !== planId
|
||||
|| compareDshVersions(plan.previous.dsh, target.value.desired.dsh.version) > 0
|
||||
) throw new Error('no verified pre-Experimental Stable recovery point is available')
|
||||
verifiedStable = target.value
|
||||
}
|
||||
const result = await this.completeRecovery.restore(planId, {
|
||||
...options,
|
||||
onProgress: (() => {
|
||||
let previousKey
|
||||
return (rollbackPhase, progress, metrics = {}) => {
|
||||
const key = [
|
||||
rollbackPhase,
|
||||
progress,
|
||||
metricPercentage(metrics.processedBytes, metrics.totalBytes),
|
||||
metricPercentage(metrics.processedItems, metrics.totalItems),
|
||||
].join(':')
|
||||
if (key === previousKey) return undefined
|
||||
previousKey = key
|
||||
return this.transition('restoring-data', {
|
||||
taskId,
|
||||
progress,
|
||||
operation,
|
||||
rollbackPhase,
|
||||
processedBytes: metrics.processedBytes ?? null,
|
||||
totalBytes: metrics.totalBytes ?? null,
|
||||
processedItems: metrics.processedItems ?? null,
|
||||
totalItems: metrics.totalItems ?? null,
|
||||
})
|
||||
}
|
||||
})(),
|
||||
})
|
||||
await this.reportHealth('restoring-data', {
|
||||
taskId,
|
||||
progress: 95,
|
||||
operation,
|
||||
rollbackPhase: 'verifying',
|
||||
})
|
||||
const current = await this.activator.currentDeployment()
|
||||
if (
|
||||
rollbackPlan === null
|
||||
|| current.dsh !== rollbackPlan.previous.dsh
|
||||
|| current.environment !== rollbackPlan.previous.environment
|
||||
|| current.runtime !== rollbackPlan.previous.runtime
|
||||
) throw new Error('restored Deployment differs from the rollback plan')
|
||||
const [stored, local] = await Promise.all([
|
||||
this.state.read(),
|
||||
this.channelState?.read() ?? Promise.resolve({ updateChannel: 'stable', holds: [], experimentalBlocked: null }),
|
||||
])
|
||||
const supported = verifiedStable === null
|
||||
? stored.supported ?? null
|
||||
: {
|
||||
dsh: verifiedStable.desired.dsh.version,
|
||||
environment: verifiedStable.desired.environment.version,
|
||||
}
|
||||
let aheadOfStable = false
|
||||
let updateAvailable = false
|
||||
if (supported !== null) {
|
||||
const upstream = local.updateChannel === 'experimental'
|
||||
&& typeof stored.upstream?.version === 'string'
|
||||
&& compareDshVersions(stored.upstream.version, supported.dsh) > 0
|
||||
? stored.upstream
|
||||
: null
|
||||
const plan = planDesiredState({
|
||||
local,
|
||||
current,
|
||||
supported,
|
||||
stableTargetSequence: verifiedStable?.targetSequence ?? stored.available?.targetSequence,
|
||||
upstream,
|
||||
})
|
||||
aheadOfStable = plan.aheadOfStable
|
||||
updateAvailable = ['stable', 'experimental'].includes(plan.action)
|
||||
}
|
||||
await this.bestEffort('update.notifications.cleanup.failed', () => this.clearSatisfiedNotifications(), undefined, { taskId })
|
||||
await this.transition('success', {
|
||||
taskId,
|
||||
progress: 100,
|
||||
current: { dsh: current.dsh, environment: current.environment, runtime: current.runtime },
|
||||
aheadOfStable,
|
||||
updateAvailable,
|
||||
experimentalBlocked: local.experimentalBlocked,
|
||||
holds: local.holds,
|
||||
outcome: null,
|
||||
probationUntil: null,
|
||||
error: null,
|
||||
})
|
||||
return result
|
||||
} catch (error) {
|
||||
await this.record('update.rollback.failed', { error, taskId, operation })
|
||||
await this.transition('failed', { taskId, error: error instanceof Error ? error.message : 'rollback failed' })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async desiredState() {
|
||||
if (this.channelState === undefined) throw new Error('update channels are not configured')
|
||||
const stable = (await this.metadata.check()).value
|
||||
const [current, local] = await Promise.all([this.activator.currentDeployment(), this.channelState.read()])
|
||||
const supported = { dsh: stable.desired.dsh.version, environment: stable.desired.environment.version }
|
||||
let upstream = null
|
||||
let upstreamCandidate = null
|
||||
if (
|
||||
local.updateChannel === 'experimental'
|
||||
&& compareDshVersions(current.dsh, supported.dsh) <= 0
|
||||
&& current.environment === supported.environment
|
||||
) {
|
||||
try {
|
||||
upstream = await this.npm.discover(stable.officialDshPolicy)
|
||||
} catch (error) {
|
||||
const scoped = new Error(error instanceof Error ? error.message : 'official DSH discovery failed', { cause: error })
|
||||
scoped.name = error instanceof Error ? error.name : 'Error'
|
||||
scoped.code = error?.code
|
||||
scoped.remoteCheckSource = 'upstream'
|
||||
throw scoped
|
||||
}
|
||||
upstreamCandidate = compareDshVersions(upstream.version, supported.dsh) > 0 ? upstream : null
|
||||
}
|
||||
return Object.freeze({
|
||||
...planDesiredState({ local, current, supported, stableTargetSequence: stable.targetSequence, upstream: upstreamCandidate }),
|
||||
upstream,
|
||||
target: stable,
|
||||
})
|
||||
}
|
||||
|
||||
async publicStatus() {
|
||||
const [update, local, current, rollbackPlan, journal, automatic] = await Promise.all([
|
||||
this.state.read(),
|
||||
this.channelState?.read() ?? Promise.resolve({ updateChannel: 'stable', holds: [], experimentalBlocked: null }),
|
||||
this.bestEffort(
|
||||
'update.status.current.failed',
|
||||
() => this.activator.currentDeployment({ required: false }),
|
||||
null,
|
||||
{},
|
||||
60_000,
|
||||
),
|
||||
this.bestEffort('update.status.rollback-plan.failed', () => this.rollbackPlan(), null, {}, 60_000),
|
||||
this.journal === undefined
|
||||
? Promise.resolve(undefined)
|
||||
: this.bestEffort('update.status.journal.failed', () => this.journal.read(), undefined, {}, 60_000),
|
||||
this.automaticChecks?.read() ?? Promise.resolve(null),
|
||||
])
|
||||
const returnStableAvailable = rollbackPlan !== null && rollbackPlan.snapshot !== null && update.supported?.dsh !== undefined
|
||||
&& compareDshVersions(rollbackPlan.previous.dsh, update.supported.dsh) <= 0
|
||||
return Object.freeze({
|
||||
update,
|
||||
updateChannel: local.updateChannel,
|
||||
current: current === null ? null : { dsh: current.dsh, environment: current.environment, runtime: current.runtime },
|
||||
supported: update.supported ?? null,
|
||||
upstream: update.upstream ?? null,
|
||||
aheadOfStable: update.aheadOfStable ?? false,
|
||||
experimentalBlocked: local.experimentalBlocked,
|
||||
holds: local.holds,
|
||||
probation: journal?.phase === 'probation' ? { until: journal.probationUntil } : null,
|
||||
rollbackPlan: rollbackPlan === null ? null : { ...rollbackPlan, returnStableAvailable },
|
||||
...(automatic === null ? {} : {
|
||||
automaticCheck: automatic.automaticCheck,
|
||||
latestAutomatic: automatic.latestAutomatic,
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
async runReconcile(taskId) {
|
||||
try {
|
||||
const plan = await this.desiredState()
|
||||
if (plan.action === 'stable') {
|
||||
const blockExperimentalCombination = plan.current?.authority === 'experimental'
|
||||
&& plan.current.environment !== plan.supported?.environment
|
||||
await this.run(taskId, { complete: false })
|
||||
const next = await this.desiredState()
|
||||
if (next.action === 'experimental') {
|
||||
return this.runExperimental(taskId, { blockCombination: blockExperimentalCombination })
|
||||
}
|
||||
return this.transition('success', {
|
||||
taskId,
|
||||
progress: 100,
|
||||
current: { dsh: next.current.dsh, environment: next.current.environment, runtime: next.current.runtime },
|
||||
aheadOfStable: next.aheadOfStable,
|
||||
experimentalBlocked: next.experimentalBlocked,
|
||||
holds: next.holds,
|
||||
updateAvailable: ['stable', 'experimental'].includes(next.action),
|
||||
error: null,
|
||||
outcome: next.action,
|
||||
})
|
||||
}
|
||||
if (plan.action === 'experimental') return this.runExperimental(taskId)
|
||||
return this.transition('success', {
|
||||
taskId, progress: 100, error: null,
|
||||
outcome: plan.action,
|
||||
})
|
||||
} catch (error) {
|
||||
await this.record('update.reconcile.failed', { error, taskId })
|
||||
const current = await this.state.read()
|
||||
if (current.status !== 'failed') {
|
||||
await this.transition('failed', { taskId, error: error instanceof Error ? error.message : 'update planning failed' })
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async run(taskId, { complete = true } = {}) {
|
||||
try {
|
||||
await this.transition('planning', {
|
||||
taskId, progress: 0, operation: 'update', rollbackPhase: null, rollbackIncludesSnapshot: null,
|
||||
processedBytes: null, totalBytes: null, processedItems: null, totalItems: null,
|
||||
readyServices: null, totalServices: null, detail: null, error: null,
|
||||
})
|
||||
const target = await this.metadata.check()
|
||||
const artifacts = Array.isArray(target.value.artifacts) ? target.value.artifacts : []
|
||||
await this.transition('downloading', {
|
||||
taskId,
|
||||
progress: 10,
|
||||
targetSequence: target.value.targetSequence,
|
||||
processedBytes: 0,
|
||||
totalBytes: artifacts.reduce((total, artifact) => total + artifact.size, 0),
|
||||
processedItems: 0,
|
||||
totalItems: artifacts.length,
|
||||
detail: 'downloading',
|
||||
})
|
||||
let previousDownloadKey
|
||||
let previousDownloadProgress = 10
|
||||
const prepared = await this.preparer.prepare(target.value, {
|
||||
onProgress: metrics => {
|
||||
const byteRatio = Number.isFinite(metrics.totalBytes) && metrics.totalBytes > 0
|
||||
? metrics.processedBytes / metrics.totalBytes
|
||||
: null
|
||||
const itemRatio = Number.isFinite(metrics.totalItems) && metrics.totalItems > 0
|
||||
? metrics.processedItems / metrics.totalItems
|
||||
: null
|
||||
const ratio = byteRatio ?? itemRatio
|
||||
const measuredProgress = ratio === null ? 10 : 10 + Math.round(Math.max(0, Math.min(1, ratio)) * 55)
|
||||
const progress = Math.max(previousDownloadProgress, measuredProgress)
|
||||
previousDownloadProgress = progress
|
||||
const key = `${progress}:${metrics.processedItems}:${metrics.totalItems}`
|
||||
if (key === previousDownloadKey) return undefined
|
||||
previousDownloadKey = key
|
||||
return this.transition('downloading', {
|
||||
taskId,
|
||||
progress,
|
||||
processedBytes: metrics.processedBytes,
|
||||
totalBytes: metrics.totalBytes,
|
||||
processedItems: metrics.processedItems,
|
||||
totalItems: metrics.totalItems,
|
||||
detail: metrics.detail ?? null,
|
||||
})
|
||||
},
|
||||
})
|
||||
await this.transition('validating', { taskId, progress: 70 })
|
||||
await this.transition('building-candidate', { taskId, progress: 75 })
|
||||
let previousBuildProgress = 75
|
||||
let buildComplete = false
|
||||
await this.activator.activate(prepared, {
|
||||
onProgress: progress => {
|
||||
if (typeof progress === 'number') {
|
||||
const stageProgress = Math.max(previousBuildProgress, progress)
|
||||
if (stageProgress === previousBuildProgress) return undefined
|
||||
previousBuildProgress = stageProgress
|
||||
return this.transition('building-candidate', { taskId, progress: stageProgress })
|
||||
}
|
||||
const ratio = progress.totalBytes > 0 ? progress.processedBytes / progress.totalBytes : 0
|
||||
const stageProgress = Math.max(
|
||||
previousBuildProgress,
|
||||
75 + Math.round(Math.max(0, Math.min(1, ratio)) * 14),
|
||||
)
|
||||
const complete = progress.processedBytes === progress.totalBytes
|
||||
&& progress.processedItems === progress.totalItems
|
||||
if (stageProgress === previousBuildProgress && (!complete || buildComplete)) return undefined
|
||||
previousBuildProgress = stageProgress
|
||||
buildComplete = complete
|
||||
return this.transition('building-candidate', {
|
||||
taskId,
|
||||
progress: stageProgress,
|
||||
processedBytes: progress.processedBytes,
|
||||
totalBytes: progress.totalBytes,
|
||||
processedItems: progress.processedItems,
|
||||
totalItems: progress.totalItems,
|
||||
})
|
||||
},
|
||||
onSwitching: () => this.transition('switching', { taskId, progress: 90 }),
|
||||
})
|
||||
const health = await this.reportHealth('switching', { taskId, progress: 95 })
|
||||
const current = this.activator.currentDeployment === undefined
|
||||
? null
|
||||
: await this.activator.currentDeployment()
|
||||
await this.bestEffort('update.notifications.cleanup.failed', () => this.clearSatisfiedNotifications(), undefined, { taskId })
|
||||
return complete ? this.transition('success', {
|
||||
taskId,
|
||||
progress: 100,
|
||||
...(current === null ? {} : {
|
||||
current: { dsh: current.dsh, environment: current.environment, runtime: current.runtime },
|
||||
updateAvailable: false,
|
||||
}),
|
||||
error: null,
|
||||
}) : health
|
||||
} catch (error) {
|
||||
await this.record('update.stable.failed', { error, taskId })
|
||||
await this.transition('failed', { taskId, error: error instanceof Error ? error.message : 'update failed' })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
trustActivate(prepared) {
|
||||
return this.preparer.trust.activate(prepared.receiptTokens)
|
||||
}
|
||||
|
||||
async runExperimental(taskId, { blockCombination = false } = {}) {
|
||||
let transaction
|
||||
let candidate
|
||||
let failureClass = 'check'
|
||||
let obsoleteSnapshotId = null
|
||||
try {
|
||||
await this.transition('checking-upstream', {
|
||||
taskId, progress: 0, operation: 'update', rollbackPhase: null, rollbackIncludesSnapshot: null, error: null,
|
||||
})
|
||||
const stable = (await this.metadata.check()).value
|
||||
candidate = await this.npm.latest(stable)
|
||||
if (candidate === null) return this.transition('success', { taskId, progress: 100, error: null })
|
||||
const from = await this.activator.currentDeployment()
|
||||
const bootstrap = await this.activator.bootstrap.status()
|
||||
if (
|
||||
from.environment !== stable.desired.environment.version
|
||||
|| bootstrap.bootstrapVersion !== stable.desired.bootstrap.version
|
||||
) throw new Error('Stable Environment and Bootstrap must converge before Experimental DSH')
|
||||
|
||||
await this.transition('downloading', { taskId, progress: 10 })
|
||||
failureClass = 'candidate'
|
||||
let previousDownloadProgress = 10
|
||||
const prepared = await this.preparer.prepareExperimental(candidate, {
|
||||
onProgress: metrics => {
|
||||
const ratio = Number.isFinite(metrics.totalBytes) && metrics.totalBytes > 0
|
||||
? metrics.processedBytes / metrics.totalBytes
|
||||
: metrics.processedItems / metrics.totalItems
|
||||
const progress = Math.max(previousDownloadProgress, 10 + Math.round(Math.max(0, Math.min(1, ratio)) * 20))
|
||||
previousDownloadProgress = progress
|
||||
return this.transition('downloading', {
|
||||
taskId,
|
||||
progress,
|
||||
processedBytes: metrics.processedBytes,
|
||||
totalBytes: metrics.totalBytes,
|
||||
processedItems: metrics.processedItems,
|
||||
totalItems: metrics.totalItems,
|
||||
detail: metrics.detail ?? null,
|
||||
})
|
||||
},
|
||||
})
|
||||
await this.transition('building-candidate', { taskId, progress: 35 })
|
||||
const built = await this.activator.prepareExperimental(prepared, { targetSequence: stable.targetSequence })
|
||||
const previousJournal = await this.journal.read()
|
||||
if (previousJournal !== undefined && ['committed', 'rolled-back', 'failed'].includes(previousJournal.phase)) {
|
||||
obsoleteSnapshotId = previousJournal.snapshotId
|
||||
}
|
||||
transaction = await this.journal.begin({
|
||||
transactionId: taskId,
|
||||
mode: 'experimental',
|
||||
from: journalDeployment(from),
|
||||
to: { dsh: built.dshVersion, environment: built.environmentVersion, runtime: built.runtimeId },
|
||||
})
|
||||
transaction = await this.journal.transition('candidate-ready', { receiptTokens: prepared.receiptTokens })
|
||||
|
||||
await this.transition('snapshotting-data', { taskId, progress: 55 })
|
||||
failureClass = 'snapshot'
|
||||
await this.activator.suspendDsh()
|
||||
transaction = await this.journal.transition('suspended')
|
||||
let previousSnapshotProgress
|
||||
const snapshot = await this.snapshots.create({
|
||||
id: taskId,
|
||||
runtimeId: from.runtime,
|
||||
environmentVersion: from.environment,
|
||||
dshVersion: from.dsh,
|
||||
onProgress: metrics => {
|
||||
const metricProgress = metricPercentage(metrics.processedItems, metrics.totalItems)
|
||||
?? metricPercentage(metrics.processedBytes, metrics.totalBytes)
|
||||
if (metricProgress === previousSnapshotProgress) return undefined
|
||||
previousSnapshotProgress = metricProgress
|
||||
return this.transition('snapshotting-data', {
|
||||
taskId,
|
||||
progress: 55,
|
||||
processedBytes: metrics.processedBytes ?? null,
|
||||
totalBytes: metrics.totalBytes ?? null,
|
||||
processedItems: metrics.processedItems ?? null,
|
||||
totalItems: metrics.totalItems ?? null,
|
||||
})
|
||||
},
|
||||
})
|
||||
const candidateRuntimeId = this.activator.bindExperimentalSnapshot === undefined
|
||||
? built.runtimeId
|
||||
: await this.activator.bindExperimentalSnapshot(built.runtimeId, snapshot.id)
|
||||
transaction = await this.journal.transition('snapshot-created', {
|
||||
snapshotId: snapshot.id,
|
||||
to: { ...transaction.to, runtime: candidateRuntimeId },
|
||||
})
|
||||
|
||||
await this.transition('switching', { taskId, progress: 70 })
|
||||
await this.activator.switchExperimental(candidateRuntimeId)
|
||||
failureClass = 'combination'
|
||||
transaction = await this.journal.transition('switched')
|
||||
const probationUntil = new Date(this.now().valueOf() + this.probationSeconds * 1000).toISOString()
|
||||
transaction = await this.journal.transition('probation', { probationUntil })
|
||||
await this.transition('probation', { taskId, progress: 85, probationUntil })
|
||||
do {
|
||||
const health = await this.activator.health()
|
||||
const remainingMs = Math.max(0, new Date(probationUntil).valueOf() - this.now().valueOf())
|
||||
const elapsedRatio = this.probationSeconds <= 0
|
||||
? 1
|
||||
: Math.max(0, Math.min(1, 1 - remainingMs / (this.probationSeconds * 1000)))
|
||||
await this.transition('probation', {
|
||||
taskId,
|
||||
progress: 85 + Math.round(elapsedRatio * 14),
|
||||
probationUntil,
|
||||
detail: `probation:${String(Math.ceil(remainingMs / 1000))}`,
|
||||
...healthMetrics(health),
|
||||
})
|
||||
if (!health.healthy) throw new Error('Experimental Runtime failed probation health checks')
|
||||
if (this.now().valueOf() >= new Date(probationUntil).valueOf()) break
|
||||
await this.sleep(Math.min(1_000, Math.max(0, new Date(probationUntil).valueOf() - this.now().valueOf())))
|
||||
} while (true)
|
||||
await this.activator.commitExperimental(candidateRuntimeId)
|
||||
transaction = await this.journal.transition('committed')
|
||||
await this.bestEffort('update.notifications.cleanup.failed', () => this.clearSatisfiedNotifications(), undefined, { taskId })
|
||||
if (this.activator.cleanup !== undefined) {
|
||||
await this.bestEffort('update.assets.cleanup.failed', () => this.activator.cleanup(), undefined, { taskId })
|
||||
}
|
||||
if (obsoleteSnapshotId !== null && obsoleteSnapshotId !== transaction.snapshotId) {
|
||||
if (this.snapshots.remove !== undefined) {
|
||||
await this.bestEffort('update.snapshot.cleanup.failed', () => this.snapshots.remove(obsoleteSnapshotId), undefined, {
|
||||
snapshotId: obsoleteSnapshotId,
|
||||
taskId,
|
||||
})
|
||||
}
|
||||
}
|
||||
return this.transition('success', {
|
||||
taskId,
|
||||
progress: 100,
|
||||
current: { dsh: built.dshVersion, environment: built.environmentVersion, runtime: candidateRuntimeId },
|
||||
aheadOfStable: true,
|
||||
updateAvailable: false,
|
||||
outcome: 'frozen',
|
||||
probationUntil: null,
|
||||
error: null,
|
||||
})
|
||||
} catch (error) {
|
||||
let message = error instanceof Error ? error.message : 'Experimental update failed'
|
||||
const reason = holdReason(message)
|
||||
const failureState = await this.state.read()
|
||||
const failurePhase = failureState.phase ?? failureState.status
|
||||
let recoveryCompleted = false
|
||||
await this.record('update.experimental.failed', { error, failureClass, taskId })
|
||||
if (candidate !== undefined && this.channelState !== undefined) {
|
||||
if (failureClass === 'candidate') {
|
||||
await this.bestEffort('update.hold.persist.failed', () => this.channelState.addHold({
|
||||
type: 'version', dshVersion: candidate.version, reason,
|
||||
}), undefined, { dshVersion: candidate.version, holdType: 'version', taskId })
|
||||
} else if (failureClass === 'combination') {
|
||||
const environmentVersion = transaction?.to.environment
|
||||
if (environmentVersion !== undefined) {
|
||||
const persist = blockCombination
|
||||
? () => this.channelState.block({ dshVersion: candidate.version, environmentVersion, reason })
|
||||
: () => this.channelState.addHold({
|
||||
type: 'combination', dshVersion: candidate.version, environmentVersion, reason,
|
||||
})
|
||||
await this.bestEffort(
|
||||
blockCombination ? 'update.experimental-block.persist.failed' : 'update.hold.persist.failed',
|
||||
persist,
|
||||
undefined,
|
||||
{
|
||||
dshVersion: candidate.version,
|
||||
environmentVersion,
|
||||
...(blockCombination ? { blocked: true } : { holdType: 'combination' }),
|
||||
taskId,
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.journal !== undefined) {
|
||||
transaction = await this.bestEffort('update.journal.read.failed', () => this.journal.read(), transaction, { taskId })
|
||||
}
|
||||
if (transaction !== undefined && !['committed', 'rolled-back', 'failed'].includes(transaction.phase)) {
|
||||
try {
|
||||
if (['snapshot-created', 'switched', 'probation', 'restoring-data'].includes(transaction.phase)) {
|
||||
if (transaction.phase !== 'restoring-data') transaction = await this.journal.transition('restoring-data', { error: message })
|
||||
const recoveryProgress = offset => Math.min(99, Math.max(Number(failureState.progress) || 0, offset))
|
||||
await this.transition('restoring-data', {
|
||||
taskId,
|
||||
operation: 'update',
|
||||
progress: recoveryProgress(86),
|
||||
detail: 'recovery:suspend',
|
||||
rollbackIncludesSnapshot: transaction.snapshotId !== null,
|
||||
error: null,
|
||||
})
|
||||
await this.bestEffort('update.rollback.suspend.failed', () => this.activator.suspendDsh(), undefined, { taskId })
|
||||
await this.transition('restoring-data', {
|
||||
taskId, operation: 'update', progress: recoveryProgress(88), detail: 'recovery:deployment',
|
||||
})
|
||||
await this.activator.restoreDeployment(transaction.from, { resume: false })
|
||||
if (transaction.snapshotId !== null) {
|
||||
await this.transition('restoring-data', {
|
||||
taskId, operation: 'update', progress: recoveryProgress(90), detail: 'recovery:snapshot',
|
||||
})
|
||||
let previousRestoreProgress
|
||||
await this.snapshots.restore(transaction.snapshotId, {
|
||||
onProgress: metrics => {
|
||||
const metricProgress = metricPercentage(metrics.processedItems, metrics.totalItems)
|
||||
?? metricPercentage(metrics.processedBytes, metrics.totalBytes)
|
||||
if (metricProgress === previousRestoreProgress) return undefined
|
||||
previousRestoreProgress = metricProgress
|
||||
return this.transition('restoring-data', {
|
||||
taskId,
|
||||
operation: 'update',
|
||||
progress: recoveryProgress(90 + Math.round((metricProgress ?? 0) * 0.05)),
|
||||
detail: 'recovery:snapshot',
|
||||
processedBytes: metrics.processedBytes ?? null,
|
||||
totalBytes: metrics.totalBytes ?? null,
|
||||
processedItems: metrics.processedItems ?? null,
|
||||
totalItems: metrics.totalItems ?? null,
|
||||
})
|
||||
},
|
||||
})
|
||||
}
|
||||
await this.transition('restoring-data', {
|
||||
taskId,
|
||||
operation: 'update',
|
||||
progress: recoveryProgress(96),
|
||||
detail: 'recovery:runtime',
|
||||
processedBytes: null,
|
||||
totalBytes: null,
|
||||
processedItems: null,
|
||||
totalItems: null,
|
||||
})
|
||||
await this.activator.resumeDsh()
|
||||
const health = await this.reportHealth('restoring-data', {
|
||||
taskId, operation: 'update', progress: recoveryProgress(99), detail: 'recovery:health',
|
||||
})
|
||||
if (health?.healthy === false) throw new Error('restored Deployment failed health checks')
|
||||
await this.journal.transition('rolled-back', { error: message })
|
||||
recoveryCompleted = true
|
||||
await this.record('update.experimental.rollback.completed', { taskId })
|
||||
} else {
|
||||
await this.bestEffort('update.rollback.resume.failed', () => this.activator.resumeDsh(), undefined, { taskId })
|
||||
await this.journal.transition('failed', { error: message })
|
||||
}
|
||||
} catch (rollbackError) {
|
||||
await this.record('update.experimental.rollback.failed', { error: rollbackError, taskId })
|
||||
const rollbackMessage = rollbackError instanceof Error ? rollbackError.message : 'unknown error'
|
||||
message = `${message}; rollback failed: ${rollbackMessage}`
|
||||
}
|
||||
}
|
||||
await this.transition('failed', {
|
||||
taskId,
|
||||
error: message,
|
||||
...(recoveryCompleted ? {
|
||||
phase: failurePhase,
|
||||
progress: failureState.progress,
|
||||
detail: failureState.detail,
|
||||
processedBytes: failureState.processedBytes,
|
||||
totalBytes: failureState.totalBytes,
|
||||
processedItems: failureState.processedItems,
|
||||
totalItems: failureState.totalItems,
|
||||
readyServices: failureState.readyServices,
|
||||
totalServices: failureState.totalServices,
|
||||
} : {}),
|
||||
})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async recover() {
|
||||
if (this.journal === undefined || this.snapshots === undefined) return undefined
|
||||
return recoverInterruptedUpdate({ journal: this.journal, snapshots: this.snapshots, activator: this.activator })
|
||||
}
|
||||
|
||||
async clearSatisfiedNotifications() {
|
||||
if (this.automaticChecks === undefined) return
|
||||
await this.automaticChecks.clearSatisfied(await this.activator.currentDeployment())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { open, readFile, rm } from 'node:fs/promises'
|
||||
import { dirname } from 'node:path'
|
||||
import { durableReplace } from '../../../../platform/lib/atomic.mjs'
|
||||
import { exactKeys, isoTimestamp, plainObject, TrustError } from '../../../../platform/lib/validation.mjs'
|
||||
|
||||
const PHASES = new Set([
|
||||
'planning', 'candidate-ready', 'suspended', 'snapshot-created', 'switched', 'probation',
|
||||
'committed', 'restoring-data', 'rolled-back', 'failed',
|
||||
])
|
||||
|
||||
const TRANSITIONS = new Map([
|
||||
['planning', new Set(['candidate-ready', 'failed'])],
|
||||
['candidate-ready', new Set(['suspended', 'failed'])],
|
||||
['suspended', new Set(['snapshot-created', 'failed'])],
|
||||
['snapshot-created', new Set(['switched', 'restoring-data', 'failed'])],
|
||||
['switched', new Set(['probation', 'restoring-data'])],
|
||||
['probation', new Set(['committed', 'restoring-data'])],
|
||||
['restoring-data', new Set(['rolled-back', 'failed'])],
|
||||
['committed', new Set(['restoring-data'])],
|
||||
['rolled-back', new Set()],
|
||||
['failed', new Set()],
|
||||
])
|
||||
|
||||
function deployment(value, label, snapshot) {
|
||||
const object = plainObject(value, label)
|
||||
exactKeys(object, ['dsh', 'environment', 'runtime', ...(snapshot ? ['dataSnapshot', 'receiptTokens'] : [])], label)
|
||||
for (const name of ['dsh', 'environment', 'runtime']) {
|
||||
if (typeof object[name] !== 'string' || object[name] === '') throw new TrustError(`${label}.${name} is invalid`)
|
||||
}
|
||||
if (snapshot && object.dataSnapshot !== null && typeof object.dataSnapshot !== 'string') {
|
||||
throw new TrustError(`${label}.dataSnapshot is invalid`)
|
||||
}
|
||||
if (snapshot && (!Array.isArray(object.receiptTokens) || object.receiptTokens.some(token => typeof token !== 'string'))) {
|
||||
throw new TrustError(`${label}.receiptTokens is invalid`)
|
||||
}
|
||||
return Object.freeze({ ...object, ...(snapshot ? { receiptTokens: Object.freeze([...object.receiptTokens]) } : {}) })
|
||||
}
|
||||
|
||||
function parseJournal(value) {
|
||||
const object = plainObject(value, 'update journal')
|
||||
exactKeys(object, [
|
||||
'error', 'from', 'mode', 'phase', 'probationUntil', 'receiptTokens', 'schema',
|
||||
'snapshotId', 'to', 'transactionId', 'updatedAt',
|
||||
], 'update journal')
|
||||
if (object.schema !== 1) throw new TrustError('update journal schema must be 1')
|
||||
if (typeof object.transactionId !== 'string' || !/^[A-Za-z0-9-]{1,128}$/.test(object.transactionId)) {
|
||||
throw new TrustError('update journal transaction ID is invalid')
|
||||
}
|
||||
if (!['stable', 'experimental'].includes(object.mode)) throw new TrustError('update journal mode is invalid')
|
||||
if (!PHASES.has(object.phase)) throw new TrustError('update journal phase is invalid')
|
||||
if (!Array.isArray(object.receiptTokens) || object.receiptTokens.some(token => typeof token !== 'string')) {
|
||||
throw new TrustError('update journal receipt tokens are invalid')
|
||||
}
|
||||
for (const name of ['snapshotId', 'probationUntil', 'error']) {
|
||||
if (object[name] !== null && typeof object[name] !== 'string') throw new TrustError(`update journal ${name} is invalid`)
|
||||
}
|
||||
if (object.probationUntil !== null) isoTimestamp(object.probationUntil, 'update journal probation deadline')
|
||||
isoTimestamp(object.updatedAt, 'update journal update time')
|
||||
return Object.freeze({
|
||||
...object,
|
||||
from: deployment(object.from, 'update journal from', true),
|
||||
to: deployment(object.to, 'update journal to', false),
|
||||
receiptTokens: Object.freeze([...object.receiptTokens]),
|
||||
})
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
export class UpdateJournal {
|
||||
constructor(path, now = () => new Date()) {
|
||||
this.path = path
|
||||
this.now = now
|
||||
this.queue = Promise.resolve()
|
||||
}
|
||||
|
||||
exclusive(operation) {
|
||||
const result = this.queue.then(operation, operation)
|
||||
this.queue = result.then(() => undefined, () => undefined)
|
||||
return result
|
||||
}
|
||||
|
||||
async read() {
|
||||
try { return parseJournal(JSON.parse(await readFile(this.path, 'utf8'))) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
begin(value) {
|
||||
return this.exclusive(async () => {
|
||||
const previous = await this.read()
|
||||
if (previous !== undefined && !['committed', 'rolled-back', 'failed'].includes(previous.phase)) {
|
||||
throw new TrustError('an update transaction journal already exists')
|
||||
}
|
||||
const journal = parseJournal({
|
||||
schema: 1,
|
||||
transactionId: value.transactionId,
|
||||
mode: value.mode,
|
||||
phase: 'planning',
|
||||
from: value.from,
|
||||
to: value.to,
|
||||
snapshotId: null,
|
||||
receiptTokens: [],
|
||||
probationUntil: null,
|
||||
error: null,
|
||||
updatedAt: this.now().toISOString(),
|
||||
})
|
||||
await durableReplace(this.path, `${JSON.stringify(journal)}\n`)
|
||||
return journal
|
||||
})
|
||||
}
|
||||
|
||||
transition(phase, fields = {}) {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
if (current === undefined) throw new TrustError('update transaction journal does not exist')
|
||||
if (!TRANSITIONS.get(current.phase)?.has(phase)) {
|
||||
throw new TrustError(`update journal cannot transition from ${current.phase} to ${phase}`)
|
||||
}
|
||||
const next = parseJournal({ ...current, ...fields, phase, updatedAt: this.now().toISOString() })
|
||||
if (next.transactionId !== current.transactionId || next.mode !== current.mode) {
|
||||
throw new TrustError('update journal identity cannot change')
|
||||
}
|
||||
await durableReplace(this.path, `${JSON.stringify(next)}\n`)
|
||||
return next
|
||||
})
|
||||
}
|
||||
|
||||
clear() {
|
||||
return this.exclusive(async () => {
|
||||
const current = await this.read()
|
||||
if (current !== undefined && !['committed', 'rolled-back', 'failed'].includes(current.phase)) {
|
||||
throw new TrustError('cannot clear a nonterminal update journal')
|
||||
}
|
||||
await rm(this.path, { force: true })
|
||||
await syncDirectory(dirname(this.path)).catch(error => { if (error?.code !== 'ENOENT') throw error })
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { cp, lstat, mkdir, readFile, rename, rm, symlink } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { buildRuntime } from '../../patch-manager/index.mjs'
|
||||
import { reconcileSystemPlugins } from '../../plugin-manager/system.mjs'
|
||||
import { artifactForReference, parseEnvironmentManifest } from '../../../../platform/lib/contracts.mjs'
|
||||
import { deriveRecordId, parseDeploymentRecord } from '../../../../platform/lib/deployment-contracts.mjs'
|
||||
import { hashTree } from '../../../../platform/lib/tree-hash.mjs'
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolve(Buffer.concat(stdout).toString('utf8'))
|
||||
: reject(new Error(Buffer.concat(stderr).toString('utf8'))))
|
||||
})
|
||||
}
|
||||
|
||||
const NPM_INSTALL_ATTEMPTS = 7
|
||||
const NPM_INSTALL_RETRY_MS = 5_000
|
||||
|
||||
function retryableNpmInstall(error) {
|
||||
return /(?:^|\n)npm error (?:code (?:ETARGET|E404|ENOVERSIONS)|notarget\b)/u.test(
|
||||
error instanceof Error ? error.message : '',
|
||||
)
|
||||
}
|
||||
|
||||
async function verifyDirectory(path, expectedHash) {
|
||||
const details = await lstat(path)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) throw new Error(`Managed Store asset is not an immutable directory: ${path}`)
|
||||
const sha256 = await hashTree(path)
|
||||
if (expectedHash !== undefined && sha256 !== expectedHash) throw new Error(`Managed Store asset content conflicts at ${path}`)
|
||||
return sha256
|
||||
}
|
||||
|
||||
async function publish(staging, root, id) {
|
||||
const sha256 = await hashTree(staging)
|
||||
const destination = join(root, id)
|
||||
try {
|
||||
await rename(staging, destination)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
|
||||
await verifyDirectory(destination, sha256)
|
||||
}
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
|
||||
function storeReference(kind, asset) {
|
||||
return Object.freeze({ storage: 'store', kind, id: asset.id, sha256: asset.sha256 })
|
||||
}
|
||||
|
||||
export class ManagedDeploymentBuilder {
|
||||
constructor({
|
||||
paths,
|
||||
npmInstall = (installRoot, archivePath, { preferOnline }) => run('npm', [
|
||||
'install', '--global', '--prefix', installRoot, '--omit=dev', '--ignore-scripts',
|
||||
'--no-audit', '--no-fund', preferOnline ? '--prefer-online' : '--prefer-offline',
|
||||
'--cache', paths.npmCacheRoot, archivePath,
|
||||
]),
|
||||
sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)),
|
||||
}) {
|
||||
this.paths = paths
|
||||
this.npmInstall = npmInstall
|
||||
this.sleep = sleep
|
||||
}
|
||||
|
||||
async pristine(version, receipt) {
|
||||
const id = `pristine-npm-${receipt.objectSha256}`
|
||||
const destination = join(this.paths.pristineRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
const installRoot = join(this.paths.pristineRoot, `.install.${randomUUID()}.tmp`)
|
||||
const staging = join(this.paths.pristineRoot, `.${id}.${randomUUID()}.tmp`)
|
||||
await mkdir(installRoot, { recursive: false })
|
||||
try {
|
||||
const entries = (await run('tar', ['-tzf', receipt.path])).split('\n').filter(Boolean)
|
||||
if (!entries.includes('package/package.json') || entries.some(entry => (
|
||||
entry.startsWith('/') || !entry.startsWith('package/') || entry.split('/').includes('..')
|
||||
))) throw new Error('official DSH archive contains an unsafe path')
|
||||
for (let attempt = 1; attempt <= NPM_INSTALL_ATTEMPTS; attempt += 1) {
|
||||
const attemptRoot = join(installRoot, `attempt-${String(attempt)}`)
|
||||
await mkdir(attemptRoot, { recursive: false })
|
||||
try {
|
||||
const archivePath = join(attemptRoot, 'official-dsh.tgz')
|
||||
await symlink(receipt.path, archivePath, 'file')
|
||||
await this.npmInstall(attemptRoot, archivePath, { preferOnline: attempt > 1 })
|
||||
const installed = join(attemptRoot, 'lib', 'node_modules', '@deepseek-ai', 'dsh')
|
||||
await rename(installed, staging)
|
||||
break
|
||||
} catch (error) {
|
||||
if (attempt === NPM_INSTALL_ATTEMPTS || !retryableNpmInstall(error)) throw error
|
||||
await this.sleep(Math.min(30_000, NPM_INSTALL_RETRY_MS * (2 ** (attempt - 1))))
|
||||
} finally {
|
||||
await rm(attemptRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
const metadata = JSON.parse(await readFile(join(staging, 'package.json'), 'utf8'))
|
||||
if (metadata.name !== '@deepseek-ai/dsh' || metadata.version !== version) {
|
||||
throw new Error('official DSH package metadata differs from its requested version')
|
||||
}
|
||||
return await publish(staging, this.paths.pristineRoot, id)
|
||||
} finally {
|
||||
await rm(installRoot, { recursive: true, force: true })
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async environment(prepared) {
|
||||
const manifest = prepared.environment.manifest
|
||||
const identity = {
|
||||
schema: 1,
|
||||
version: manifest.version,
|
||||
targetSequence: prepared.stable.targetSequence,
|
||||
manifest: prepared.environment.manifestReceipt.objectSha256,
|
||||
artifacts: manifest.artifacts.map(artifact => ({ id: artifact.id, sha256: artifact.sha256 })),
|
||||
}
|
||||
const id = deriveRecordId('environment', identity)
|
||||
const destination = join(this.paths.environmentsRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
const staging = join(this.paths.environmentsRoot, `.${id}.${randomUUID()}.tmp`)
|
||||
await mkdir(join(staging, 'artifacts'), { recursive: true })
|
||||
try {
|
||||
await cp(prepared.paths.get(prepared.stable.desired.environment.manifestArtifactId), join(staging, 'environment.manifest.json'))
|
||||
for (const descriptor of manifest.artifacts) {
|
||||
await cp(prepared.paths.get(descriptor.id), join(staging, 'artifacts', descriptor.id))
|
||||
}
|
||||
return await publish(staging, this.paths.environmentsRoot, id)
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async runtime(prepared, pristine, environment, { onProgress = async () => {} } = {}) {
|
||||
const patches = prepared.environment.manifest.patches.map(reference => {
|
||||
const artifact = artifactForReference(prepared.environment.manifest, reference)
|
||||
return { id: reference.id, artifactId: artifact.id, sha256: artifact.sha256 }
|
||||
})
|
||||
const id = deriveRecordId('runtime', {
|
||||
schema: 1,
|
||||
pristine: pristine.sha256,
|
||||
environment: environment.sha256,
|
||||
patches,
|
||||
})
|
||||
const destination = join(this.paths.runtimesRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
await buildRuntime({
|
||||
pristineRoot: pristine.path,
|
||||
versionsRoot: this.paths.runtimesRoot,
|
||||
runtimeId: id,
|
||||
patchPaths: patches.map(patch => prepared.paths.get(patch.artifactId)),
|
||||
onProgress,
|
||||
})
|
||||
return Object.freeze({ id, sha256: await verifyDirectory(destination), path: destination })
|
||||
}
|
||||
|
||||
async systemPlugins(prepared, environment) {
|
||||
const plugins = prepared.environment.manifest.systemPlugins.map(reference => {
|
||||
const artifact = artifactForReference(prepared.environment.manifest, reference)
|
||||
return { id: reference.id, artifactId: artifact.id, sha256: artifact.sha256 }
|
||||
})
|
||||
const id = deriveRecordId('system-plugins', {
|
||||
schema: 1,
|
||||
environment: environment.sha256,
|
||||
plugins,
|
||||
})
|
||||
const destination = join(this.paths.systemPluginsRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
const buildRoot = join(this.paths.systemPluginsRoot, `.build.${randomUUID()}.tmp`)
|
||||
try {
|
||||
const built = await reconcileSystemPlugins({
|
||||
root: buildRoot,
|
||||
environmentVersion: id,
|
||||
plugins: prepared.environment.manifest.systemPlugins,
|
||||
artifactPath: reference => prepared.paths.get(artifactForReference(prepared.environment.manifest, reference).id),
|
||||
})
|
||||
return await publish(built, this.paths.systemPluginsRoot, id)
|
||||
} finally {
|
||||
await rm(buildRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async buildStable(prepared, { onProgress = async () => {} } = {}) {
|
||||
const pristine = await this.pristine(prepared.dsh.version, prepared.dsh.receipt)
|
||||
await onProgress(80)
|
||||
const environment = await this.environment(prepared)
|
||||
await onProgress(82)
|
||||
const runtime = await this.runtime(prepared, pristine, environment, {
|
||||
onProgress: metrics => onProgress({ stage: 'runtime', ...metrics }),
|
||||
})
|
||||
await onProgress(87)
|
||||
const systemPlugins = await this.systemPlugins(prepared, environment)
|
||||
await onProgress(89)
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: 'stable',
|
||||
targetSequence: prepared.stable.targetSequence,
|
||||
dshVersion: prepared.dsh.version,
|
||||
environmentVersion: prepared.environment.manifest.version,
|
||||
environment: storeReference('environment', environment),
|
||||
pristine: storeReference('pristine', pristine),
|
||||
runtime: storeReference('runtime', runtime),
|
||||
systemPlugins: storeReference('system-plugins', systemPlugins),
|
||||
receiptTokens: [...prepared.receiptTokens],
|
||||
snapshotId: null,
|
||||
}
|
||||
return Object.freeze({
|
||||
record: parseDeploymentRecord({ ...content, id: deriveRecordId('deployment-record', content) }),
|
||||
assets: Object.freeze({ pristine, environment, runtime, systemPlugins }),
|
||||
})
|
||||
}
|
||||
|
||||
async buildExperimental(prepared, currentValue, receiptTokens, { targetSequence }) {
|
||||
const current = parseDeploymentRecord(currentValue)
|
||||
const pristine = await this.pristine(prepared.version, prepared.receipt)
|
||||
const environmentRoot = join(this.paths.viewsRoot, 'environment')
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(environmentRoot, 'environment.manifest.json')))
|
||||
if (manifest.version !== current.environmentVersion) throw new Error('current Environment view differs from its Deployment Record')
|
||||
const patches = manifest.patches.map(reference => {
|
||||
const artifact = artifactForReference(manifest, reference)
|
||||
return { id: reference.id, artifactId: artifact.id, sha256: artifact.sha256 }
|
||||
})
|
||||
const runtimeId = deriveRecordId('runtime', {
|
||||
schema: 1,
|
||||
pristine: pristine.sha256,
|
||||
environment: current.environment.sha256,
|
||||
patches,
|
||||
})
|
||||
const runtimePath = join(this.paths.runtimesRoot, runtimeId)
|
||||
if (!await exists(runtimePath)) {
|
||||
await buildRuntime({
|
||||
pristineRoot: pristine.path,
|
||||
versionsRoot: this.paths.runtimesRoot,
|
||||
runtimeId,
|
||||
patchPaths: patches.map(patch => join(environmentRoot, 'artifacts', patch.artifactId)),
|
||||
})
|
||||
}
|
||||
const runtime = Object.freeze({ id: runtimeId, sha256: await verifyDirectory(runtimePath), path: runtimePath })
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: 'experimental',
|
||||
targetSequence,
|
||||
dshVersion: prepared.version,
|
||||
environmentVersion: current.environmentVersion,
|
||||
environment: current.environment,
|
||||
pristine: storeReference('pristine', pristine),
|
||||
runtime: storeReference('runtime', runtime),
|
||||
systemPlugins: current.systemPlugins,
|
||||
receiptTokens: [...receiptTokens],
|
||||
snapshotId: null,
|
||||
}
|
||||
const record = parseDeploymentRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
return Object.freeze({ record, runtime })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { parseStable } from '../../../../platform/lib/contracts.mjs'
|
||||
import { compareDshVersions } from '../../../../platform/lib/supported-target.mjs'
|
||||
|
||||
export class MetadataUnavailableError extends Error {
|
||||
constructor() {
|
||||
super('signed update metadata has not been published')
|
||||
this.name = 'MetadataUnavailableError'
|
||||
this.code = 'METADATA_UNAVAILABLE'
|
||||
}
|
||||
}
|
||||
|
||||
async function responseBytes(response, label, maxBytes = 10 * 1024 * 1024) {
|
||||
if (!response.ok) throw new HttpResponseError(label, response.status)
|
||||
const bytes = Buffer.from(await response.arrayBuffer())
|
||||
if (bytes.byteLength > maxBytes) throw new Error(`${label} exceeds the download limit`)
|
||||
return bytes
|
||||
}
|
||||
|
||||
class HttpResponseError extends Error {
|
||||
constructor(label, status) {
|
||||
super(`${label} returned HTTP ${String(status)}`)
|
||||
this.name = 'HttpResponseError'
|
||||
this.status = status
|
||||
}
|
||||
}
|
||||
|
||||
function retryableRequest(error) {
|
||||
return !(error instanceof HttpResponseError)
|
||||
|| error.status === 408
|
||||
|| error.status === 429
|
||||
|| error.status >= 500
|
||||
}
|
||||
|
||||
function exhaustedRequest(label, error, attempts) {
|
||||
const timeout = error?.name === 'TimeoutError'
|
||||
const detail = error instanceof Error && error.message !== '' ? `: ${error.message}` : ''
|
||||
const wrapped = new Error(
|
||||
timeout
|
||||
? `${label} timed out after ${String(attempts)} attempts`
|
||||
: `${label} failed after ${String(attempts)} attempts${detail}`,
|
||||
{ cause: error },
|
||||
)
|
||||
wrapped.name = timeout ? 'TimeoutError' : 'Error'
|
||||
if (error?.code !== undefined) wrapped.code = error.code
|
||||
wrapped.requestExhausted = true
|
||||
return wrapped
|
||||
}
|
||||
|
||||
function timeoutForAttempt(initial, retry, attempt) {
|
||||
return attempt === 1 ? initial : retry
|
||||
}
|
||||
|
||||
export class MetadataClient {
|
||||
constructor({
|
||||
baseUrl,
|
||||
trust,
|
||||
fetchImpl = fetch,
|
||||
attempts = 2,
|
||||
retryMs = 250,
|
||||
requestTimeoutMs = 5_000,
|
||||
retryRequestTimeoutMs = requestTimeoutMs,
|
||||
}) {
|
||||
this.baseUrl = new URL(baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`)
|
||||
if (this.baseUrl.protocol !== 'https:' && this.baseUrl.hostname !== '127.0.0.1' && this.baseUrl.hostname !== 'localhost') {
|
||||
throw new Error('update metadata URL must use HTTPS')
|
||||
}
|
||||
this.trust = trust
|
||||
this.fetchImpl = fetchImpl
|
||||
this.attempts = attempts
|
||||
this.retryMs = retryMs
|
||||
this.requestTimeoutMs = requestTimeoutMs
|
||||
this.retryRequestTimeoutMs = retryRequestTimeoutMs
|
||||
}
|
||||
|
||||
async file(name) {
|
||||
let lastError
|
||||
for (let attempt = 1; attempt <= this.attempts; attempt += 1) {
|
||||
try {
|
||||
const response = await this.fetchImpl(new URL(name, this.baseUrl), {
|
||||
signal: AbortSignal.timeout(timeoutForAttempt(this.requestTimeoutMs, this.retryRequestTimeoutMs, attempt)),
|
||||
})
|
||||
if (response.status === 404) throw new MetadataUnavailableError()
|
||||
return await responseBytes(response, name)
|
||||
} catch (error) {
|
||||
if (error instanceof MetadataUnavailableError) throw error
|
||||
if (!retryableRequest(error)) throw error
|
||||
lastError = error
|
||||
if (attempt < this.attempts) await delay(this.retryMs)
|
||||
}
|
||||
}
|
||||
throw exhaustedRequest(name, lastError, this.attempts)
|
||||
}
|
||||
|
||||
async verifiedPair(payloadName, signatureName, accept) {
|
||||
let lastError
|
||||
for (let attempt = 1; attempt <= this.attempts; attempt += 1) {
|
||||
try {
|
||||
const [payload, signatureBytes] = await Promise.all([
|
||||
this.file(payloadName), this.file(signatureName),
|
||||
])
|
||||
await accept(payload, JSON.parse(signatureBytes.toString('utf8')))
|
||||
return payload
|
||||
} catch (error) {
|
||||
if (error instanceof MetadataUnavailableError
|
||||
|| error instanceof HttpResponseError
|
||||
|| error?.requestExhausted === true) throw error
|
||||
lastError = error
|
||||
if (attempt < this.attempts) await delay(this.retryMs)
|
||||
}
|
||||
}
|
||||
throw lastError
|
||||
}
|
||||
|
||||
async check() {
|
||||
await this.verifiedPair('keyring.json', 'keyring.sig.json', (bytes, signature) => (
|
||||
this.trust.acceptKeyring(bytes, signature)
|
||||
))
|
||||
const stable = await this.verifiedPair('stable.json', 'stable.sig.json', (bytes, signature) => (
|
||||
this.trust.acceptTarget(bytes, signature)
|
||||
))
|
||||
return Object.freeze({ bytes: stable, value: parseStable(stable) })
|
||||
}
|
||||
}
|
||||
|
||||
export class NpmRegistryClient {
|
||||
constructor({
|
||||
fetchImpl = fetch,
|
||||
registry = 'https://registry.npmjs.org/',
|
||||
packageName = '@deepseek-ai/dsh',
|
||||
attempts = 2,
|
||||
retryMs = 250,
|
||||
requestTimeoutMs = 5_000,
|
||||
retryRequestTimeoutMs = requestTimeoutMs,
|
||||
}) {
|
||||
this.fetchImpl = fetchImpl
|
||||
this.registry = registry
|
||||
this.packageName = packageName
|
||||
this.attempts = attempts
|
||||
this.retryMs = retryMs
|
||||
this.requestTimeoutMs = requestTimeoutMs
|
||||
this.retryRequestTimeoutMs = retryRequestTimeoutMs
|
||||
}
|
||||
|
||||
async discover(policy = {}) {
|
||||
const registry = policy.registry ?? this.registry
|
||||
const packageName = policy.packageName ?? this.packageName
|
||||
const packagePath = encodeURIComponent(packageName)
|
||||
let bytes
|
||||
let lastError
|
||||
for (let attempt = 1; attempt <= this.attempts; attempt += 1) {
|
||||
try {
|
||||
const response = await this.fetchImpl(new URL(packagePath, registry), {
|
||||
headers: { accept: 'application/vnd.npm.install-v1+json' },
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(timeoutForAttempt(this.requestTimeoutMs, this.retryRequestTimeoutMs, attempt)),
|
||||
})
|
||||
bytes = await responseBytes(response, 'npm packument', 20 * 1024 * 1024)
|
||||
break
|
||||
} catch (error) {
|
||||
if (!retryableRequest(error)) throw error
|
||||
lastError = error
|
||||
if (attempt < this.attempts) await delay(this.retryMs)
|
||||
}
|
||||
}
|
||||
if (bytes === undefined) throw exhaustedRequest('npm packument', lastError, this.attempts)
|
||||
let packument
|
||||
try { packument = JSON.parse(bytes.toString('utf8')) } catch { throw new Error('npm packument is not valid JSON') }
|
||||
const latest = packument?.['dist-tags']?.latest
|
||||
const version = typeof latest === 'string' ? packument?.versions?.[latest]?.version : undefined
|
||||
if (version !== latest) throw new Error('npm packument has no coherent latest DSH version')
|
||||
return Object.freeze({ version })
|
||||
}
|
||||
|
||||
async latest(stable) {
|
||||
const found = await this.discover(stable.officialDshPolicy)
|
||||
const version = found.version
|
||||
return compareDshVersions(version, stable.desired.dsh.version) > 0 ? Object.freeze({ version }) : null
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
import { constants } from 'node:fs'
|
||||
import { createWriteStream } from 'node:fs'
|
||||
import { mkdir, open, readFile, rename, rm } from 'node:fs/promises'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { join } from 'node:path'
|
||||
import { Readable, Transform } from 'node:stream'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import {
|
||||
OFFICIAL_DSH_METADATA_LIMIT,
|
||||
OFFICIAL_DSH_PACKAGE,
|
||||
OFFICIAL_DSH_TARBALL_LIMIT,
|
||||
officialDshMetadataUrl,
|
||||
officialDshTarballUrl,
|
||||
} from '../../../../platform/lib/official-dsh-contracts.mjs'
|
||||
import { compareDshVersions } from '../../../../platform/lib/supported-target.mjs'
|
||||
|
||||
function exactResponse(response, url, label) {
|
||||
if (response.redirected || (response.url !== undefined && response.url !== '' && response.url !== url.href)) {
|
||||
throw new Error(`${label} changed origin or redirected`)
|
||||
}
|
||||
if (!response.ok) {
|
||||
const error = new Error(`${label} returned HTTP ${String(response.status)}`)
|
||||
error.status = response.status
|
||||
throw error
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
function responseLength(response) {
|
||||
const header = response.headers?.get?.('content-length')
|
||||
if (header === null || header === undefined || header === '') return null
|
||||
const value = Number(header)
|
||||
return Number.isSafeInteger(value) && value >= 0 ? value : null
|
||||
}
|
||||
|
||||
async function boundedBytes(response, label, maxBytes) {
|
||||
const declared = responseLength(response)
|
||||
if (declared !== null && declared > maxBytes) throw new Error(`${label} exceeds the download limit`)
|
||||
const bytes = Buffer.from(await response.arrayBuffer())
|
||||
if (bytes.byteLength > maxBytes) throw new Error(`${label} exceeds the download limit`)
|
||||
return bytes
|
||||
}
|
||||
|
||||
async function atomicBytes(path, bytes) {
|
||||
const temporary = `${path}.${randomUUID()}.tmp`
|
||||
try {
|
||||
const handle = await open(temporary, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 0o600)
|
||||
try {
|
||||
await handle.writeFile(bytes)
|
||||
await handle.sync()
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
await rename(temporary, path)
|
||||
} catch (error) {
|
||||
await rm(temporary, { force: true }).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function streamToFile(response, path, label, maxBytes, onProgress) {
|
||||
if (response.body === null || response.body === undefined) throw new Error(`${label} has no response body`)
|
||||
const declared = responseLength(response)
|
||||
if (declared !== null && declared > maxBytes) throw new Error(`${label} exceeds the download limit`)
|
||||
const temporary = `${path}.${randomUUID()}.tmp`
|
||||
let processedBytes = 0
|
||||
try {
|
||||
const meter = new Transform({
|
||||
transform(chunk, _encoding, callback) {
|
||||
processedBytes += chunk.byteLength
|
||||
if (processedBytes > maxBytes) {
|
||||
callback(new Error(`${label} exceeds the download limit`))
|
||||
return
|
||||
}
|
||||
Promise.resolve(onProgress({ processedBytes, totalBytes: Number.isFinite(declared) ? declared : null }))
|
||||
.then(() => callback(null, chunk), callback)
|
||||
},
|
||||
})
|
||||
await pipeline(Readable.fromWeb(response.body), meter, createWriteStream(temporary, { flags: 'wx', mode: 0o600 }))
|
||||
const handle = await open(temporary, constants.O_RDWR)
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
await rename(temporary, path)
|
||||
return { processedBytes, totalBytes: declared }
|
||||
} catch (error) {
|
||||
await rm(temporary, { force: true }).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export class OfficialDshDownloader {
|
||||
constructor({ fetchImpl = fetch, attempts = 3, retryMs = 250 }) {
|
||||
this.fetchImpl = fetchImpl
|
||||
this.attempts = attempts
|
||||
this.retryMs = retryMs
|
||||
}
|
||||
|
||||
async retry(operation) {
|
||||
let lastError
|
||||
for (let attempt = 1; attempt <= this.attempts; attempt += 1) {
|
||||
try {
|
||||
return await operation(attempt)
|
||||
} catch (error) {
|
||||
lastError = error
|
||||
const status = error?.status
|
||||
if (error?.retryable === false
|
||||
|| (Number.isInteger(status) && status < 500 && status !== 408 && status !== 429)
|
||||
|| attempt === this.attempts) {
|
||||
throw error
|
||||
}
|
||||
await delay(this.retryMs)
|
||||
}
|
||||
}
|
||||
throw lastError
|
||||
}
|
||||
|
||||
async download(version, root, { onProgress = async () => {} } = {}) {
|
||||
compareDshVersions(version, version)
|
||||
await mkdir(root, { recursive: true })
|
||||
const metadataUrl = officialDshMetadataUrl()
|
||||
const metadata = await this.retry(async () => {
|
||||
const metadataResponse = exactResponse(await this.fetchImpl(metadataUrl, {
|
||||
headers: { accept: 'application/vnd.npm.install-v1+json', 'accept-encoding': 'identity' },
|
||||
redirect: 'error',
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
}), metadataUrl, 'official DSH metadata')
|
||||
return boundedBytes(metadataResponse, 'official DSH metadata', OFFICIAL_DSH_METADATA_LIMIT)
|
||||
})
|
||||
let packument
|
||||
try { packument = JSON.parse(metadata.toString('utf8')) } catch { throw new Error('official DSH metadata is not valid JSON') }
|
||||
const selected = packument?.versions?.[version]
|
||||
if (selected?.name !== OFFICIAL_DSH_PACKAGE || selected?.version !== version) {
|
||||
throw new Error('official DSH metadata has no coherent requested version')
|
||||
}
|
||||
const tarballUrl = officialDshTarballUrl(version)
|
||||
if (selected?.dist?.tarball !== tarballUrl.href) throw new Error('official DSH tarball URL is not canonical')
|
||||
const metadataPath = join(root, 'official-dsh-packument.json')
|
||||
const tarballPath = join(root, 'official-dsh.tgz')
|
||||
await atomicBytes(metadataPath, metadata)
|
||||
try {
|
||||
let reportedBytes = metadata.byteLength
|
||||
const result = await this.retry(async () => {
|
||||
const tarballResponse = exactResponse(await this.fetchImpl(tarballUrl, {
|
||||
headers: { accept: 'application/octet-stream', 'accept-encoding': 'identity' },
|
||||
redirect: 'error',
|
||||
}), tarballUrl, 'official DSH tarball')
|
||||
if (tarballResponse.headers?.get?.('content-encoding')) {
|
||||
const error = new Error('official DSH tarball must not use content encoding')
|
||||
error.retryable = false
|
||||
throw error
|
||||
}
|
||||
const declaredTarball = responseLength(tarballResponse)
|
||||
const knownTotal = declaredTarball === null ? null : metadata.byteLength + declaredTarball
|
||||
await onProgress({
|
||||
processedBytes: reportedBytes,
|
||||
totalBytes: knownTotal,
|
||||
processedItems: 1,
|
||||
totalItems: 2,
|
||||
detail: 'official-dsh-metadata',
|
||||
})
|
||||
return streamToFile(
|
||||
tarballResponse,
|
||||
tarballPath,
|
||||
'official DSH tarball',
|
||||
OFFICIAL_DSH_TARBALL_LIMIT,
|
||||
metrics => {
|
||||
reportedBytes = Math.max(reportedBytes, metadata.byteLength + metrics.processedBytes)
|
||||
return onProgress({
|
||||
processedBytes: reportedBytes,
|
||||
totalBytes: knownTotal,
|
||||
processedItems: 1,
|
||||
totalItems: 2,
|
||||
detail: 'official-dsh-tarball',
|
||||
})
|
||||
},
|
||||
)
|
||||
})
|
||||
await onProgress({
|
||||
processedBytes: Math.max(reportedBytes, metadata.byteLength + result.processedBytes),
|
||||
totalBytes: result.totalBytes === null ? null : metadata.byteLength + result.totalBytes,
|
||||
processedItems: 2,
|
||||
totalItems: 2,
|
||||
detail: 'official-dsh-tarball',
|
||||
})
|
||||
return Object.freeze({ metadataPath, tarballPath })
|
||||
} catch (error) {
|
||||
await rm(metadataPath, { force: true }).catch(() => {})
|
||||
await rm(tarballPath, { force: true }).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { createWriteStream } from 'node:fs'
|
||||
import { link, mkdir, open, readFile, rm } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { Readable, Transform } from 'node:stream'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import { setTimeout as delay } from 'node:timers/promises'
|
||||
import { parseBootstrapManifest, parseEnvironmentManifest } from '../../../../platform/lib/contracts.mjs'
|
||||
import { OfficialDshDownloader } from './official-dsh-download.mjs'
|
||||
|
||||
async function verifyExisting(path, descriptor) {
|
||||
const handle = await open(path, 'r')
|
||||
const hash = createHash('sha256')
|
||||
let size = 0
|
||||
try {
|
||||
for await (const chunk of handle.createReadStream({ autoClose: false })) {
|
||||
size += chunk.byteLength
|
||||
if (size > descriptor.size) return false
|
||||
hash.update(chunk)
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return size === descriptor.size && hash.digest('hex') === descriptor.sha256
|
||||
}
|
||||
|
||||
async function download(descriptor, destination, fetchImpl, onChunk) {
|
||||
let lastError
|
||||
let reportedBytes = 0
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
const temporary = `${destination}.${randomUUID()}.tmp`
|
||||
try {
|
||||
const response = await fetchImpl(descriptor.url, {
|
||||
headers: { 'accept-encoding': 'identity' },
|
||||
})
|
||||
if (!response.ok) {
|
||||
const error = new Error(`Artifact ${descriptor.id} returned HTTP ${String(response.status)}`)
|
||||
error.status = response.status
|
||||
throw error
|
||||
}
|
||||
if (response.body === null || response.body === undefined) throw new Error(`Artifact ${descriptor.id} has no response body`)
|
||||
const contentEncoding = response.headers?.get?.('content-encoding')
|
||||
if (contentEncoding !== null && contentEncoding !== undefined && contentEncoding !== '') {
|
||||
const error = new Error(`Artifact ${descriptor.id} must not use content encoding`)
|
||||
error.retryable = false
|
||||
throw error
|
||||
}
|
||||
const declaredHeader = response.headers?.get?.('content-length')
|
||||
if (declaredHeader !== null && declaredHeader !== undefined && declaredHeader !== '') {
|
||||
const declared = Number(declaredHeader)
|
||||
if (!Number.isSafeInteger(declared) || declared !== descriptor.size) {
|
||||
throw new Error(`Artifact ${descriptor.id} does not match stable metadata`)
|
||||
}
|
||||
}
|
||||
const hash = createHash('sha256')
|
||||
let size = 0
|
||||
const meter = new Transform({
|
||||
transform(chunk, _encoding, callback) {
|
||||
size += chunk.byteLength
|
||||
if (size > descriptor.size) {
|
||||
callback(new Error(`Artifact ${descriptor.id} does not match stable metadata`))
|
||||
return
|
||||
}
|
||||
hash.update(chunk)
|
||||
reportedBytes = Math.max(reportedBytes, size)
|
||||
Promise.resolve(onChunk(reportedBytes)).then(() => callback(null, chunk), callback)
|
||||
},
|
||||
})
|
||||
await pipeline(Readable.fromWeb(response.body), meter, createWriteStream(temporary, { flags: 'wx', mode: 0o600 }))
|
||||
if (size !== descriptor.size || hash.digest('hex') !== descriptor.sha256) {
|
||||
throw new Error(`Artifact ${descriptor.id} does not match stable metadata`)
|
||||
}
|
||||
const handle = await open(temporary, 'r+')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
try {
|
||||
await link(temporary, destination)
|
||||
await rm(temporary)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST') throw error
|
||||
if (!await verifyExisting(destination, descriptor)) {
|
||||
throw new Error(`existing Artifact ${descriptor.id} differs from the target`)
|
||||
}
|
||||
await rm(temporary, { force: true })
|
||||
}
|
||||
return destination
|
||||
} catch (error) {
|
||||
await rm(temporary, { force: true }).catch(() => {})
|
||||
lastError = error
|
||||
const status = error?.status
|
||||
if (error?.retryable === false || (Number.isInteger(status) && status < 500 && status !== 408 && status !== 429) || attempt === 3) throw error
|
||||
await delay(250)
|
||||
}
|
||||
}
|
||||
throw lastError
|
||||
}
|
||||
|
||||
export class TargetPreparer {
|
||||
constructor({ untrustedRoot, trust, fetchImpl = fetch, officialDsh = new OfficialDshDownloader({ fetchImpl }) }) {
|
||||
this.untrustedRoot = untrustedRoot
|
||||
this.trust = trust
|
||||
this.fetchImpl = fetchImpl
|
||||
this.officialDsh = officialDsh
|
||||
}
|
||||
|
||||
async prepare(stable, { onProgress = async () => {} } = {}) {
|
||||
const taskRoot = join(this.untrustedRoot, `target-${String(stable.targetSequence)}`)
|
||||
await mkdir(taskRoot, { recursive: true })
|
||||
const downloads = new Map()
|
||||
const paths = new Map()
|
||||
const receipts = new Map()
|
||||
const authorities = new Map()
|
||||
let downloadedBytes = 0
|
||||
let completedDownloads = 0
|
||||
let totalBytes = stable.artifacts.reduce((total, descriptor) => total + descriptor.size, 0)
|
||||
let totalItems = stable.artifacts.length
|
||||
const importDescriptor = async (descriptor, parentReceipt = null) => {
|
||||
const authority = parentReceipt ?? 'stable'
|
||||
if (authorities.has(descriptor.id) && authorities.get(descriptor.id) !== authority) {
|
||||
throw new Error(`Artifact ID ${descriptor.id} is reused across authorities`)
|
||||
}
|
||||
authorities.set(descriptor.id, authority)
|
||||
if (!downloads.has(descriptor.id) && !stable.artifacts.some(root => root.id === descriptor.id)) {
|
||||
totalBytes += descriptor.size
|
||||
totalItems += 1
|
||||
}
|
||||
if (!downloads.has(descriptor.id)) {
|
||||
const downloadBase = downloadedBytes
|
||||
const downloaded = await download(
|
||||
descriptor,
|
||||
join(taskRoot, descriptor.id),
|
||||
this.fetchImpl,
|
||||
async processed => {
|
||||
await onProgress({
|
||||
processedBytes: downloadBase + processed,
|
||||
totalBytes,
|
||||
processedItems: completedDownloads,
|
||||
totalItems,
|
||||
})
|
||||
},
|
||||
)
|
||||
downloadedBytes = downloadBase + descriptor.size
|
||||
downloads.set(descriptor.id, downloaded)
|
||||
completedDownloads += 1
|
||||
await onProgress({ processedBytes: downloadedBytes, totalBytes, processedItems: completedDownloads, totalItems })
|
||||
}
|
||||
if (!receipts.has(descriptor.id)) {
|
||||
const receipt = await this.trust.importArtifact(
|
||||
descriptor.id,
|
||||
downloads.get(descriptor.id),
|
||||
parentReceipt,
|
||||
)
|
||||
receipts.set(descriptor.id, receipt)
|
||||
paths.set(descriptor.id, receipt.path)
|
||||
}
|
||||
return receipts.get(descriptor.id)
|
||||
}
|
||||
for (const descriptor of stable.artifacts) await importDescriptor(descriptor)
|
||||
|
||||
const prepareManifest = async (reference, parser) => {
|
||||
const manifestReceipt = receipts.get(reference.manifestArtifactId)
|
||||
const signatureReceipt = receipts.get(reference.signatureArtifactId)
|
||||
await this.trust.acceptManifest(manifestReceipt.token, signatureReceipt.token)
|
||||
const manifest = parser(await readFile(paths.get(reference.manifestArtifactId)))
|
||||
for (const descriptor of manifest.artifacts) await importDescriptor(descriptor, manifestReceipt.token)
|
||||
return { manifest, manifestReceipt }
|
||||
}
|
||||
const bootstrap = await prepareManifest(stable.desired.bootstrap, parseBootstrapManifest)
|
||||
const environment = await prepareManifest(stable.desired.environment, parseEnvironmentManifest)
|
||||
const dshBaseBytes = downloadedBytes
|
||||
const dshBaseItems = completedDownloads
|
||||
const dshSources = await this.officialDsh.download(
|
||||
stable.desired.dsh.version,
|
||||
join(taskRoot, 'official-dsh'),
|
||||
{
|
||||
onProgress: metrics => onProgress({
|
||||
processedBytes: dshBaseBytes + metrics.processedBytes,
|
||||
totalBytes: metrics.totalBytes === null ? null : totalBytes + metrics.totalBytes,
|
||||
processedItems: dshBaseItems + metrics.processedItems,
|
||||
totalItems: totalItems + metrics.totalItems,
|
||||
detail: metrics.detail,
|
||||
}),
|
||||
},
|
||||
)
|
||||
const dshReceipt = await this.trust.ensureOfficialDsh(
|
||||
stable.desired.dsh.version,
|
||||
dshSources.metadataPath,
|
||||
dshSources.tarballPath,
|
||||
)
|
||||
return Object.freeze({
|
||||
stable,
|
||||
paths,
|
||||
receipts,
|
||||
bootstrap,
|
||||
environment,
|
||||
dsh: Object.freeze({ version: stable.desired.dsh.version, receipt: dshReceipt }),
|
||||
receiptTokens: Object.freeze([...receipts.values(), dshReceipt].map(receipt => receipt.token)),
|
||||
})
|
||||
}
|
||||
|
||||
async prepareExperimental(candidate, { onProgress = async () => {} } = {}) {
|
||||
const taskRoot = join(this.untrustedRoot, `experimental-${candidate.version}`)
|
||||
const sources = await this.officialDsh.download(candidate.version, taskRoot, { onProgress })
|
||||
const receipt = await this.trust.ensureOfficialDsh(candidate.version, sources.metadataPath, sources.tarballPath)
|
||||
return Object.freeze({ version: candidate.version, receipt, receiptTokens: Object.freeze([receipt.token]) })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
const TERMINAL_PHASES = new Set(['committed', 'rolled-back', 'failed'])
|
||||
const PRE_SWITCH_PHASES = new Set(['planning', 'candidate-ready', 'suspended'])
|
||||
|
||||
export async function recoverInterruptedUpdate({ journal, snapshots, activator, resume = true }) {
|
||||
const transaction = await journal.read()
|
||||
if (transaction === undefined || TERMINAL_PHASES.has(transaction.phase)) return transaction
|
||||
|
||||
if (PRE_SWITCH_PHASES.has(transaction.phase)) {
|
||||
if (resume) await activator.resumeDsh().catch(() => {})
|
||||
return journal.transition('failed', { error: 'interrupted before Runtime switch' })
|
||||
}
|
||||
|
||||
const restoring = transaction.phase === 'restoring-data'
|
||||
? transaction
|
||||
: await journal.transition('restoring-data', { error: 'recovering interrupted update' })
|
||||
if (resume) await activator.suspendDsh().catch(() => {})
|
||||
await activator.restoreDeployment(restoring.from, { resume: false })
|
||||
if (restoring.snapshotId !== null) await snapshots.restore(restoring.snapshotId)
|
||||
if (resume) await activator.resumeDsh()
|
||||
return journal.transition('rolled-back', { error: 'recovered interrupted update' })
|
||||
}
|
||||
|
||||
export async function reconcileRecoveredState({ journal, state }) {
|
||||
const [transaction, persisted] = await Promise.all([journal.read(), state.read()])
|
||||
const sameTask = transaction !== undefined && persisted.taskId === transaction.transactionId
|
||||
// A metadata check has no journal and cannot be resumed after Management
|
||||
// restarts. Leaving this state behind would keep every UI showing a spinner
|
||||
// and could make startup mistake the check for an interrupted update.
|
||||
if (persisted.status === 'checking' && !sameTask) {
|
||||
return {
|
||||
transaction,
|
||||
persisted: await state.write('idle', {
|
||||
taskId: null,
|
||||
progress: 0,
|
||||
error: null,
|
||||
checkSource: null,
|
||||
}),
|
||||
}
|
||||
}
|
||||
if (sameTask && !['idle', 'success', 'failed'].includes(persisted.status)) {
|
||||
if (transaction.phase === 'committed') {
|
||||
return { transaction, persisted: await state.write('success', { progress: 100, error: null }) }
|
||||
}
|
||||
if (['rolled-back', 'failed'].includes(transaction.phase)) {
|
||||
return {
|
||||
transaction,
|
||||
persisted: await state.write('failed', { error: transaction.error ?? 'interrupted update was not committed' }),
|
||||
}
|
||||
}
|
||||
}
|
||||
return { transaction, persisted }
|
||||
}
|
||||
|
||||
export function resumeInterruptedReconcile({ coordinator, persisted, report, audit }) {
|
||||
const task = coordinator.startReconcile()
|
||||
const context = {
|
||||
taskId: task.taskId,
|
||||
interruptedTaskId: persisted.taskId ?? null,
|
||||
}
|
||||
const started = Promise.resolve()
|
||||
.then(() => persisted.taskId === null || persisted.taskId === undefined
|
||||
? undefined
|
||||
: audit('update.failed', {
|
||||
taskId: persisted.taskId,
|
||||
outcome: 'interrupted',
|
||||
resumedByTaskId: task.taskId,
|
||||
level: 'warning',
|
||||
}))
|
||||
.then(() => report('update.resume.started', context))
|
||||
.catch(() => {})
|
||||
void task.completion.then(
|
||||
async () => {
|
||||
await started
|
||||
await report('update.resume.completed', context)
|
||||
},
|
||||
async error => {
|
||||
await started
|
||||
await report('update.resume.failed', { ...context, error })
|
||||
},
|
||||
).catch(() => {})
|
||||
return task
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { canonicalJson } from '../../../../platform/lib/canonical-json.mjs'
|
||||
import { TrustError } from '../../../../platform/lib/validation.mjs'
|
||||
|
||||
function deploymentIdentity(deployment) {
|
||||
return {
|
||||
dsh: deployment.dsh,
|
||||
environment: deployment.environment,
|
||||
runtime: deployment.runtime,
|
||||
receiptTokens: [...deployment.receiptTokens].sort(),
|
||||
}
|
||||
}
|
||||
|
||||
function recordIdentity(record) {
|
||||
return deploymentIdentity({
|
||||
dsh: record.dshVersion,
|
||||
environment: record.environmentVersion,
|
||||
runtime: record.id,
|
||||
receiptTokens: record.receiptTokens,
|
||||
})
|
||||
}
|
||||
|
||||
function recordMatchesDeployment(record, deployment) {
|
||||
return record.dshVersion === deployment.dsh
|
||||
&& record.environmentVersion === deployment.environment
|
||||
&& [...record.receiptTokens].sort().join('\0') === [...deployment.receiptTokens].sort().join('\0')
|
||||
}
|
||||
|
||||
function sameRecordContent(left, right) {
|
||||
return left.authority === right.authority
|
||||
&& left.targetSequence === right.targetSequence
|
||||
&& left.dshVersion === right.dshVersion
|
||||
&& left.environmentVersion === right.environmentVersion
|
||||
&& left.environment.sha256 === right.environment.sha256
|
||||
&& left.pristine.sha256 === right.pristine.sha256
|
||||
&& left.runtime.sha256 === right.runtime.sha256
|
||||
&& left.systemPlugins.sha256 === right.systemPlugins.sha256
|
||||
}
|
||||
|
||||
function bindPlan(value) {
|
||||
return Object.freeze({
|
||||
...value,
|
||||
planId: createHash('sha256').update(canonicalJson(value)).digest('hex'),
|
||||
})
|
||||
}
|
||||
|
||||
export class CompleteStateRecovery {
|
||||
constructor({ journal, snapshots, activator }) {
|
||||
this.journal = journal
|
||||
this.snapshots = snapshots
|
||||
this.activator = activator
|
||||
}
|
||||
|
||||
async plan() {
|
||||
const transaction = await this.journal.read()
|
||||
if (transaction?.phase === 'committed' && transaction.snapshotId !== null) {
|
||||
const current = await this.activator.currentDeployment()
|
||||
if (
|
||||
current.dsh === transaction.to.dsh
|
||||
&& current.environment === transaction.to.environment
|
||||
&& current.runtime === transaction.to.runtime
|
||||
) {
|
||||
let previous = deploymentIdentity(transaction.from)
|
||||
if (this.activator.rollbackDeployments !== undefined) {
|
||||
const slots = await this.activator.rollbackDeployments()
|
||||
if (
|
||||
slots.current?.id !== current.runtime
|
||||
|| slots.previous === null
|
||||
|| !recordMatchesDeployment(slots.previous, transaction.from)
|
||||
) return null
|
||||
previous = recordIdentity(slots.previous)
|
||||
}
|
||||
const snapshot = await this.snapshots.inspect(transaction.snapshotId)
|
||||
if (
|
||||
snapshot.runtimeId !== transaction.from.runtime
|
||||
|| snapshot.environmentVersion !== transaction.from.environment
|
||||
|| snapshot.dshVersion !== transaction.from.dsh
|
||||
) throw new TrustError('rollback snapshot does not describe the previous deployment')
|
||||
return bindPlan({
|
||||
transactionId: transaction.transactionId,
|
||||
mode: transaction.mode,
|
||||
current: deploymentIdentity(current),
|
||||
previous,
|
||||
snapshot: {
|
||||
id: snapshot.id,
|
||||
createdAt: snapshot.createdAt,
|
||||
sha256: snapshot.archiveSha256,
|
||||
size: snapshot.archiveSize,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
if (this.activator.rollbackDeployments === undefined) return null
|
||||
const { current, previous } = await this.activator.rollbackDeployments()
|
||||
if (current === null || previous === null) return null
|
||||
if (sameRecordContent(current, previous)) return null
|
||||
return bindPlan({
|
||||
transactionId: null,
|
||||
mode: 'stable',
|
||||
current: recordIdentity(current),
|
||||
previous: recordIdentity(previous),
|
||||
snapshot: null,
|
||||
})
|
||||
}
|
||||
|
||||
async restore(planId, {
|
||||
confirmDataLoss = false,
|
||||
requireConfirmation = false,
|
||||
onProgress = async () => {},
|
||||
} = {}) {
|
||||
if (requireConfirmation && confirmDataLoss !== true) {
|
||||
throw new TrustError('return to Stable requires explicit data-loss confirmation')
|
||||
}
|
||||
const plan = await this.plan()
|
||||
if (plan === null || plan.planId !== planId) throw new TrustError('rollback plan is stale or unavailable')
|
||||
if (requireConfirmation && plan.snapshot === null) {
|
||||
throw new TrustError('return to Stable requires a verified data snapshot')
|
||||
}
|
||||
if (plan.snapshot === null) {
|
||||
await onProgress('switching', 35, {})
|
||||
await this.activator.rollback(plan.previous.runtime)
|
||||
await onProgress('verifying', 90, {})
|
||||
return Object.freeze({ status: 'rolled-back', transactionId: null })
|
||||
}
|
||||
let transaction = await this.journal.transition('restoring-data', { error: 'manual complete-state rollback' })
|
||||
await onProgress('stopping', 15, {})
|
||||
await this.activator.suspendDsh()
|
||||
try {
|
||||
await onProgress('switching', 35, {})
|
||||
await this.activator.restoreDeployment({ ...transaction.from, runtime: plan.previous.runtime }, { resume: false })
|
||||
await onProgress('restoring-data', 65, {})
|
||||
await this.snapshots.restore(transaction.snapshotId, {
|
||||
onProgress: metrics => onProgress('restoring-data', 65, metrics),
|
||||
})
|
||||
await onProgress('verifying', 85, {})
|
||||
await this.activator.resumeDsh()
|
||||
transaction = await this.journal.transition('rolled-back', { error: 'manual complete-state rollback completed' })
|
||||
return Object.freeze({ status: 'rolled-back', transactionId: transaction.transactionId })
|
||||
} catch (error) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
export class UpdateScheduler {
|
||||
constructor({
|
||||
check,
|
||||
intervalSeconds = 21_600,
|
||||
onError = async () => {},
|
||||
random = Math.random,
|
||||
setTimer = setTimeout,
|
||||
clearTimer = clearTimeout,
|
||||
}) {
|
||||
if (!Number.isSafeInteger(intervalSeconds) || intervalSeconds < 60) throw new Error('update interval must be at least 60 seconds')
|
||||
this.check = check
|
||||
this.onError = onError
|
||||
this.intervalMs = intervalSeconds * 1_000
|
||||
this.random = random
|
||||
this.setTimer = setTimer
|
||||
this.clearTimer = clearTimer
|
||||
this.timer = undefined
|
||||
this.enabled = false
|
||||
}
|
||||
|
||||
schedule() {
|
||||
const jitter = 0.9 + this.random() * 0.2
|
||||
const timer = this.setTimer(async () => {
|
||||
if (this.timer !== timer) return
|
||||
this.timer = undefined
|
||||
try {
|
||||
await this.check()
|
||||
} catch (error) {
|
||||
await Promise.resolve().then(() => this.onError(error)).catch(() => {})
|
||||
}
|
||||
if (this.enabled && this.timer === undefined) this.schedule()
|
||||
}, Math.round(this.intervalMs * jitter))
|
||||
this.timer = timer
|
||||
timer.unref?.()
|
||||
}
|
||||
|
||||
start() {
|
||||
this.enabled = true
|
||||
if (this.timer === undefined) this.schedule()
|
||||
}
|
||||
|
||||
configure({ enabled, intervalSeconds }) {
|
||||
if (typeof enabled !== 'boolean') throw new Error('automatic check enabled must be boolean')
|
||||
if (!Number.isSafeInteger(intervalSeconds) || intervalSeconds < 60) throw new Error('update interval must be at least 60 seconds')
|
||||
this.stop()
|
||||
this.intervalMs = intervalSeconds * 1_000
|
||||
if (enabled) this.start()
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.enabled = false
|
||||
if (this.timer !== undefined) this.clearTimer(this.timer)
|
||||
this.timer = undefined
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
export class SnapshotClient {
|
||||
constructor(api, scope) {
|
||||
if (!['dsh', 'user-plugin'].includes(scope)) throw new Error('snapshot scope is invalid')
|
||||
this.api = api
|
||||
this.scope = scope
|
||||
}
|
||||
|
||||
async call(operation, body, onProgress) {
|
||||
const response = await this.api.request('POST', `/v1/${this.scope}/${operation}`, body)
|
||||
if (response.progress !== null && onProgress !== undefined) await onProgress(response.progress)
|
||||
return response.result
|
||||
}
|
||||
|
||||
create(value = undefined) {
|
||||
if (this.scope === 'dsh') {
|
||||
const { onProgress, ...body } = value
|
||||
return this.call('create', body, onProgress)
|
||||
}
|
||||
return this.call('create', { id: value })
|
||||
}
|
||||
|
||||
inspect(id) { return this.call('inspect', { id }) }
|
||||
|
||||
restore(id, { onProgress } = {}) {
|
||||
return this.call('restore', { id }, onProgress)
|
||||
}
|
||||
|
||||
remove(id) { return this.call('remove', { id }) }
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { lstat, mkdir, open, readFile, readdir, rename, rm } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { canonicalJson } from '../../../../platform/lib/canonical-json.mjs'
|
||||
import { exactKeys, isoTimestamp, parseJsonDocument, plainObject, TrustError } from '../../../../platform/lib/validation.mjs'
|
||||
|
||||
function validId(value) {
|
||||
if (typeof value !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(value)) {
|
||||
throw new TrustError('snapshot ID is invalid')
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function version(value, label) {
|
||||
if (typeof value !== 'string' || value === '' || value.length > 128 || /[\0/\\]/.test(value)) {
|
||||
throw new TrustError(`${label} is invalid`)
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
function parseManifest(bytes) {
|
||||
const value = parseJsonDocument(bytes, 'snapshot manifest')
|
||||
exactKeys(value, [
|
||||
'archiveSha256', 'archiveSize', 'createdAt', 'dshVersion', 'environmentVersion',
|
||||
'id', 'runtimeId', 'schema', 'source',
|
||||
], 'snapshot manifest')
|
||||
if (value.schema !== 1) throw new TrustError('snapshot manifest schema must be 1')
|
||||
validId(value.id)
|
||||
if (value.source !== '/data/dsh') throw new TrustError('snapshot source is invalid')
|
||||
if (typeof value.archiveSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.archiveSha256)) {
|
||||
throw new TrustError('snapshot archive SHA-256 is invalid')
|
||||
}
|
||||
if (!Number.isSafeInteger(value.archiveSize) || value.archiveSize < 0) {
|
||||
throw new TrustError('snapshot archive size is invalid')
|
||||
}
|
||||
isoTimestamp(value.createdAt, 'snapshot creation time')
|
||||
version(value.runtimeId, 'snapshot runtime ID')
|
||||
version(value.environmentVersion, 'snapshot Environment version')
|
||||
version(value.dshVersion, 'snapshot DSH version')
|
||||
return Object.freeze(value)
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolveRun, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolveRun(Buffer.concat(stdout).toString('utf8'))
|
||||
: reject(new Error(`${command} failed: ${Buffer.concat(stderr).toString('utf8').trim()}`)))
|
||||
})
|
||||
}
|
||||
|
||||
async function hashFile(path) {
|
||||
const handle = await open(path, 'r')
|
||||
const hash = createHash('sha256')
|
||||
let size = 0
|
||||
try {
|
||||
for await (const chunk of handle.createReadStream({ autoClose: false })) {
|
||||
hash.update(chunk)
|
||||
size += chunk.byteLength
|
||||
}
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return { sha256: hash.digest('hex'), size }
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
async function countTree(path) {
|
||||
const details = await lstat(path)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) return 1
|
||||
let count = 0
|
||||
for (const entry of await readdir(path)) count += await countTree(join(path, entry))
|
||||
return count
|
||||
}
|
||||
|
||||
export class PersistentStateSnapshots {
|
||||
constructor({ root, sourceRoot = '/data/dsh', now = () => new Date(), runImpl = run }) {
|
||||
this.root = resolve(root)
|
||||
this.sourceRoot = resolve(sourceRoot)
|
||||
this.now = now
|
||||
this.run = runImpl
|
||||
}
|
||||
|
||||
path(id) {
|
||||
return join(this.root, 'versions', validId(id))
|
||||
}
|
||||
|
||||
async create({ id = randomUUID(), runtimeId, environmentVersion, dshVersion, onProgress = async () => {} }) {
|
||||
validId(id)
|
||||
const sourceDetails = await lstat(this.sourceRoot)
|
||||
if (!sourceDetails.isDirectory() || sourceDetails.isSymbolicLink()) {
|
||||
throw new TrustError('snapshot source must be a real directory')
|
||||
}
|
||||
const versions = join(this.root, 'versions')
|
||||
const destination = this.path(id)
|
||||
await lstat(destination).then(
|
||||
() => { throw new TrustError(`snapshot ${id} already exists`) },
|
||||
error => { if (error?.code !== 'ENOENT') throw error },
|
||||
)
|
||||
await mkdir(versions, { recursive: true })
|
||||
const staging = join(versions, `.${id}.${randomUUID()}.tmp`)
|
||||
await mkdir(staging)
|
||||
const archive = join(staging, 'data.tar.gz')
|
||||
try {
|
||||
const totalItems = await countTree(this.sourceRoot)
|
||||
await onProgress({ processedItems: 0, totalItems })
|
||||
await this.run('tar', ['-czpf', archive, '-C', this.sourceRoot, '.'])
|
||||
await onProgress({ processedItems: totalItems, totalItems })
|
||||
const measured = await hashFile(archive)
|
||||
const manifest = canonicalJson({
|
||||
schema: 1,
|
||||
id,
|
||||
createdAt: this.now().toISOString(),
|
||||
source: '/data/dsh',
|
||||
runtimeId: version(runtimeId, 'snapshot runtime ID'),
|
||||
environmentVersion: version(environmentVersion, 'snapshot Environment version'),
|
||||
dshVersion: version(dshVersion, 'snapshot DSH version'),
|
||||
archiveSha256: measured.sha256,
|
||||
archiveSize: measured.size,
|
||||
})
|
||||
parseManifest(manifest)
|
||||
const handle = await open(join(staging, 'manifest.json'), 'wx', 0o600)
|
||||
try { await handle.writeFile(manifest); await handle.sync() } finally { await handle.close() }
|
||||
await syncDirectory(staging)
|
||||
await rename(staging, destination)
|
||||
await syncDirectory(versions)
|
||||
return Object.freeze({ ...parseManifest(manifest), path: destination })
|
||||
} catch (error) {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async inspect(id) {
|
||||
const root = this.path(id)
|
||||
const manifest = parseManifest(await readFile(join(root, 'manifest.json')))
|
||||
if (manifest.id !== id) throw new TrustError('snapshot manifest ID does not match its directory')
|
||||
const archive = join(root, 'data.tar.gz')
|
||||
const measured = await hashFile(archive)
|
||||
if (measured.sha256 !== manifest.archiveSha256 || measured.size !== manifest.archiveSize) {
|
||||
throw new TrustError('snapshot archive does not match its manifest', 'TRUST_ARTIFACT_MISMATCH')
|
||||
}
|
||||
const entries = (await this.run('tar', ['-tzf', archive])).split('\n').filter(Boolean)
|
||||
if (entries.length === 0) throw new TrustError('snapshot archive is empty')
|
||||
for (const entry of entries) {
|
||||
if (entry.startsWith('/') || entry.split('/').includes('..')) {
|
||||
throw new TrustError(`snapshot archive path is unsafe: ${entry}`)
|
||||
}
|
||||
}
|
||||
return Object.freeze({ ...manifest, archive })
|
||||
}
|
||||
|
||||
async restore(id, { onProgress = async () => {} } = {}) {
|
||||
const snapshot = await this.inspect(id)
|
||||
await mkdir(this.sourceRoot, { recursive: true })
|
||||
for (const entry of await readdir(this.sourceRoot)) {
|
||||
await rm(join(this.sourceRoot, entry), { recursive: true, force: true })
|
||||
}
|
||||
const entries = (await this.run('tar', ['-tzf', snapshot.archive])).split('\n').filter(Boolean)
|
||||
await onProgress({ processedItems: 0, totalItems: entries.length })
|
||||
const ownership = process.getuid?.() === 0 ? [] : ['--no-same-owner']
|
||||
await this.run('tar', ['-xzpf', snapshot.archive, ...ownership, '-C', this.sourceRoot])
|
||||
await onProgress({ processedItems: entries.length, totalItems: entries.length })
|
||||
await syncDirectory(this.sourceRoot)
|
||||
return snapshot
|
||||
}
|
||||
|
||||
async list() {
|
||||
let names
|
||||
try { names = await readdir(join(this.root, 'versions')) } catch (error) {
|
||||
if (error?.code === 'ENOENT') return []
|
||||
throw error
|
||||
}
|
||||
const snapshots = []
|
||||
for (const name of names.filter(name => !name.startsWith('.')).sort()) snapshots.push(await this.inspect(name))
|
||||
return Object.freeze(snapshots)
|
||||
}
|
||||
|
||||
async remove(id) {
|
||||
try {
|
||||
await this.inspect(id)
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return false
|
||||
throw error
|
||||
}
|
||||
await rm(this.path(id), { recursive: true, force: true })
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { durableReplace } from '../../../../platform/lib/atomic.mjs'
|
||||
|
||||
export const TERMINAL_UPDATE_STATES = new Set(['idle', 'success', 'failed'])
|
||||
|
||||
export class UpdateStateStore {
|
||||
constructor(path, now = () => new Date()) {
|
||||
this.path = path
|
||||
this.now = now
|
||||
}
|
||||
|
||||
async read() {
|
||||
try {
|
||||
return JSON.parse(await readFile(this.path, 'utf8'))
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return { status: 'idle', taskId: null, progress: 0, error: null }
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async write(status, fields = {}) {
|
||||
const previous = await this.read()
|
||||
const next = {
|
||||
...previous,
|
||||
...fields,
|
||||
status,
|
||||
updatedAt: this.now().toISOString(),
|
||||
}
|
||||
await durableReplace(this.path, `${JSON.stringify(next)}\n`)
|
||||
return Object.freeze(next)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user