dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,275 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { cp, lstat, mkdir, readFile, rename, rm, symlink } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { buildRuntime } from '../../patch-manager/index.mjs'
|
||||
import { reconcileSystemPlugins } from '../../plugin-manager/system.mjs'
|
||||
import { artifactForReference, parseEnvironmentManifest } from '../../../../platform/lib/contracts.mjs'
|
||||
import { deriveRecordId, parseDeploymentRecord } from '../../../../platform/lib/deployment-contracts.mjs'
|
||||
import { hashTree } from '../../../../platform/lib/tree-hash.mjs'
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] })
|
||||
const stdout = []
|
||||
const stderr = []
|
||||
child.stdout.on('data', chunk => stdout.push(chunk))
|
||||
child.stderr.on('data', chunk => stderr.push(chunk))
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => code === 0
|
||||
? resolve(Buffer.concat(stdout).toString('utf8'))
|
||||
: reject(new Error(Buffer.concat(stderr).toString('utf8'))))
|
||||
})
|
||||
}
|
||||
|
||||
const NPM_INSTALL_ATTEMPTS = 7
|
||||
const NPM_INSTALL_RETRY_MS = 5_000
|
||||
|
||||
function retryableNpmInstall(error) {
|
||||
return /(?:^|\n)npm error (?:code (?:ETARGET|E404|ENOVERSIONS)|notarget\b)/u.test(
|
||||
error instanceof Error ? error.message : '',
|
||||
)
|
||||
}
|
||||
|
||||
async function verifyDirectory(path, expectedHash) {
|
||||
const details = await lstat(path)
|
||||
if (!details.isDirectory() || details.isSymbolicLink()) throw new Error(`Managed Store asset is not an immutable directory: ${path}`)
|
||||
const sha256 = await hashTree(path)
|
||||
if (expectedHash !== undefined && sha256 !== expectedHash) throw new Error(`Managed Store asset content conflicts at ${path}`)
|
||||
return sha256
|
||||
}
|
||||
|
||||
async function publish(staging, root, id) {
|
||||
const sha256 = await hashTree(staging)
|
||||
const destination = join(root, id)
|
||||
try {
|
||||
await rename(staging, destination)
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST' && error?.code !== 'ENOTEMPTY') throw error
|
||||
await verifyDirectory(destination, sha256)
|
||||
}
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
|
||||
function storeReference(kind, asset) {
|
||||
return Object.freeze({ storage: 'store', kind, id: asset.id, sha256: asset.sha256 })
|
||||
}
|
||||
|
||||
export class ManagedDeploymentBuilder {
|
||||
constructor({
|
||||
paths,
|
||||
npmInstall = (installRoot, archivePath, { preferOnline }) => run('npm', [
|
||||
'install', '--global', '--prefix', installRoot, '--omit=dev', '--ignore-scripts',
|
||||
'--no-audit', '--no-fund', preferOnline ? '--prefer-online' : '--prefer-offline',
|
||||
'--cache', paths.npmCacheRoot, archivePath,
|
||||
]),
|
||||
sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)),
|
||||
}) {
|
||||
this.paths = paths
|
||||
this.npmInstall = npmInstall
|
||||
this.sleep = sleep
|
||||
}
|
||||
|
||||
async pristine(version, receipt) {
|
||||
const id = `pristine-npm-${receipt.objectSha256}`
|
||||
const destination = join(this.paths.pristineRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
const installRoot = join(this.paths.pristineRoot, `.install.${randomUUID()}.tmp`)
|
||||
const staging = join(this.paths.pristineRoot, `.${id}.${randomUUID()}.tmp`)
|
||||
await mkdir(installRoot, { recursive: false })
|
||||
try {
|
||||
const entries = (await run('tar', ['-tzf', receipt.path])).split('\n').filter(Boolean)
|
||||
if (!entries.includes('package/package.json') || entries.some(entry => (
|
||||
entry.startsWith('/') || !entry.startsWith('package/') || entry.split('/').includes('..')
|
||||
))) throw new Error('official DSH archive contains an unsafe path')
|
||||
for (let attempt = 1; attempt <= NPM_INSTALL_ATTEMPTS; attempt += 1) {
|
||||
const attemptRoot = join(installRoot, `attempt-${String(attempt)}`)
|
||||
await mkdir(attemptRoot, { recursive: false })
|
||||
try {
|
||||
const archivePath = join(attemptRoot, 'official-dsh.tgz')
|
||||
await symlink(receipt.path, archivePath, 'file')
|
||||
await this.npmInstall(attemptRoot, archivePath, { preferOnline: attempt > 1 })
|
||||
const installed = join(attemptRoot, 'lib', 'node_modules', '@deepseek-ai', 'dsh')
|
||||
await rename(installed, staging)
|
||||
break
|
||||
} catch (error) {
|
||||
if (attempt === NPM_INSTALL_ATTEMPTS || !retryableNpmInstall(error)) throw error
|
||||
await this.sleep(Math.min(30_000, NPM_INSTALL_RETRY_MS * (2 ** (attempt - 1))))
|
||||
} finally {
|
||||
await rm(attemptRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
const metadata = JSON.parse(await readFile(join(staging, 'package.json'), 'utf8'))
|
||||
if (metadata.name !== '@deepseek-ai/dsh' || metadata.version !== version) {
|
||||
throw new Error('official DSH package metadata differs from its requested version')
|
||||
}
|
||||
return await publish(staging, this.paths.pristineRoot, id)
|
||||
} finally {
|
||||
await rm(installRoot, { recursive: true, force: true })
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async environment(prepared) {
|
||||
const manifest = prepared.environment.manifest
|
||||
const identity = {
|
||||
schema: 1,
|
||||
version: manifest.version,
|
||||
targetSequence: prepared.stable.targetSequence,
|
||||
manifest: prepared.environment.manifestReceipt.objectSha256,
|
||||
artifacts: manifest.artifacts.map(artifact => ({ id: artifact.id, sha256: artifact.sha256 })),
|
||||
}
|
||||
const id = deriveRecordId('environment', identity)
|
||||
const destination = join(this.paths.environmentsRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
const staging = join(this.paths.environmentsRoot, `.${id}.${randomUUID()}.tmp`)
|
||||
await mkdir(join(staging, 'artifacts'), { recursive: true })
|
||||
try {
|
||||
await cp(prepared.paths.get(prepared.stable.desired.environment.manifestArtifactId), join(staging, 'environment.manifest.json'))
|
||||
for (const descriptor of manifest.artifacts) {
|
||||
await cp(prepared.paths.get(descriptor.id), join(staging, 'artifacts', descriptor.id))
|
||||
}
|
||||
return await publish(staging, this.paths.environmentsRoot, id)
|
||||
} finally {
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async runtime(prepared, pristine, environment, { onProgress = async () => {} } = {}) {
|
||||
const patches = prepared.environment.manifest.patches.map(reference => {
|
||||
const artifact = artifactForReference(prepared.environment.manifest, reference)
|
||||
return { id: reference.id, artifactId: artifact.id, sha256: artifact.sha256 }
|
||||
})
|
||||
const id = deriveRecordId('runtime', {
|
||||
schema: 1,
|
||||
pristine: pristine.sha256,
|
||||
environment: environment.sha256,
|
||||
patches,
|
||||
})
|
||||
const destination = join(this.paths.runtimesRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
await buildRuntime({
|
||||
pristineRoot: pristine.path,
|
||||
versionsRoot: this.paths.runtimesRoot,
|
||||
runtimeId: id,
|
||||
patchPaths: patches.map(patch => prepared.paths.get(patch.artifactId)),
|
||||
onProgress,
|
||||
})
|
||||
return Object.freeze({ id, sha256: await verifyDirectory(destination), path: destination })
|
||||
}
|
||||
|
||||
async systemPlugins(prepared, environment) {
|
||||
const plugins = prepared.environment.manifest.systemPlugins.map(reference => {
|
||||
const artifact = artifactForReference(prepared.environment.manifest, reference)
|
||||
return { id: reference.id, artifactId: artifact.id, sha256: artifact.sha256 }
|
||||
})
|
||||
const id = deriveRecordId('system-plugins', {
|
||||
schema: 1,
|
||||
environment: environment.sha256,
|
||||
plugins,
|
||||
})
|
||||
const destination = join(this.paths.systemPluginsRoot, id)
|
||||
if (await exists(destination)) {
|
||||
const sha256 = await verifyDirectory(destination)
|
||||
return Object.freeze({ id, sha256, path: destination })
|
||||
}
|
||||
const buildRoot = join(this.paths.systemPluginsRoot, `.build.${randomUUID()}.tmp`)
|
||||
try {
|
||||
const built = await reconcileSystemPlugins({
|
||||
root: buildRoot,
|
||||
environmentVersion: id,
|
||||
plugins: prepared.environment.manifest.systemPlugins,
|
||||
artifactPath: reference => prepared.paths.get(artifactForReference(prepared.environment.manifest, reference).id),
|
||||
})
|
||||
return await publish(built, this.paths.systemPluginsRoot, id)
|
||||
} finally {
|
||||
await rm(buildRoot, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
async buildStable(prepared, { onProgress = async () => {} } = {}) {
|
||||
const pristine = await this.pristine(prepared.dsh.version, prepared.dsh.receipt)
|
||||
await onProgress(80)
|
||||
const environment = await this.environment(prepared)
|
||||
await onProgress(82)
|
||||
const runtime = await this.runtime(prepared, pristine, environment, {
|
||||
onProgress: metrics => onProgress({ stage: 'runtime', ...metrics }),
|
||||
})
|
||||
await onProgress(87)
|
||||
const systemPlugins = await this.systemPlugins(prepared, environment)
|
||||
await onProgress(89)
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: 'stable',
|
||||
targetSequence: prepared.stable.targetSequence,
|
||||
dshVersion: prepared.dsh.version,
|
||||
environmentVersion: prepared.environment.manifest.version,
|
||||
environment: storeReference('environment', environment),
|
||||
pristine: storeReference('pristine', pristine),
|
||||
runtime: storeReference('runtime', runtime),
|
||||
systemPlugins: storeReference('system-plugins', systemPlugins),
|
||||
receiptTokens: [...prepared.receiptTokens],
|
||||
snapshotId: null,
|
||||
}
|
||||
return Object.freeze({
|
||||
record: parseDeploymentRecord({ ...content, id: deriveRecordId('deployment-record', content) }),
|
||||
assets: Object.freeze({ pristine, environment, runtime, systemPlugins }),
|
||||
})
|
||||
}
|
||||
|
||||
async buildExperimental(prepared, currentValue, receiptTokens, { targetSequence }) {
|
||||
const current = parseDeploymentRecord(currentValue)
|
||||
const pristine = await this.pristine(prepared.version, prepared.receipt)
|
||||
const environmentRoot = join(this.paths.viewsRoot, 'environment')
|
||||
const manifest = parseEnvironmentManifest(await readFile(join(environmentRoot, 'environment.manifest.json')))
|
||||
if (manifest.version !== current.environmentVersion) throw new Error('current Environment view differs from its Deployment Record')
|
||||
const patches = manifest.patches.map(reference => {
|
||||
const artifact = artifactForReference(manifest, reference)
|
||||
return { id: reference.id, artifactId: artifact.id, sha256: artifact.sha256 }
|
||||
})
|
||||
const runtimeId = deriveRecordId('runtime', {
|
||||
schema: 1,
|
||||
pristine: pristine.sha256,
|
||||
environment: current.environment.sha256,
|
||||
patches,
|
||||
})
|
||||
const runtimePath = join(this.paths.runtimesRoot, runtimeId)
|
||||
if (!await exists(runtimePath)) {
|
||||
await buildRuntime({
|
||||
pristineRoot: pristine.path,
|
||||
versionsRoot: this.paths.runtimesRoot,
|
||||
runtimeId,
|
||||
patchPaths: patches.map(patch => join(environmentRoot, 'artifacts', patch.artifactId)),
|
||||
})
|
||||
}
|
||||
const runtime = Object.freeze({ id: runtimeId, sha256: await verifyDirectory(runtimePath), path: runtimePath })
|
||||
const content = {
|
||||
schema: 1,
|
||||
authority: 'experimental',
|
||||
targetSequence,
|
||||
dshVersion: prepared.version,
|
||||
environmentVersion: current.environmentVersion,
|
||||
environment: current.environment,
|
||||
pristine: storeReference('pristine', pristine),
|
||||
runtime: storeReference('runtime', runtime),
|
||||
systemPlugins: current.systemPlugins,
|
||||
receiptTokens: [...receiptTokens],
|
||||
snapshotId: null,
|
||||
}
|
||||
const record = parseDeploymentRecord({ ...content, id: deriveRecordId('deployment-record', content) })
|
||||
return Object.freeze({ record, runtime })
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user