dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,450 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { chmod, mkdir, mkdtemp, readFile, readdir, stat, symlink, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import test from 'node:test'
|
||||
import { FileInventory } from '../../control-plane/modules/file-manager/index.mjs'
|
||||
import { FileTaskManager, fileTaskInternals } from '../../control-plane/modules/file-manager/tasks.mjs'
|
||||
|
||||
async function setup(prefix = 'dsh-file-tasks-') {
|
||||
const root = await mkdtemp(join(tmpdir(), prefix))
|
||||
const files = join(root, 'files')
|
||||
const tasks = join(root, 'tasks')
|
||||
await mkdir(files)
|
||||
const manager = new FileTaskManager({ root: tasks })
|
||||
await manager.initialize()
|
||||
return { root, files, tasks, manager, inventory: new FileInventory() }
|
||||
}
|
||||
|
||||
async function finish(manager, task) {
|
||||
await manager.tasks.get(task.taskId).completion
|
||||
return manager.get(task.taskId)
|
||||
}
|
||||
|
||||
test('runs mkdir, touch, rename, copy, move, and permanent delete tasks', async () => {
|
||||
const { files, manager, inventory } = await setup()
|
||||
assert.equal((await finish(manager, manager.start({ operation: 'mkdir', destination: join(files, 'dir') }))).status, 'success')
|
||||
assert.equal((await stat(join(files, 'dir'))).isDirectory(), true)
|
||||
assert.equal((await finish(manager, manager.start({ operation: 'touch', destination: join(files, 'empty') }))).status, 'success')
|
||||
const empty = await inventory.stat(join(files, 'empty'))
|
||||
const renamed = await finish(manager, manager.start({
|
||||
operation: 'rename', sources: [{ path: empty.path, revision: empty.revision }], destination: join(files, 'renamed'),
|
||||
}))
|
||||
assert.equal(renamed.status, 'success')
|
||||
await writeFile(join(files, 'dir', 'nested.txt'), 'nested')
|
||||
await symlink('nested.txt', join(files, 'dir', 'link'))
|
||||
await mkdir(join(files, 'copies'))
|
||||
const source = await inventory.stat(join(files, 'dir'))
|
||||
const copied = await finish(manager, manager.start({
|
||||
operation: 'copy', sources: [{ path: source.path, revision: source.revision }], destination: join(files, 'copies'), conflict: 'reject',
|
||||
}))
|
||||
assert.equal(copied.status, 'success')
|
||||
assert.equal(await readFile(join(files, 'copies', 'dir', 'nested.txt'), 'utf8'), 'nested')
|
||||
const link = await inventory.stat(join(files, 'copies', 'dir', 'link'))
|
||||
assert.equal(link.type, 'symlink')
|
||||
const movedSource = await inventory.stat(join(files, 'renamed'))
|
||||
const moved = await finish(manager, manager.start({
|
||||
operation: 'move', sources: [{ path: movedSource.path, revision: movedSource.revision }], destination: join(files, 'copies'),
|
||||
}))
|
||||
assert.equal(moved.status, 'success')
|
||||
const deleteSource = await inventory.stat(join(files, 'copies', 'dir'))
|
||||
const removed = await finish(manager, manager.start({ operation: 'delete', sources: [{ path: deleteSource.path, revision: deleteSource.revision }] }))
|
||||
assert.equal(removed.status, 'success')
|
||||
assert.equal(removed.processedBytes, removed.totalBytes)
|
||||
assert.equal(removed.processedEntries, removed.totalEntries)
|
||||
assert.equal(await stat(deleteSource.path).then(() => true, error => error.code !== 'ENOENT'), false)
|
||||
assert.equal((await readdir(join(files, 'copies'))).some(name => name.startsWith('.dsh-')), false)
|
||||
})
|
||||
|
||||
test('releases the mutation lease before publishing a terminal task state', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-file-terminal-lease-'))
|
||||
const files = join(root, 'files')
|
||||
const tasks = join(root, 'tasks')
|
||||
const destination = join(files, 'destination')
|
||||
await mkdir(destination, { recursive: true })
|
||||
await writeFile(join(files, 'item.txt'), 'source')
|
||||
await writeFile(join(destination, 'item.txt'), 'existing')
|
||||
let releaseReport
|
||||
const blockedReport = new Promise(resolve => { releaseReport = resolve })
|
||||
let observeFailure
|
||||
const failurePublished = new Promise(resolve => { observeFailure = resolve })
|
||||
const inventory = new FileInventory()
|
||||
const manager = new FileTaskManager({
|
||||
root: tasks,
|
||||
onState: task => { if (task.status === 'failed') observeFailure() },
|
||||
report: message => message === 'file-task.copy.failed' ? blockedReport : undefined,
|
||||
})
|
||||
await manager.initialize()
|
||||
const source = await inventory.stat(join(files, 'item.txt'))
|
||||
const rejected = manager.start({
|
||||
operation: 'copy', sources: [{ path: source.path, revision: source.revision }], destination, conflict: 'reject',
|
||||
})
|
||||
await failurePublished
|
||||
assert.equal(manager.activeMutation, undefined)
|
||||
assert.equal(manager.queue.some(task => task.taskId === rejected.taskId), false)
|
||||
manager.activeMutation = manager.tasks.get(rejected.taskId)
|
||||
const renamed = manager.start({
|
||||
operation: 'copy', sources: [{ path: source.path, revision: source.revision }], destination, conflict: 'rename',
|
||||
})
|
||||
releaseReport()
|
||||
assert.equal((await finish(manager, renamed)).status, 'success')
|
||||
assert.equal((await finish(manager, rejected)).status, 'failed')
|
||||
assert.equal((await readdir(destination)).length, 2)
|
||||
})
|
||||
|
||||
test('changes file ownership and permissions recursively without following symbolic links', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-file-attributes-'))
|
||||
const files = join(root, 'files')
|
||||
const tasks = join(root, 'tasks')
|
||||
const tree = join(files, 'tree')
|
||||
const nested = join(tree, 'nested')
|
||||
const target = join(root, 'outside.txt')
|
||||
await mkdir(nested, { recursive: true })
|
||||
await writeFile(join(nested, 'inside.txt'), 'inside')
|
||||
await writeFile(target, 'outside')
|
||||
await chmod(target, 0o640)
|
||||
await symlink(target, join(tree, 'outside-link'))
|
||||
const identity = {
|
||||
resolve: async () => ({ user: 'current', group: 'current' }),
|
||||
userId: async value => value === 'current' ? process.getuid() : Promise.reject(new Error('unknown user')),
|
||||
groupId: async value => value === 'current' ? process.getgid() : Promise.reject(new Error('unknown group')),
|
||||
}
|
||||
const inventory = new FileInventory({ identity })
|
||||
const reports = []
|
||||
const manager = new FileTaskManager({ root: tasks, inventory, identity, report: (message, fields) => reports.push({ message, fields }) })
|
||||
await manager.initialize()
|
||||
const source = await inventory.stat(tree)
|
||||
const started = manager.start({
|
||||
operation: 'attributes',
|
||||
sources: [{ path: tree, revision: source.revision }],
|
||||
attributes: { user: 'current', group: 'current', mode: '0750', recursive: true },
|
||||
})
|
||||
const result = await manager.completion(started.taskId)
|
||||
assert.equal(result.status, 'success')
|
||||
assert.equal(result.errorCode, null)
|
||||
assert.equal(result.processedEntries, 4)
|
||||
assert.equal((await stat(tree)).mode & 0o7777, 0o750)
|
||||
assert.equal((await stat(nested)).mode & 0o7777, 0o750)
|
||||
assert.equal((await stat(join(nested, 'inside.txt'))).mode & 0o7777, 0o750)
|
||||
assert.equal((await stat(target)).mode & 0o7777, 0o640)
|
||||
assert.equal(reports.at(-1).message, 'file-task.attributes.completed')
|
||||
})
|
||||
|
||||
test('rejects filesystems that silently ignore Unix ownership or mode changes', () => {
|
||||
const attributes = { uid: 0, gid: 0, mode: 0o600 }
|
||||
assert.doesNotThrow(() => fileTaskInternals.assertAttributesApplied('/supported', { uid: 0, gid: 0, mode: 0o100600 }, attributes, false))
|
||||
assert.throws(
|
||||
() => fileTaskInternals.assertAttributesApplied('/unsupported', { uid: 1000, gid: 1000, mode: 0o100755 }, attributes, false),
|
||||
error => error.code === 'FILE_ATTRIBUTES_UNSUPPORTED' && error.statusCode === 403,
|
||||
)
|
||||
})
|
||||
|
||||
test('reports attribute failures instead of a false completion', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-file-attributes-failed-log-'))
|
||||
const path = join(root, 'item')
|
||||
await writeFile(path, 'value')
|
||||
const identity = {
|
||||
resolve: async () => ({ user: 'current', group: 'current' }),
|
||||
userId: async () => { throw Object.assign(new Error('ownership is unsupported'), { code: 'FILE_ATTRIBUTES_UNSUPPORTED' }) },
|
||||
groupId: async () => process.getgid(),
|
||||
}
|
||||
const inventory = new FileInventory({ identity })
|
||||
const reports = []
|
||||
const manager = new FileTaskManager({ root: join(root, 'tasks'), inventory, identity, report: message => reports.push(message) })
|
||||
await manager.initialize()
|
||||
const source = await inventory.stat(path)
|
||||
const started = manager.start({
|
||||
operation: 'attributes', sources: [{ path, revision: source.revision }],
|
||||
attributes: { user: 'current', group: 'current', mode: '0600', recursive: false },
|
||||
})
|
||||
const result = await manager.completion(started.taskId)
|
||||
assert.equal(result.status, 'failed')
|
||||
assert.equal(result.errorCode, 'FILE_ATTRIBUTES_UNSUPPORTED')
|
||||
assert.deepEqual(reports, ['file-task.attributes.failed'])
|
||||
})
|
||||
|
||||
test('validates attribute requests and resumes an interrupted idempotent change', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-file-attributes-recovery-'))
|
||||
const files = join(root, 'files')
|
||||
const tasks = join(root, 'tasks')
|
||||
await mkdir(files)
|
||||
await mkdir(tasks)
|
||||
const path = join(files, 'item')
|
||||
await writeFile(path, 'value')
|
||||
const identity = {
|
||||
resolve: async () => ({ user: 'current', group: 'current' }),
|
||||
userId: async () => process.getuid(),
|
||||
groupId: async () => process.getgid(),
|
||||
}
|
||||
const inventory = new FileInventory({ identity })
|
||||
const source = await inventory.stat(path)
|
||||
const manager = new FileTaskManager({ root: tasks, inventory, identity })
|
||||
assert.throws(() => manager.start({
|
||||
operation: 'attributes', sources: [{ path, revision: source.revision }],
|
||||
attributes: { user: 'current', group: 'current', mode: '888', recursive: false },
|
||||
}), /attributes/)
|
||||
const task = {
|
||||
schema: 1, taskId: '55555555-5555-4555-8555-555555555555', operation: 'attributes', status: 'running', phase: 'mutating',
|
||||
sources: [{ path, revision: source.revision }], destination: null, destinationRevision: null, conflict: 'reject', managed: false,
|
||||
attributes: { user: 'current', group: 'current', mode: '0600', recursive: false },
|
||||
processedBytes: 0, totalBytes: 5, processedEntries: 0, totalEntries: 1,
|
||||
published: [], staging: [], hidden: [], currentPath: path, currentSource: null, currentDestination: null,
|
||||
error: null, createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), cancelRequested: false,
|
||||
}
|
||||
await writeFile(join(tasks, `${task.taskId}.json`), JSON.stringify(task))
|
||||
const recovered = new FileTaskManager({ root: tasks, inventory, identity })
|
||||
await recovered.initialize()
|
||||
assert.equal(recovered.get(task.taskId).status, 'success')
|
||||
assert.equal((await stat(path)).mode & 0o7777, 0o600)
|
||||
})
|
||||
|
||||
test('rejects stale revisions, protected roots, and recursive copies while queueing concurrent mutations', async () => {
|
||||
const { files, manager, inventory } = await setup()
|
||||
const sourcePath = join(files, 'source')
|
||||
await writeFile(sourcePath, 'one')
|
||||
const source = await inventory.stat(sourcePath)
|
||||
await writeFile(sourcePath, 'two')
|
||||
const stale = manager.start({ operation: 'delete', sources: [{ path: sourcePath, revision: source.revision }] })
|
||||
assert.equal((await finish(manager, stale)).status, 'failed')
|
||||
assert.throws(() => manager.start({ operation: 'delete', sources: [{ path: '/', revision: 'x' }] }), error => error.statusCode === 403)
|
||||
await mkdir(join(files, 'tree'))
|
||||
const tree = await inventory.stat(join(files, 'tree'))
|
||||
const recursive = manager.start({ operation: 'copy', sources: [{ path: tree.path, revision: tree.revision }], destination: tree.path })
|
||||
assert.equal((await finish(manager, recursive)).status, 'failed')
|
||||
const first = manager.start({ operation: 'touch', destination: join(files, 'first') })
|
||||
const second = manager.start({ operation: 'touch', destination: join(files, 'second') })
|
||||
assert.equal(second.status, 'queued')
|
||||
assert.equal(second.queuePosition, 1)
|
||||
await finish(manager, first)
|
||||
assert.equal((await finish(manager, second)).status, 'success')
|
||||
})
|
||||
|
||||
test('runs queued mutations in FIFO order and cancels queued work without touching files', async () => {
|
||||
const { files, manager } = await setup('dsh-file-queue-')
|
||||
const first = manager.start({ operation: 'touch', destination: join(files, 'first') })
|
||||
const second = manager.start({ operation: 'touch', destination: join(files, 'second') })
|
||||
const third = manager.start({ operation: 'touch', destination: join(files, 'third') })
|
||||
assert.deepEqual([second.queuePosition, third.queuePosition], [1, 2])
|
||||
assert.equal(manager.cancel(second.taskId).status, 'cancelled')
|
||||
assert.equal(manager.get(third.taskId).queuePosition, 1)
|
||||
assert.equal((await finish(manager, first)).status, 'success')
|
||||
assert.equal((await finish(manager, second)).status, 'cancelled')
|
||||
assert.equal((await finish(manager, third)).status, 'success')
|
||||
assert.equal(await stat(join(files, 'second')).then(() => true, error => error.code !== 'ENOENT'), false)
|
||||
})
|
||||
|
||||
test('persists queued work before the active task releases its lease', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-file-queue-persisted-'))
|
||||
const files = join(root, 'files')
|
||||
const tasks = join(root, 'tasks')
|
||||
await mkdir(files)
|
||||
let releaseReport
|
||||
const blocked = new Promise(resolve => { releaseReport = resolve })
|
||||
const manager = new FileTaskManager({ root: tasks, report: message => message === 'file-task.touch.completed' ? blocked : undefined })
|
||||
await manager.initialize()
|
||||
const first = manager.start({ operation: 'touch', destination: join(files, 'first') })
|
||||
const second = manager.start({ operation: 'touch', destination: join(files, 'second') })
|
||||
await manager.tasks.get(second.taskId).persisted
|
||||
const journal = JSON.parse(await readFile(join(tasks, `${second.taskId}.json`), 'utf8'))
|
||||
assert.equal(journal.status, 'queued')
|
||||
assert.equal(journal.queuePosition, 1)
|
||||
releaseReport()
|
||||
assert.equal((await finish(manager, first)).status, 'success')
|
||||
assert.equal((await finish(manager, second)).status, 'success')
|
||||
})
|
||||
|
||||
test('reports real transfer progress and cancels a queued transfer', async () => {
|
||||
const { files, manager } = await setup('dsh-file-transfer-progress-')
|
||||
const transfer = manager.startTransfer({ operation: 'upload', path: join(files, 'upload.bin'), totalBytes: 10 }, async ({ progress }) => {
|
||||
progress({ processedBytes: 4, totalBytes: 10 })
|
||||
progress({ processedBytes: 10, totalBytes: 10 })
|
||||
return { size: 10 }
|
||||
})
|
||||
assert.deepEqual(await transfer.result, { size: 10 })
|
||||
await manager.completion(transfer.task.taskId)
|
||||
assert.deepEqual(
|
||||
[manager.get(transfer.task.taskId).status, manager.get(transfer.task.taskId).processedBytes],
|
||||
['success', 10],
|
||||
)
|
||||
|
||||
let release
|
||||
const blocked = new Promise(resolve => { release = resolve })
|
||||
const active = manager.startTransfer({ operation: 'download', path: join(files, 'active') }, () => blocked)
|
||||
const queued = manager.startTransfer({ operation: 'download', path: join(files, 'queued') }, async () => undefined)
|
||||
const rejection = assert.rejects(queued.result, error => error.code === 'FILE_TASK_CANCELLED')
|
||||
assert.equal(manager.cancel(queued.task.taskId).status, 'cancelled')
|
||||
await rejection
|
||||
release()
|
||||
await active.result
|
||||
})
|
||||
|
||||
test('marks persisted transfers interrupted because HTTP streams cannot resume', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-file-transfer-recovery-'))
|
||||
const tasks = join(root, 'tasks')
|
||||
await mkdir(tasks)
|
||||
for (const [index, status] of ['running', 'queued'].entries()) {
|
||||
const taskId = `${String(index + 8).repeat(8)}-${String(index + 8).repeat(4)}-4${String(index + 8).repeat(3)}-8${String(index + 8).repeat(3)}-${String(index + 8).repeat(12)}`
|
||||
await writeFile(join(tasks, `${taskId}.json`), JSON.stringify({
|
||||
schema: 1, taskId, operation: index === 0 ? 'upload' : 'download', status,
|
||||
phase: status === 'queued' ? 'queued' : 'transferring', path: join(root, 'value'),
|
||||
managed: false, staging: [], hidden: [], published: [], createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}))
|
||||
}
|
||||
const recovered = new FileTaskManager({ root: tasks })
|
||||
await recovered.initialize()
|
||||
assert.equal(recovered.list().filter(task => ['upload', 'download'].includes(task.operation)).every(task => task.status === 'interrupted'), true)
|
||||
})
|
||||
|
||||
test('recovers committed deletes and marks uncommitted tasks interrupted', async () => {
|
||||
const { files, tasks } = await setup()
|
||||
const hidden = join(files, '.dsh-delete-task.tmp')
|
||||
await writeFile(hidden, 'remove')
|
||||
const runningDelete = {
|
||||
schema: 1, taskId: '11111111-1111-4111-8111-111111111111', operation: 'delete', status: 'running', phase: 'source-hidden',
|
||||
sources: [], destination: null, managed: false, hidden: [hidden], staging: [], published: [], createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
const staging = join(files, '.dsh-file-task.tmp')
|
||||
await writeFile(staging, 'partial')
|
||||
const runningCopy = {
|
||||
...runningDelete, taskId: '22222222-2222-4222-8222-222222222222', operation: 'copy', phase: 'mutating', hidden: [], staging: [staging],
|
||||
}
|
||||
await writeFile(join(tasks, `${runningDelete.taskId}.json`), JSON.stringify(runningDelete))
|
||||
await writeFile(join(tasks, `${runningCopy.taskId}.json`), JSON.stringify(runningCopy))
|
||||
const recovered = new FileTaskManager({ root: tasks })
|
||||
await recovered.initialize()
|
||||
assert.equal(recovered.get(runningDelete.taskId).status, 'success')
|
||||
assert.equal(recovered.get(runningCopy.taskId).status, 'interrupted')
|
||||
assert.equal(await stat(hidden).then(() => true, error => error.code !== 'ENOENT'), false)
|
||||
assert.equal(await stat(staging).then(() => true, error => error.code !== 'ENOENT'), false)
|
||||
})
|
||||
|
||||
test('move recovery cleans only the source bound to the committed destination', async () => {
|
||||
const { files, tasks } = await setup('dsh-file-move-recovery-')
|
||||
const committedSource = join(files, 'committed-source')
|
||||
const untouchedSource = join(files, 'untouched-source')
|
||||
const destination = join(files, 'destination')
|
||||
await writeFile(committedSource, 'old duplicate')
|
||||
await writeFile(untouchedSource, 'must remain')
|
||||
await writeFile(destination, 'committed')
|
||||
const task = {
|
||||
schema: 1, taskId: '33333333-3333-4333-8333-333333333333', operation: 'move', status: 'running', phase: 'destination-committed',
|
||||
sources: [{ path: committedSource, revision: 'old' }, { path: untouchedSource, revision: 'untouched' }],
|
||||
destination: files, currentSource: committedSource, currentDestination: destination,
|
||||
managed: false, hidden: [], staging: [], published: [destination], createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
await writeFile(join(tasks, `${task.taskId}.json`), JSON.stringify(task))
|
||||
const recovered = new FileTaskManager({ root: tasks })
|
||||
await recovered.initialize()
|
||||
assert.equal(recovered.get(task.taskId).status, 'success')
|
||||
assert.equal(await stat(committedSource).then(() => true, error => error.code !== 'ENOENT'), false)
|
||||
assert.equal(await readFile(untouchedSource, 'utf8'), 'must remain')
|
||||
})
|
||||
|
||||
test('move recovery recognizes an atomic rename completed before its journal commit', async () => {
|
||||
const { files, tasks } = await setup('dsh-file-rename-recovery-')
|
||||
const source = join(files, 'source')
|
||||
const destination = join(files, 'destination')
|
||||
await writeFile(destination, 'moved')
|
||||
const task = {
|
||||
schema: 1, taskId: '44444444-4444-4444-8444-444444444444', operation: 'move', status: 'running', phase: 'move-prepared',
|
||||
sources: [{ path: source, revision: 'old' }], destination: files,
|
||||
currentSource: source, currentDestination: destination, managed: false, hidden: [], staging: [], published: [],
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
await writeFile(join(tasks, `${task.taskId}.json`), JSON.stringify(task))
|
||||
const recovered = new FileTaskManager({ root: tasks })
|
||||
await recovered.initialize()
|
||||
assert.equal(recovered.get(task.taskId).status, 'success')
|
||||
assert.deepEqual(recovered.get(task.taskId).published, [destination])
|
||||
})
|
||||
|
||||
test('managed mutations expose a lease and reject active platform work', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'dsh-managed-tasks-'))
|
||||
const manager = new FileTaskManager({ root: join(root, 'tasks'), platformBusy: () => true })
|
||||
await manager.initialize()
|
||||
assert.equal(manager.wouldManage({ operation: 'touch', destination: '/data/platform/state/test' }), true)
|
||||
assert.throws(() => manager.start({ operation: 'touch', destination: '/data/platform/state/test' }), error => error.statusCode === 409)
|
||||
|
||||
const custom = new FileTaskManager({ root: join(root, 'custom-tasks'), protectedRoots: ['/custom/workspace'] })
|
||||
assert.throws(() => custom.start({
|
||||
operation: 'delete', sources: [{ path: '/custom/workspace', revision: 'sha256:unused' }],
|
||||
}), error => error.statusCode === 403)
|
||||
})
|
||||
|
||||
test('creates and extracts zip, 7z, and tar.gz archives through file tasks', async () => {
|
||||
for (const [format, extension] of [['zip', 'zip'], ['7z', '7z'], ['tar.gz', 'tar.gz']]) {
|
||||
const { files, manager, inventory } = await setup(`dsh-file-archive-${format.replace('.', '-')}-`)
|
||||
const sourcePath = join(files, 'source')
|
||||
await mkdir(sourcePath)
|
||||
await writeFile(join(sourcePath, 'nested.txt'), `content-${format}`)
|
||||
const source = await inventory.stat(sourcePath)
|
||||
const archivePath = join(files, `bundle.${extension}`)
|
||||
const archived = await finish(manager, manager.start({
|
||||
operation: 'archive', archiveFormat: format,
|
||||
sources: [{ path: source.path, revision: source.revision }], destination: archivePath,
|
||||
}))
|
||||
assert.equal(archived.status, 'success')
|
||||
assert.equal((await stat(archivePath)).isFile(), true)
|
||||
const extractRoot = join(files, 'extracted')
|
||||
await mkdir(extractRoot)
|
||||
const archive = await inventory.stat(archivePath)
|
||||
const extracted = await finish(manager, manager.start({
|
||||
operation: 'extract', archiveFormat: format,
|
||||
sources: [{ path: archive.path, revision: archive.revision }], destination: extractRoot,
|
||||
}))
|
||||
assert.equal(extracted.status, 'success')
|
||||
assert.equal(await readFile(join(extractRoot, 'source', 'nested.txt'), 'utf8'), `content-${format}`)
|
||||
}
|
||||
})
|
||||
|
||||
test('rejects invalid archive formats and escaping extracted links', async () => {
|
||||
const { files, manager, inventory } = await setup('dsh-file-archive-unsafe-')
|
||||
const sourcePath = join(files, 'source')
|
||||
await mkdir(sourcePath)
|
||||
await symlink('../../outside', join(sourcePath, 'escape'))
|
||||
const source = await inventory.stat(sourcePath)
|
||||
assert.throws(() => manager.start({
|
||||
operation: 'archive', archiveFormat: 'rar', sources: [{ path: source.path, revision: source.revision }],
|
||||
destination: join(files, 'bad.rar'),
|
||||
}), /format/)
|
||||
const archivePath = join(files, 'unsafe.tar.gz')
|
||||
assert.equal((await finish(manager, manager.start({
|
||||
operation: 'archive', archiveFormat: 'tar.gz', sources: [{ path: source.path, revision: source.revision }], destination: archivePath,
|
||||
}))).status, 'success')
|
||||
const archive = await inventory.stat(archivePath)
|
||||
const destination = join(files, 'destination')
|
||||
await mkdir(destination)
|
||||
const extracted = await finish(manager, manager.start({
|
||||
operation: 'extract', archiveFormat: 'tar.gz', sources: [{ path: archive.path, revision: archive.revision }], destination,
|
||||
}))
|
||||
assert.equal(extracted.status, 'failed')
|
||||
assert.equal(extracted.errorCode, 'ARCHIVE_UNSAFE')
|
||||
})
|
||||
|
||||
test('recovers committed archives and resumes staged extraction publication', async () => {
|
||||
const { files, tasks } = await setup('dsh-file-archive-recovery-')
|
||||
const archivePath = join(files, 'published.zip')
|
||||
await writeFile(archivePath, 'published')
|
||||
const archiveTask = {
|
||||
schema: 1, taskId: '66666666-6666-4666-8666-666666666666', operation: 'archive', status: 'running', phase: 'destination-committed',
|
||||
sources: [], destination: archivePath, managed: false, hidden: [], staging: [], published: [archivePath],
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
const destination = join(files, 'destination')
|
||||
const staging = join(destination, '.dsh-extract-recovery.tmp')
|
||||
await mkdir(staging, { recursive: true })
|
||||
await writeFile(join(staging, 'remaining.txt'), 'remaining')
|
||||
const extractTask = {
|
||||
schema: 1, taskId: '77777777-7777-4777-8777-777777777777', operation: 'extract', status: 'running', phase: 'publishing',
|
||||
sources: [], destination, conflict: 'reject', managed: false, hidden: [], staging: [staging], published: [],
|
||||
processedBytes: 0, totalBytes: 9, processedEntries: 0, totalEntries: 1,
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), cancelRequested: false,
|
||||
}
|
||||
await writeFile(join(tasks, `${archiveTask.taskId}.json`), JSON.stringify(archiveTask))
|
||||
await writeFile(join(tasks, `${extractTask.taskId}.json`), JSON.stringify(extractTask))
|
||||
const recovered = new FileTaskManager({ root: tasks })
|
||||
await recovered.initialize()
|
||||
assert.equal(recovered.get(archiveTask.taskId).status, 'success')
|
||||
assert.equal(recovered.get(extractTask.taskId).status, 'success')
|
||||
assert.equal(await readFile(join(destination, 'remaining.txt'), 'utf8'), 'remaining')
|
||||
})
|
||||
Reference in new issue
Block a user