dsh 0.2.0-rc.2: in-container update (experimental channel), path-agnostic compose, upstream patches adapted to 0.1.x/0.2.x
This commit is contained in:
hermes committed
commit
e6dcef9bf5
302 files changed
+75178
No files matched your search
@@ -0,0 +1,110 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { lstat, mkdir, readdir, readlink, rm } from 'node:fs/promises'
|
||||
import { dirname, join, relative, resolve, sep } from 'node:path'
|
||||
import { FileManagerError } from './index.mjs'
|
||||
|
||||
export const ARCHIVE_FORMATS = Object.freeze({
|
||||
zip: { extension: '.zip', command: 'zip' },
|
||||
'7z': { extension: '.7z', command: '7z' },
|
||||
'tar.gz': { extension: '.tar.gz', command: 'tar' },
|
||||
})
|
||||
|
||||
export function normalizeArchiveFormat(value) {
|
||||
if (!Object.hasOwn(ARCHIVE_FORMATS, value)) throw new FileManagerError('archive format is invalid')
|
||||
return value
|
||||
}
|
||||
|
||||
function run(command, args, { cwd, signal, capture = false } = {}) {
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
const child = spawn(command, args, { cwd, signal, stdio: ['ignore', capture ? 'pipe' : 'ignore', 'pipe'] })
|
||||
let errorOutput = ''
|
||||
let output = ''
|
||||
if (capture) {
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stdout.on('data', chunk => { if (output.length < 8 * 1024 * 1024) output += chunk })
|
||||
}
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', chunk => { if (errorOutput.length < 16_384) errorOutput += chunk })
|
||||
child.once('error', reject)
|
||||
child.once('exit', (code, signalName) => {
|
||||
if (code === 0) resolvePromise(output)
|
||||
else reject(new FileManagerError(
|
||||
`archive tool failed${errorOutput.trim() === '' ? '' : `: ${errorOutput.trim()}`}`,
|
||||
400,
|
||||
signalName === 'SIGTERM' ? 'FILE_TASK_CANCELLED' : 'ARCHIVE_TOOL_FAILED',
|
||||
))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function validateEntryNames(names) {
|
||||
for (const original of names) {
|
||||
const name = original.replace(/^(\.\/)+/u, '')
|
||||
if (name === '') continue
|
||||
if (name.includes('\0') || name.startsWith('/') || name.split('/').includes('..')) {
|
||||
throw new FileManagerError('archive contains an unsafe path', 400, 'ARCHIVE_UNSAFE')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function inside(root, path) {
|
||||
const value = relative(root, path)
|
||||
return value === '' || (!value.startsWith(`..${sep}`) && value !== '..')
|
||||
}
|
||||
|
||||
export async function validateExtractedTree(root) {
|
||||
const pending = [root]
|
||||
let entries = 0
|
||||
let bytes = 0
|
||||
while (pending.length > 0) {
|
||||
const path = pending.pop()
|
||||
const details = await lstat(path)
|
||||
if (details.isDirectory() && !details.isSymbolicLink()) {
|
||||
for (const name of await readdir(path)) pending.push(join(path, name))
|
||||
} else if (details.isSymbolicLink()) {
|
||||
const target = await readlink(path)
|
||||
if (target.startsWith('/') || !inside(root, resolve(dirname(path), target))) {
|
||||
throw new FileManagerError('archive contains a symbolic link outside its destination', 400, 'ARCHIVE_UNSAFE')
|
||||
}
|
||||
} else if (!details.isFile()) {
|
||||
throw new FileManagerError('archive contains an unsupported special file', 415, 'ARCHIVE_UNSAFE')
|
||||
}
|
||||
entries += 1
|
||||
if (details.isFile()) bytes += details.size
|
||||
}
|
||||
return { entries: Math.max(0, entries - 1), bytes }
|
||||
}
|
||||
|
||||
export async function createArchive({ format, sourceRoot, output, entries = ['.'], signal }) {
|
||||
normalizeArchiveFormat(format)
|
||||
if (!Array.isArray(entries) || entries.length === 0 || entries.some(entry => typeof entry !== 'string' || entry === '' || entry.includes('\0') || entry.includes('/'))) {
|
||||
throw new FileManagerError('archive entry names are invalid')
|
||||
}
|
||||
if (format === 'zip') await run('zip', ['-q', '-r', output, '--', ...entries], { cwd: sourceRoot, signal })
|
||||
else if (format === '7z') await run('7z', ['a', '-bd', '-y', '-t7z', output, '--', ...entries], { cwd: sourceRoot, signal })
|
||||
else await run('tar', ['-czf', output, '-C', sourceRoot, '--', ...entries], { signal })
|
||||
}
|
||||
|
||||
export async function extractArchive({ format, archive, output, signal }) {
|
||||
normalizeArchiveFormat(format)
|
||||
await mkdir(output, { recursive: true })
|
||||
try {
|
||||
if (format === 'zip') {
|
||||
validateEntryNames((await run('unzip', ['-Z1', archive], { signal, capture: true })).split('\n'))
|
||||
await run('unzip', ['-qq', archive, '-d', output], { signal })
|
||||
} else if (format === '7z') {
|
||||
const listed = await run('7z', ['l', '-slt', '--', archive], { signal, capture: true })
|
||||
validateEntryNames(listed.split('\n').filter(line => line.startsWith('Path = ')).slice(1).map(line => line.slice(7)))
|
||||
await run('7z', ['x', '-bd', '-y', `-o${output}`, '--', archive], { signal })
|
||||
} else {
|
||||
validateEntryNames((await run('tar', ['-tzf', archive], { signal, capture: true })).split('\n'))
|
||||
await run('tar', ['--extract', '--gzip', '--file', archive, '--directory', output, '--no-same-owner', '--no-same-permissions'], { signal })
|
||||
}
|
||||
return await validateExtractedTree(output)
|
||||
} catch (error) {
|
||||
await rm(output, { recursive: true, force: true }).catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export const archiveInternals = Object.freeze({ inside, run, validateEntryNames })
|
||||
@@ -0,0 +1,78 @@
|
||||
import { link, open, rename, rm, stat } from 'node:fs/promises'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
FileManagerError, FileRevisionConflictError, MAX_TEXT_BYTES, fileError,
|
||||
fileManagerInternals, isManagedPath, normalizeAbsolutePath,
|
||||
} from './index.mjs'
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
function bytes(value) {
|
||||
if (typeof value !== 'string') throw new FileManagerError('content must be a string')
|
||||
const result = Buffer.from(value, 'utf8')
|
||||
if (result.byteLength > MAX_TEXT_BYTES) throw new FileManagerError('file is too large to edit', 413, 'FILE_TOO_LARGE')
|
||||
if (result.includes(0)) throw new FileManagerError('content contains NUL', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
return result
|
||||
}
|
||||
|
||||
export class AtomicFileEditor {
|
||||
constructor({ isManaged = isManagedPath } = {}) {
|
||||
this.isManaged = isManaged
|
||||
}
|
||||
|
||||
async write(value, content, revision, { create = false } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
const body = bytes(content)
|
||||
const parent = dirname(path)
|
||||
let original
|
||||
try {
|
||||
original = await stat(path, { bigint: true })
|
||||
if (!original.isFile()) throw new FileManagerError('path is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
if (create) throw new FileManagerError('file already exists', 409, 'FILE_EXISTS')
|
||||
if (typeof revision !== 'string' || revision !== fileManagerInternals.revisionFor(path, original)) {
|
||||
throw new FileRevisionConflictError()
|
||||
}
|
||||
} catch (error) {
|
||||
if (error?.code !== 'ENOENT') throw fileError(error)
|
||||
if (!create) throw fileError(error)
|
||||
if (revision !== null && revision !== undefined) throw new FileManagerError('new file revision must be null')
|
||||
}
|
||||
const temporary = join(parent, `.dsh-edit-${randomUUID()}.tmp`)
|
||||
let handle
|
||||
try {
|
||||
handle = await open(temporary, 'wx', original === undefined ? 0o644 : Number(original.mode & 0o7777n))
|
||||
await handle.writeFile(body)
|
||||
await handle.sync()
|
||||
if (original !== undefined) {
|
||||
await handle.chmod(Number(original.mode & 0o7777n))
|
||||
const current = await handle.stat({ bigint: true })
|
||||
if (current.uid !== original.uid || current.gid !== original.gid) {
|
||||
try { await handle.chown(Number(original.uid), Number(original.gid)) } catch {
|
||||
throw new FileManagerError('cannot preserve file ownership; use the terminal', 403, 'OWNERSHIP_PRESERVE_FAILED')
|
||||
}
|
||||
}
|
||||
}
|
||||
await handle.close()
|
||||
handle = undefined
|
||||
if (original === undefined) {
|
||||
await link(temporary, path)
|
||||
await rm(temporary)
|
||||
} else await rename(temporary, path)
|
||||
await syncDirectory(parent)
|
||||
const saved = await stat(path, { bigint: true })
|
||||
return {
|
||||
path, revision: fileManagerInternals.revisionFor(path, saved), size: body.byteLength,
|
||||
mode: Number(saved.mode & 0o7777n), modifiedAt: new Date(Number(saved.mtimeNs / 1_000_000n)).toISOString(),
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {})
|
||||
await rm(temporary, { force: true }).catch(() => {})
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
import { createHash, randomUUID } from 'node:crypto'
|
||||
import { constants } from 'node:fs'
|
||||
import { access, lstat, readFile, readdir, readlink, realpath, stat } from 'node:fs/promises'
|
||||
import { basename, dirname, join, normalize, resolve } from 'node:path'
|
||||
|
||||
export const MAX_PATH_BYTES = 4096
|
||||
export const MAX_TEXT_BYTES = 2 * 1024 * 1024
|
||||
export const DEFAULT_LIST_LIMIT = 200
|
||||
export const MAX_LIST_LIMIT = 1000
|
||||
export const SEARCH_MAX_DEPTH = 16
|
||||
export const SEARCH_MAX_SCANNED = 10_000
|
||||
export const SEARCH_MAX_RESULTS = 1000
|
||||
|
||||
const SORTS = new Set(['name', 'size', 'type', 'modifiedAt'])
|
||||
const ORDERS = new Set(['asc', 'desc'])
|
||||
|
||||
export class FileManagerError extends Error {
|
||||
constructor(message, statusCode = 400, code = 'FILE_INVALID') {
|
||||
super(message)
|
||||
this.name = 'FileManagerError'
|
||||
this.statusCode = statusCode
|
||||
this.code = code
|
||||
}
|
||||
}
|
||||
|
||||
export class FileRevisionConflictError extends FileManagerError {
|
||||
constructor(message = 'file revision changed') {
|
||||
super(message, 409, 'REVISION_CONFLICT')
|
||||
this.name = 'FileRevisionConflictError'
|
||||
}
|
||||
}
|
||||
|
||||
function statusCode(error) {
|
||||
if (['EACCES', 'EPERM', 'EROFS'].includes(error?.code)) return 403
|
||||
if (['ENOENT', 'ENOTDIR'].includes(error?.code)) return 404
|
||||
if (error?.code === 'EFBIG') return 413
|
||||
return 400
|
||||
}
|
||||
|
||||
export function fileError(error, fallback = 'file operation failed') {
|
||||
if (error instanceof FileManagerError) return error
|
||||
const wrapped = new FileManagerError(error instanceof Error ? error.message : fallback, statusCode(error), error?.code ?? 'FILE_ERROR')
|
||||
if (error instanceof Error) wrapped.cause = error
|
||||
return wrapped
|
||||
}
|
||||
|
||||
export function normalizeAbsolutePath(value) {
|
||||
if (typeof value !== 'string' || value.length === 0) throw new FileManagerError('path must be a non-empty string')
|
||||
if (!value.startsWith('/')) throw new FileManagerError('path must be absolute')
|
||||
if (Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES) throw new FileManagerError('path is too long')
|
||||
if(/[\u0000-\u001f\u007f]/u.test(value)) throw new FileManagerError('path contains control characters')
|
||||
const path = normalize(value)
|
||||
if (!path.startsWith('/')) throw new FileManagerError('path must be absolute')
|
||||
return path
|
||||
}
|
||||
|
||||
export function createManagedPathMatcher(platformData = '/data/platform') {
|
||||
const data = normalizeAbsolutePath(platformData)
|
||||
const roots = Object.freeze(['/opt/dsh-platform', '/run/dsh-platform', join(data, 'state'), join(data, 'store')])
|
||||
return value => {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
return roots.some(root => path === root || path.startsWith(`${root}/`))
|
||||
}
|
||||
}
|
||||
|
||||
export const isManagedPath = createManagedPathMatcher()
|
||||
|
||||
export function fileManagerLocations({
|
||||
platformData = '/data/platform', dshHome = '/data/dsh', defaultWorkspace = '/workspace',
|
||||
} = {}) {
|
||||
const defaultPath = normalizeAbsolutePath(defaultWorkspace)
|
||||
const shortcuts = [...new Set([defaultPath, normalizeAbsolutePath(dshHome), normalizeAbsolutePath(platformData), '/'])]
|
||||
return Object.freeze({ defaultPath, shortcuts: Object.freeze(shortcuts) })
|
||||
}
|
||||
|
||||
function parseIdentityFile(value, idIndex) {
|
||||
const result = new Map()
|
||||
for (const line of value.split('\n')) {
|
||||
if (line === '' || line.startsWith('#')) continue
|
||||
const fields = line.split(':')
|
||||
const id = Number(fields[idIndex])
|
||||
if (fields[0] !== '' && Number.isSafeInteger(id) && id >= 0 && !result.has(id)) result.set(id, fields[0])
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export class UnixIdentityResolver {
|
||||
constructor({ passwdPath = '/etc/passwd', groupPath = '/etc/group' } = {}) {
|
||||
this.passwdPath = passwdPath
|
||||
this.groupPath = groupPath
|
||||
this.loading = undefined
|
||||
}
|
||||
|
||||
async #identities() {
|
||||
this.loading ??= Promise.all([
|
||||
readFile(this.passwdPath, 'utf8').then(value => parseIdentityFile(value, 2), () => new Map()),
|
||||
readFile(this.groupPath, 'utf8').then(value => parseIdentityFile(value, 2), () => new Map()),
|
||||
])
|
||||
return this.loading
|
||||
}
|
||||
|
||||
async resolve(uid, gid) {
|
||||
const [users, groups] = await this.#identities()
|
||||
return { user: users.get(uid) ?? String(uid), group: groups.get(gid) ?? String(gid) }
|
||||
}
|
||||
|
||||
async userId(value) { return this.#identityId(value, 0, 'user') }
|
||||
|
||||
async groupId(value) { return this.#identityId(value, 1, 'group') }
|
||||
|
||||
async #identityId(value, index, label) {
|
||||
if (typeof value !== 'string' || value === '' || value.length > 256 || /[:\u0000-\u001f\u007f]/u.test(value)) {
|
||||
throw new FileManagerError(`${label} is invalid`, 400, 'FILE_IDENTITY_INVALID')
|
||||
}
|
||||
if (/^[0-9]+$/u.test(value)) {
|
||||
const id = Number(value)
|
||||
if (Number.isSafeInteger(id) && id >= 0 && id < 0xffff_ffff) return id
|
||||
throw new FileManagerError(`${label} id is invalid`, 400, 'FILE_IDENTITY_INVALID')
|
||||
}
|
||||
const identities = (await this.#identities())[index]
|
||||
for (const [id, name] of identities) if (name === value) return id
|
||||
throw new FileManagerError(`${label} was not found`, 400, 'FILE_IDENTITY_INVALID')
|
||||
}
|
||||
}
|
||||
|
||||
function kind(stats) {
|
||||
if (stats.isFile()) return 'file'
|
||||
if (stats.isDirectory()) return 'directory'
|
||||
if (stats.isSymbolicLink()) return 'symlink'
|
||||
if (stats.isBlockDevice()) return 'block-device'
|
||||
if (stats.isCharacterDevice()) return 'character-device'
|
||||
if (stats.isFIFO()) return 'fifo'
|
||||
if (stats.isSocket()) return 'socket'
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
function metadata(stats) {
|
||||
return {
|
||||
type: kind(stats),
|
||||
size: Number(stats.size),
|
||||
mode: Number(stats.mode & 0o7777n),
|
||||
uid: Number(stats.uid),
|
||||
gid: Number(stats.gid),
|
||||
modifiedAt: new Date(Number(stats.mtimeNs / 1_000_000n)).toISOString(),
|
||||
createdAt: new Date(Number(stats.birthtimeNs / 1_000_000n)).toISOString(),
|
||||
}
|
||||
}
|
||||
|
||||
function revisionFor(path, stats, extra = '') {
|
||||
return `sha256:${createHash('sha256').update(JSON.stringify([
|
||||
path, stats.dev.toString(), stats.ino.toString(), kind(stats), stats.size.toString(),
|
||||
stats.mode.toString(), stats.uid.toString(), stats.gid.toString(), stats.mtimeNs.toString(), stats.ctimeNs.toString(), extra,
|
||||
])).digest('hex')}`
|
||||
}
|
||||
|
||||
async function permissions(path) {
|
||||
const allowed = async mode => access(path, mode).then(() => true, () => false)
|
||||
const [readable, writable] = await Promise.all([allowed(constants.R_OK), allowed(constants.W_OK)])
|
||||
return { readable, writable }
|
||||
}
|
||||
|
||||
async function symlinkDetails(path, stats) {
|
||||
if (!stats.isSymbolicLink()) return {}
|
||||
const linkTarget = await readlink(path)
|
||||
try {
|
||||
const target = await stat(path, { bigint: true })
|
||||
return { linkTarget, targetExists: true, targetType: kind(target), realPath: await realpath(path) }
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT' || error?.code === 'ELOOP') return { linkTarget, targetExists: false, targetType: null, realPath: null }
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function describe(path, stats = undefined, managed = isManagedPath, identity = new UnixIdentityResolver()) {
|
||||
const value = stats ?? await lstat(path, { bigint: true })
|
||||
const link = await symlinkDetails(path, value)
|
||||
const owner = await identity.resolve(Number(value.uid), Number(value.gid))
|
||||
return {
|
||||
name: basename(path) || '/',
|
||||
path,
|
||||
...metadata(value),
|
||||
...owner,
|
||||
...await permissions(path),
|
||||
managed: managed(path),
|
||||
revision: revisionFor(path, value, link.linkTarget ?? ''),
|
||||
...link,
|
||||
}
|
||||
}
|
||||
|
||||
function compareEntries(left, right, sort, order) {
|
||||
if (left.type === 'directory' && right.type !== 'directory') return -1
|
||||
if (left.type !== 'directory' && right.type === 'directory') return 1
|
||||
let result
|
||||
if (sort === 'size') result = left.size - right.size
|
||||
else if (sort === 'modifiedAt') result = left.modifiedAt.localeCompare(right.modifiedAt)
|
||||
else if (sort === 'type') result = left.type.localeCompare(right.type)
|
||||
else result = left.name.localeCompare(right.name, undefined, { numeric: true, sensitivity: 'base' })
|
||||
if (result === 0) result = left.name.localeCompare(right.name)
|
||||
return order === 'desc' ? -result : result
|
||||
}
|
||||
|
||||
function encodeCursor(revision, offset) {
|
||||
return Buffer.from(JSON.stringify({ revision, offset })).toString('base64url')
|
||||
}
|
||||
|
||||
function decodeCursor(value, revision) {
|
||||
if (value === undefined || value === null || value === '') return 0
|
||||
try {
|
||||
const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8'))
|
||||
if (parsed.revision !== revision) throw new FileRevisionConflictError('directory changed while paging')
|
||||
if (!Number.isSafeInteger(parsed.offset) || parsed.offset < 0) throw new Error('invalid offset')
|
||||
return parsed.offset
|
||||
} catch (error) {
|
||||
if (error instanceof FileRevisionConflictError) throw error
|
||||
throw new FileManagerError('cursor is invalid')
|
||||
}
|
||||
}
|
||||
|
||||
function listRevision(path, stats, entries) {
|
||||
const digest = createHash('sha256')
|
||||
for (const entry of entries) digest.update(`${entry.name}\0${entry.revision ?? entry.type}\0`)
|
||||
return revisionFor(path, stats, digest.digest('hex'))
|
||||
}
|
||||
|
||||
function directoryEntryType(entry) {
|
||||
if (entry.isDirectory()) return 'directory'
|
||||
if (entry.isFile()) return 'file'
|
||||
if (entry.isSymbolicLink()) return 'symlink'
|
||||
if (entry.isFIFO()) return 'fifo'
|
||||
if (entry.isSocket()) return 'socket'
|
||||
if (entry.isBlockDevice()) return 'block-device'
|
||||
if (entry.isCharacterDevice()) return 'character-device'
|
||||
return 'unknown'
|
||||
}
|
||||
|
||||
function compareNames(left, right, order) {
|
||||
if (left.type === 'directory' && right.type !== 'directory') return -1
|
||||
if (left.type !== 'directory' && right.type === 'directory') return 1
|
||||
const result = left.name.localeCompare(right.name, undefined, { numeric: true, sensitivity: 'base' })
|
||||
|| left.name.localeCompare(right.name)
|
||||
return order === 'desc' ? -result : result
|
||||
}
|
||||
|
||||
function parseLimit(value, fallback = DEFAULT_LIST_LIMIT) {
|
||||
if (value === undefined || value === null || value === '') return fallback
|
||||
const limit = Number(value)
|
||||
if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIST_LIMIT) throw new FileManagerError(`limit must be between 1 and ${MAX_LIST_LIMIT}`)
|
||||
return limit
|
||||
}
|
||||
|
||||
function parseOffset(value) {
|
||||
if (value === undefined || value === null || value === '') return undefined
|
||||
const offset = Number(value)
|
||||
if (!Number.isSafeInteger(offset) || offset < 0) throw new FileManagerError('offset must be a non-negative integer')
|
||||
return offset
|
||||
}
|
||||
|
||||
export class FileInventory {
|
||||
constructor({ isManaged = isManagedPath, identity = new UnixIdentityResolver() } = {}) {
|
||||
this.isManaged = isManaged
|
||||
this.identity = identity
|
||||
}
|
||||
|
||||
async stat(value) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
try {
|
||||
const item = await describe(path, undefined, this.isManaged, this.identity)
|
||||
return { ...item, realPath: item.realPath ?? await realpath(path).catch(() => path), parent: dirname(path) }
|
||||
} catch (error) {
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async list(value, { cursor, offset: requestedOffset, limit, sort = 'name', order = 'asc' } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
if (!SORTS.has(sort)) throw new FileManagerError('sort is invalid')
|
||||
if (!ORDERS.has(order)) throw new FileManagerError('order is invalid')
|
||||
try {
|
||||
const root = await lstat(path, { bigint: true })
|
||||
if (!root.isDirectory()) throw new FileManagerError('path is not a directory', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const pageSize = parseLimit(limit)
|
||||
const directOffset = parseOffset(requestedOffset)
|
||||
if (directOffset !== undefined && cursor !== undefined && cursor !== null && cursor !== '') throw new FileManagerError('cursor and offset cannot be combined')
|
||||
const names = await readdir(path, { withFileTypes: true })
|
||||
let entries
|
||||
let revision
|
||||
let offset
|
||||
if (sort === 'name') {
|
||||
const summary = names.map(entry => ({ name: entry.name, type: directoryEntryType(entry) }))
|
||||
summary.sort((left, right) => compareNames(left, right, order))
|
||||
revision = listRevision(path, root, summary)
|
||||
offset = directOffset ?? decodeCursor(cursor, revision)
|
||||
entries = await Promise.all(summary.slice(offset, offset + pageSize).map(entry => (
|
||||
describe(resolve(path, entry.name), undefined, this.isManaged, this.identity)
|
||||
)))
|
||||
} else {
|
||||
entries = await Promise.all(names.map(entry => describe(resolve(path, entry.name), undefined, this.isManaged, this.identity)))
|
||||
entries.sort((left, right) => compareEntries(left, right, sort, order))
|
||||
revision = listRevision(path, root, entries)
|
||||
offset = directOffset ?? decodeCursor(cursor, revision)
|
||||
}
|
||||
const page = sort === 'name' ? entries : entries.slice(offset, offset + pageSize)
|
||||
const nextOffset = offset + page.length
|
||||
return {
|
||||
path,
|
||||
realPath: await realpath(path),
|
||||
revision,
|
||||
entries: page,
|
||||
nextCursor: nextOffset < names.length ? encodeCursor(revision, nextOffset) : null,
|
||||
total: names.length,
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async content(value) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
try {
|
||||
const before = await stat(path, { bigint: true })
|
||||
if (!before.isFile()) throw new FileManagerError('path is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
if (before.size > BigInt(MAX_TEXT_BYTES)) throw new FileManagerError('file is too large to edit', 413, 'FILE_TOO_LARGE')
|
||||
const bytes = await readFile(path)
|
||||
const after = await stat(path, { bigint: true })
|
||||
const beforeRevision = revisionFor(path, before)
|
||||
if (beforeRevision !== revisionFor(path, after)) throw new FileRevisionConflictError('file changed while reading')
|
||||
if (bytes.includes(0)) throw new FileManagerError('file is not UTF-8 text', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const content = new TextDecoder('utf-8', { fatal: true }).decode(bytes)
|
||||
return {
|
||||
path,
|
||||
realPath: await realpath(path),
|
||||
content,
|
||||
encoding: 'utf-8',
|
||||
newline: content.includes('\r\n') ? 'crlf' : 'lf',
|
||||
revision: beforeRevision,
|
||||
mode: Number(before.mode & 0o7777n),
|
||||
size: bytes.byteLength,
|
||||
modifiedAt: new Date(Number(before.mtimeNs / 1_000_000n)).toISOString(),
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export class FileSearchManager {
|
||||
constructor({ inventory = new FileInventory(), onState = () => {} } = {}) {
|
||||
this.inventory = inventory
|
||||
this.onState = onState
|
||||
this.tasks = new Map()
|
||||
}
|
||||
|
||||
start({ path: value, revision, query }) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
if (typeof query !== 'string' || query.trim().length === 0 || query.length > 256) throw new FileManagerError('search query is invalid')
|
||||
const task = {
|
||||
taskId: randomUUID(), operation: 'search', status: 'running', path, revision, query,
|
||||
scanned: 0, matches: [], truncated: false, error: null, cancelRequested: false,
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#publish(task)
|
||||
task.completion = this.#run(task).catch(error => {
|
||||
task.status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
this.#publish(task)
|
||||
})
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
get(taskId, { cursor, limit } = {}) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
const offset = cursor === undefined || cursor === '' ? 0 : Number(cursor)
|
||||
if (!Number.isSafeInteger(offset) || offset < 0) throw new FileManagerError('task cursor is invalid')
|
||||
const size = parseLimit(limit)
|
||||
return { ...this.public(task), results: task.matches.slice(offset, offset + size), nextCursor: offset + size < task.matches.length ? String(offset + size) : null }
|
||||
}
|
||||
|
||||
list() {
|
||||
return [...this.tasks.values()].map(task => this.public(task)).reverse()
|
||||
}
|
||||
|
||||
cancel(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
if (task.status !== 'running') throw new FileManagerError('file task is already complete', 409, 'FILE_TASK_COMPLETE')
|
||||
task.cancelRequested = true
|
||||
this.#publish(task)
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
public(task) {
|
||||
const { completion, matches, cancelRequested, ...value } = task
|
||||
return { ...value, resultCount: matches.length, cancelRequested }
|
||||
}
|
||||
|
||||
#publish(task) {
|
||||
task.updatedAt = new Date().toISOString()
|
||||
this.onState(this.public(task))
|
||||
}
|
||||
|
||||
async #run(task) {
|
||||
const root = await this.inventory.list(task.path, { limit: 1 })
|
||||
if (task.revision !== undefined && task.revision !== null && task.revision !== root.revision) throw new FileRevisionConflictError('search root changed')
|
||||
task.revision = root.revision
|
||||
const needle = task.query.toLocaleLowerCase()
|
||||
const pending = [{ path: task.path, depth: 0 }]
|
||||
while (pending.length > 0 && task.scanned < SEARCH_MAX_SCANNED && task.matches.length < SEARCH_MAX_RESULTS) {
|
||||
if (task.cancelRequested) {
|
||||
task.status = 'cancelled'
|
||||
this.#publish(task)
|
||||
return
|
||||
}
|
||||
const current = pending.shift()
|
||||
let names
|
||||
try { names = await readdir(current.path, { withFileTypes: true }) } catch (error) {
|
||||
if (['EACCES', 'EPERM', 'ENOENT'].includes(error?.code)) continue
|
||||
throw error
|
||||
}
|
||||
for (const entry of names) {
|
||||
task.scanned += 1
|
||||
const child = resolve(current.path, entry.name)
|
||||
const relative = child.slice(task.path === '/' ? 1 : task.path.length + 1)
|
||||
if (relative.toLocaleLowerCase().includes(needle)) task.matches.push(await describe(child, undefined, this.inventory.isManaged, this.inventory.identity))
|
||||
if (entry.isDirectory() && !entry.isSymbolicLink() && current.depth < SEARCH_MAX_DEPTH) pending.push({ path: child, depth: current.depth + 1 })
|
||||
if (task.scanned >= SEARCH_MAX_SCANNED || task.matches.length >= SEARCH_MAX_RESULTS) break
|
||||
}
|
||||
this.#publish(task)
|
||||
await new Promise(resolve => setImmediate(resolve))
|
||||
}
|
||||
task.truncated = pending.length > 0 || task.scanned >= SEARCH_MAX_SCANNED || task.matches.length >= SEARCH_MAX_RESULTS
|
||||
task.status = 'success'
|
||||
this.#publish(task)
|
||||
}
|
||||
}
|
||||
|
||||
export class FileSizeManager {
|
||||
constructor({ inventory = new FileInventory(), onState = () => {} } = {}) {
|
||||
this.inventory = inventory
|
||||
this.onState = onState
|
||||
this.tasks = new Map()
|
||||
}
|
||||
|
||||
start({ path: value, revision } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
const task = {
|
||||
taskId: randomUUID(), operation: 'size', status: 'running', path, revision,
|
||||
bytes: 0, entries: 0, error: null, cancelRequested: false,
|
||||
createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#publish(task)
|
||||
task.completion = this.#run(task).catch(error => {
|
||||
task.status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
this.#publish(task)
|
||||
})
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
get(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
list() { return [...this.tasks.values()].map(task => this.public(task)).reverse() }
|
||||
|
||||
cancel(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
if (task.status !== 'running') throw new FileManagerError('file task is already complete', 409, 'FILE_TASK_COMPLETE')
|
||||
task.cancelRequested = true
|
||||
this.#publish(task)
|
||||
return this.public(task)
|
||||
}
|
||||
|
||||
public(task) {
|
||||
const { completion, cancelRequested, ...value } = task
|
||||
return { ...value, cancelRequested }
|
||||
}
|
||||
|
||||
#publish(task) {
|
||||
task.updatedAt = new Date().toISOString()
|
||||
this.onState(this.public(task))
|
||||
}
|
||||
|
||||
async #run(task) {
|
||||
const root = await this.inventory.stat(task.path)
|
||||
if (root.type !== 'directory') throw new FileManagerError('size target is not a directory', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
if (typeof task.revision !== 'string' || task.revision !== root.revision) throw new FileRevisionConflictError('size target changed')
|
||||
const pending = [task.path]
|
||||
while (pending.length > 0) {
|
||||
if (task.cancelRequested) {
|
||||
task.status = 'cancelled'
|
||||
this.#publish(task)
|
||||
return
|
||||
}
|
||||
const directory = pending.pop()
|
||||
for (const name of await readdir(directory)) {
|
||||
const path = resolve(directory, name)
|
||||
const details = await lstat(path, { bigint: true })
|
||||
task.entries += 1
|
||||
if (details.isDirectory() && !details.isSymbolicLink()) pending.push(path)
|
||||
else if (details.isFile()) task.bytes += Number(details.size)
|
||||
}
|
||||
this.#publish(task)
|
||||
await new Promise(resolveTask => setImmediate(resolveTask))
|
||||
}
|
||||
task.status = 'success'
|
||||
this.#publish(task)
|
||||
}
|
||||
}
|
||||
|
||||
export const fileManagerInternals = Object.freeze({ describe, revisionFor, parseIdentityFile, parseLimit })
|
||||
@@ -0,0 +1,645 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { chmod, cp, lchown, lstat, mkdir, open, readdir, readFile, readlink, rename, rm } from 'node:fs/promises'
|
||||
import { basename, dirname, join } from 'node:path'
|
||||
import { durableReplace } from '../../../platform/lib/atomic.mjs'
|
||||
import { createArchive, extractArchive, normalizeArchiveFormat } from './archives.mjs'
|
||||
import {
|
||||
FileInventory, FileManagerError, FileRevisionConflictError, FileSearchManager, FileSizeManager, UnixIdentityResolver,
|
||||
fileError, isManagedPath, normalizeAbsolutePath,
|
||||
} from './index.mjs'
|
||||
|
||||
const OPERATIONS = new Set(['mkdir', 'touch', 'rename', 'copy', 'move', 'delete', 'attributes', 'archive', 'extract'])
|
||||
const DEFAULT_PROTECTED_DELETE_ROOTS = ['/', '/data', '/data/dsh', '/data/platform', '/workspace', '/run/dsh-platform/deployment']
|
||||
|
||||
function now() { return new Date().toISOString() }
|
||||
|
||||
async function exists(path) {
|
||||
return lstat(path).then(() => true, error => error?.code === 'ENOENT' ? false : Promise.reject(error))
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
function publicTask(task) {
|
||||
const { completion, persisted, executor, resolveResult, rejectResult, cancelRequested, abortController, ...value } = task
|
||||
return { ...value, cancelRequested: cancelRequested === true }
|
||||
}
|
||||
|
||||
function cleanName(path) {
|
||||
const name = basename(path)
|
||||
if (name === '' || name === '.' || name === '..') throw new FileManagerError('destination name is invalid')
|
||||
return name
|
||||
}
|
||||
|
||||
function renamedPath(path, index) {
|
||||
const dot = basename(path).lastIndexOf('.')
|
||||
const directory = dirname(path)
|
||||
const name = basename(path)
|
||||
const stem = dot > 0 ? name.slice(0, dot) : name
|
||||
const extension = dot > 0 ? name.slice(dot) : ''
|
||||
return join(directory, `${stem} (${String(index)})${extension}`)
|
||||
}
|
||||
|
||||
async function uniqueDestination(path, conflict) {
|
||||
if (!await exists(path)) return path
|
||||
if (conflict === 'reject') throw new FileManagerError('destination already exists', 409, 'FILE_EXISTS')
|
||||
if (conflict === 'overwrite') return path
|
||||
for (let index = 1; index < 10_000; index += 1) {
|
||||
const candidate = renamedPath(path, index)
|
||||
if (!await exists(candidate)) return candidate
|
||||
}
|
||||
throw new FileManagerError('could not choose a unique destination', 409, 'FILE_EXISTS')
|
||||
}
|
||||
|
||||
async function copyEntry(source, staging) {
|
||||
const sourceStats = await lstat(source)
|
||||
if (sourceStats.isDirectory()) {
|
||||
await cp(source, staging, { recursive: true, dereference: false, errorOnExist: true, force: false, preserveTimestamps: true })
|
||||
} else if (sourceStats.isSymbolicLink()) {
|
||||
const target = await readlink(source)
|
||||
await import('node:fs/promises').then(fs => fs.symlink(target, staging))
|
||||
} else if (sourceStats.isFile()) {
|
||||
await cp(source, staging, { dereference: false, errorOnExist: true, force: false, preserveTimestamps: true })
|
||||
} else throw new FileManagerError('special files cannot be copied', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
}
|
||||
|
||||
async function treeMetrics(path) {
|
||||
const value = await lstat(path)
|
||||
if (!value.isDirectory() || value.isSymbolicLink()) return { entries: 1, bytes: value.isFile() ? value.size : 0 }
|
||||
let entries = 1
|
||||
let bytes = 0
|
||||
for (const name of await readdir(path)) {
|
||||
const child = await treeMetrics(join(path, name))
|
||||
entries += child.entries
|
||||
bytes += child.bytes
|
||||
}
|
||||
return { entries, bytes }
|
||||
}
|
||||
|
||||
function normalizeAttributes(value) {
|
||||
if (value === null || typeof value !== 'object' || Array.isArray(value)
|
||||
|| Object.keys(value).some(key => !['user', 'group', 'mode', 'recursive'].includes(key))
|
||||
|| typeof value.user !== 'string' || typeof value.group !== 'string'
|
||||
|| typeof value.mode !== 'string' || !/^[0-7]{3,4}$/u.test(value.mode)
|
||||
|| typeof value.recursive !== 'boolean') throw new FileManagerError('file attributes are invalid')
|
||||
return { user: value.user, group: value.group, mode: value.mode, recursive: value.recursive }
|
||||
}
|
||||
|
||||
function assertAttributesApplied(path, details, { uid, gid, mode }, symbolicLink) {
|
||||
const ownershipMatches = details.uid === uid && details.gid === gid
|
||||
const modeMatches = symbolicLink || (details.mode & 0o7777) === mode
|
||||
if (ownershipMatches && modeMatches) return
|
||||
throw new FileManagerError(
|
||||
`filesystem did not apply Unix ownership or mode for ${path}; use a filesystem mounted with Unix metadata support`,
|
||||
403,
|
||||
'FILE_ATTRIBUTES_UNSUPPORTED',
|
||||
)
|
||||
}
|
||||
|
||||
export class FileTaskManager {
|
||||
constructor({
|
||||
root, inventory = new FileInventory(), isManaged = isManagedPath,
|
||||
protectedRoots = DEFAULT_PROTECTED_DELETE_ROOTS,
|
||||
onState = () => {}, platformBusy = () => false, report = async () => {}, identity,
|
||||
} = {}) {
|
||||
if (typeof root !== 'string') throw new Error('file task root is required')
|
||||
this.root = root
|
||||
this.inventory = inventory
|
||||
this.isManaged = isManaged
|
||||
this.protectedRoots = new Set(protectedRoots.map(normalizeAbsolutePath))
|
||||
this.onState = onState
|
||||
this.platformBusy = platformBusy
|
||||
this.report = report
|
||||
this.identity = identity ?? inventory.identity ?? new UnixIdentityResolver()
|
||||
this.tasks = new Map()
|
||||
this.activeMutation = undefined
|
||||
this.queue = []
|
||||
this.queueTail = Promise.resolve()
|
||||
this.search = new FileSearchManager({ inventory, onState: state => onState(state) })
|
||||
this.sizes = new FileSizeManager({ inventory, onState: state => onState(state) })
|
||||
}
|
||||
|
||||
get hasManagedMutation() { return this.queue.some(task => ['queued', 'running'].includes(task.status) && task.managed === true) }
|
||||
|
||||
wouldManage(input) {
|
||||
if (['search', 'size'].includes(input?.operation)) return false
|
||||
const sourceManaged = Array.isArray(input?.sources) && input.sources.some(source => typeof source?.path === 'string' && this.isManaged(source.path))
|
||||
return sourceManaged || (typeof input?.destination === 'string' && this.isManaged(input.destination))
|
||||
}
|
||||
|
||||
async initialize() {
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const files = (await readdir(this.root)).filter(name => name.endsWith('.json')).sort()
|
||||
for (const name of files) {
|
||||
try {
|
||||
const task = JSON.parse(await readFile(join(this.root, name), 'utf8'))
|
||||
if (task.schema !== 1 || typeof task.taskId !== 'string') continue
|
||||
this.tasks.set(task.taskId, task)
|
||||
if (task.status === 'running') await this.#recover(task)
|
||||
else if (task.status === 'queued' && ['upload', 'download'].includes(task.operation)) {
|
||||
task.status = 'running'
|
||||
await this.#recover(task)
|
||||
} else if (task.status === 'queued') this.#schedule(task)
|
||||
} catch (error) {
|
||||
this.onState({ operation: 'recovery', status: 'failed', error: error instanceof Error ? error.message : String(error) })
|
||||
}
|
||||
}
|
||||
await this.#prune()
|
||||
}
|
||||
|
||||
start(input) {
|
||||
if (input?.operation === 'search') return this.search.start(input)
|
||||
if (input?.operation === 'size') return this.sizes.start(input)
|
||||
if (input === null || typeof input !== 'object' || !OPERATIONS.has(input.operation)) throw new FileManagerError('file task operation is invalid')
|
||||
const sources = Array.isArray(input.sources) ? input.sources.map(source => ({ path: normalizeAbsolutePath(source.path), revision: source.revision })) : []
|
||||
const destination = input.destination === undefined ? null : normalizeAbsolutePath(input.destination)
|
||||
const conflict = input.conflict ?? 'reject'
|
||||
if (!['reject', 'overwrite', 'rename'].includes(conflict)) throw new FileManagerError('file task conflict mode is invalid')
|
||||
if (['copy', 'move', 'delete', 'rename', 'archive', 'extract'].includes(input.operation) && sources.length === 0) throw new FileManagerError('file task sources are required')
|
||||
if (input.operation === 'extract' && sources.length !== 1) throw new FileManagerError('archive extraction requires one source')
|
||||
if (input.operation === 'attributes' && sources.length !== 1) throw new FileManagerError('attribute changes require one source')
|
||||
if (['mkdir', 'touch', 'copy', 'move', 'rename', 'archive', 'extract'].includes(input.operation) && destination === null) throw new FileManagerError('file task destination is required')
|
||||
if (input.operation === 'delete' && sources.some(source => this.protectedRoots.has(source.path))) throw new FileManagerError('protected root cannot be deleted', 403, 'PROTECTED_ROOT')
|
||||
const managed = sources.some(source => this.isManaged(source.path)) || (destination !== null && this.isManaged(destination))
|
||||
if (managed && this.platformBusy()) throw new FileManagerError('a platform operation is already running', 409, 'FILE_TASK_CONFLICT')
|
||||
const task = {
|
||||
schema: 1, taskId: randomUUID(), operation: input.operation,
|
||||
status: this.queue.length === 0 ? 'running' : 'queued', phase: this.queue.length === 0 ? 'validating' : 'queued',
|
||||
sources, destination, destinationRevision: input.destinationRevision ?? null, conflict, managed,
|
||||
attributes: input.operation === 'attributes' ? normalizeAttributes(input.attributes) : null,
|
||||
archiveFormat: ['archive', 'extract'].includes(input.operation) ? normalizeArchiveFormat(input.archiveFormat) : null,
|
||||
processedBytes: 0, totalBytes: 0, processedEntries: 0, totalEntries: 0, currentPath: null,
|
||||
published: [], staging: [], hidden: [], currentSource: null, currentDestination: null,
|
||||
error: null, errorCode: null, queuePosition: null, createdAt: now(), updatedAt: now(), cancelRequested: false,
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#schedule(task)
|
||||
return publicTask(task)
|
||||
}
|
||||
|
||||
startTransfer({ operation, path, totalBytes = null, managed = false }, executor) {
|
||||
if (!['upload', 'download'].includes(operation) || typeof executor !== 'function') {
|
||||
throw new FileManagerError('file transfer task is invalid')
|
||||
}
|
||||
const normalizedPath = normalizeAbsolutePath(path)
|
||||
if (managed && this.platformBusy()) throw new FileManagerError('a platform operation is already running', 409, 'FILE_TASK_CONFLICT')
|
||||
let resolveResult
|
||||
let rejectResult
|
||||
const result = new Promise((resolve, reject) => { resolveResult = resolve; rejectResult = reject })
|
||||
const task = {
|
||||
schema: 1, taskId: randomUUID(), operation,
|
||||
status: this.queue.length === 0 ? 'running' : 'queued', phase: this.queue.length === 0 ? 'transferring' : 'queued',
|
||||
path: normalizedPath, sources: [], destination: operation === 'upload' ? normalizedPath : null,
|
||||
destinationRevision: null, conflict: null, managed, attributes: null, archiveFormat: null,
|
||||
processedBytes: 0, totalBytes, processedEntries: 0, totalEntries: 1, currentPath: normalizedPath,
|
||||
published: [], staging: [], hidden: [], currentSource: null, currentDestination: null,
|
||||
error: null, errorCode: null, queuePosition: null, createdAt: now(), updatedAt: now(), cancelRequested: false,
|
||||
executor, resolveResult, rejectResult,
|
||||
}
|
||||
this.tasks.set(task.taskId, task)
|
||||
this.#schedule(task)
|
||||
return { task: publicTask(task), result }
|
||||
}
|
||||
|
||||
list() {
|
||||
return [...this.search.list(), ...this.sizes.list(), ...[...this.tasks.values()].map(publicTask)].sort((a, b) => b.createdAt.localeCompare(a.createdAt)).slice(0, 100)
|
||||
}
|
||||
|
||||
get(taskId, options = {}) {
|
||||
if (this.tasks.has(taskId)) return publicTask(this.tasks.get(taskId))
|
||||
try { return this.search.get(taskId, options) } catch (error) {
|
||||
if (error?.statusCode !== 404) throw error
|
||||
return this.sizes.get(taskId)
|
||||
}
|
||||
}
|
||||
|
||||
completion(taskId) {
|
||||
const task = this.tasks.get(taskId) ?? this.search.tasks.get(taskId) ?? this.sizes.tasks.get(taskId)
|
||||
if (task === undefined) throw new FileManagerError('file task was not found', 404, 'FILE_TASK_NOT_FOUND')
|
||||
return Promise.resolve(task.completion).then(() => this.get(taskId))
|
||||
}
|
||||
|
||||
cancel(taskId) {
|
||||
const task = this.tasks.get(taskId)
|
||||
if (task === undefined) {
|
||||
try { return this.search.cancel(taskId) } catch (error) {
|
||||
if (error?.statusCode !== 404) throw error
|
||||
return this.sizes.cancel(taskId)
|
||||
}
|
||||
}
|
||||
if (!['queued', 'running'].includes(task.status)) throw new FileManagerError('file task is already complete', 409, 'FILE_TASK_COMPLETE')
|
||||
if (task.status === 'queued') {
|
||||
task.cancelRequested = true
|
||||
task.status = 'cancelled'
|
||||
task.phase = 'cancelled'
|
||||
task.updatedAt = now()
|
||||
this.queue = this.queue.filter(value => value !== task)
|
||||
void Promise.resolve(task.persisted).then(() => this.#terminal(task, 'cancelled', 'cancelled'))
|
||||
task.rejectResult?.(Object.assign(new Error('file transfer cancelled'), { code: 'FILE_TASK_CANCELLED' }))
|
||||
this.#publishQueue()
|
||||
return publicTask(task)
|
||||
}
|
||||
if (['destination-committed', 'source-hidden', 'cleaning'].includes(task.phase)) throw new FileManagerError('file task passed its cancellation boundary', 409, 'FILE_TASK_COMMITTED')
|
||||
task.cancelRequested = true
|
||||
task.abortController?.abort()
|
||||
this.#publish(task)
|
||||
return publicTask(task)
|
||||
}
|
||||
|
||||
#schedule(task) {
|
||||
this.queue.push(task)
|
||||
this.#publishQueue()
|
||||
const previous = this.queueTail
|
||||
task.persisted = this.#persist(task)
|
||||
task.completion = Promise.all([previous.catch(() => {}), task.persisted]).then(async () => {
|
||||
if (task.cancelRequested || task.status === 'cancelled') return
|
||||
task.status = 'running'
|
||||
task.phase = 'validating'
|
||||
task.queuePosition = 0
|
||||
this.activeMutation = task
|
||||
await this.#phase(task, task.executor === undefined ? 'validating' : 'transferring')
|
||||
if (task.executor === undefined) await this.#run(task).catch(() => {})
|
||||
else await this.#runTransfer(task)
|
||||
}).finally(() => {
|
||||
if (this.activeMutation === task) this.activeMutation = undefined
|
||||
this.queue = this.queue.filter(value => value !== task)
|
||||
this.#publishQueue()
|
||||
})
|
||||
this.queueTail = task.completion
|
||||
}
|
||||
|
||||
async #runTransfer(task) {
|
||||
const controller = new AbortController()
|
||||
task.abortController = controller
|
||||
const controls = {
|
||||
signal: controller.signal,
|
||||
progress: ({ processedBytes, totalBytes, currentPath } = {}) => {
|
||||
if (Number.isSafeInteger(processedBytes) && processedBytes >= 0) task.processedBytes = processedBytes
|
||||
if (Number.isSafeInteger(totalBytes) && totalBytes >= 0) task.totalBytes = totalBytes
|
||||
if (typeof currentPath === 'string') task.currentPath = currentPath
|
||||
task.updatedAt = now()
|
||||
this.#publish(task)
|
||||
},
|
||||
}
|
||||
try {
|
||||
const result = await task.executor(controls)
|
||||
task.processedEntries = 1
|
||||
if (task.totalBytes !== null) task.processedBytes = task.totalBytes
|
||||
await this.#terminal(task, 'success', 'completed')
|
||||
await this.report(`file-task.${task.operation}.completed`, publicTask(task))
|
||||
task.resolveResult(result)
|
||||
} catch (error) {
|
||||
const status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#terminal(task, status, status)
|
||||
await this.report(`file-task.${task.operation}.${status}`, { ...publicTask(task), error })
|
||||
task.rejectResult(error)
|
||||
} finally {
|
||||
task.abortController = undefined
|
||||
await this.#prune()
|
||||
}
|
||||
}
|
||||
|
||||
#publishQueue() {
|
||||
let position = 0
|
||||
for (const task of this.queue) {
|
||||
if (task.status === 'queued') position += 1
|
||||
task.queuePosition = task.status === 'queued' ? position : 0
|
||||
this.#publish(task)
|
||||
}
|
||||
}
|
||||
|
||||
async #validate(task) {
|
||||
for (const source of task.sources) {
|
||||
const item = await this.inventory.stat(source.path)
|
||||
if (source.revision !== item.revision) throw new FileRevisionConflictError(`source changed: ${source.path}`)
|
||||
if (task.operation === 'attributes' && !['file', 'directory'].includes(item.type)) {
|
||||
throw new FileManagerError('attributes can be changed only for files and directories', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
}
|
||||
const metrics = task.operation === 'attributes' && task.attributes.recursive === false
|
||||
? { entries: 1, bytes: item.type === 'file' ? item.size : 0 }
|
||||
: await treeMetrics(source.path)
|
||||
task.totalEntries += metrics.entries
|
||||
task.totalBytes += metrics.bytes
|
||||
}
|
||||
if (task.destinationRevision !== null) {
|
||||
const destinationRoot = task.operation === 'rename' || task.operation === 'mkdir' || task.operation === 'touch'
|
||||
? dirname(task.destination) : task.destination
|
||||
const listed = await this.inventory.list(destinationRoot, { limit: 1 })
|
||||
if (listed.revision !== task.destinationRevision) throw new FileRevisionConflictError('destination directory changed')
|
||||
}
|
||||
}
|
||||
|
||||
async #run(task) {
|
||||
try {
|
||||
await this.#validate(task)
|
||||
await this.#phase(task, 'mutating')
|
||||
if (task.operation === 'mkdir') await mkdir(task.destination)
|
||||
else if (task.operation === 'touch') {
|
||||
const handle = await open(task.destination, 'wx', 0o644)
|
||||
await handle.sync(); await handle.close(); await syncDirectory(dirname(task.destination))
|
||||
} else if (task.operation === 'rename') await this.#rename(task)
|
||||
else if (task.operation === 'copy') await this.#copy(task, false)
|
||||
else if (task.operation === 'move') await this.#copy(task, true)
|
||||
else if (task.operation === 'delete') await this.#delete(task)
|
||||
else if (task.operation === 'attributes') await this.#attributes(task)
|
||||
else if (task.operation === 'archive') await this.#archive(task)
|
||||
else if (task.operation === 'extract') await this.#extract(task)
|
||||
await this.#terminal(task, 'success', 'completed')
|
||||
await this.report(`file-task.${task.operation}.completed`, publicTask(task))
|
||||
} catch (error) {
|
||||
const status = task.cancelRequested ? 'cancelled' : 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#cleanupStaging(task)
|
||||
await this.#terminal(task, status, status)
|
||||
await this.report(`file-task.${task.operation}.${status}`, { ...publicTask(task), error })
|
||||
throw error
|
||||
} finally {
|
||||
await this.#prune()
|
||||
}
|
||||
}
|
||||
|
||||
async #rename(task) {
|
||||
if (task.sources.length !== 1) throw new FileManagerError('rename requires one source')
|
||||
const source = task.sources[0].path
|
||||
if (dirname(source) !== dirname(task.destination)) throw new FileManagerError('rename destination must use the same directory')
|
||||
const destination = await uniqueDestination(task.destination, task.conflict)
|
||||
task.currentSource = source
|
||||
task.currentDestination = destination
|
||||
await this.#phase(task, 'move-prepared')
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(source, destination)
|
||||
task.published.push(destination)
|
||||
await this.#phase(task, 'destination-committed')
|
||||
await syncDirectory(dirname(destination))
|
||||
task.currentSource = null
|
||||
task.currentDestination = null
|
||||
await this.#phase(task, 'mutating')
|
||||
}
|
||||
|
||||
async #copy(task, move) {
|
||||
for (const source of task.sources) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const destination = await uniqueDestination(join(task.destination, cleanName(source.path)), task.conflict)
|
||||
if (source.path === destination || destination.startsWith(`${source.path}/`)) throw new FileManagerError('directory cannot be copied into itself')
|
||||
if (move) {
|
||||
task.currentSource = source.path
|
||||
task.currentDestination = destination
|
||||
await this.#phase(task, 'move-prepared')
|
||||
try {
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(source.path, destination)
|
||||
task.published.push(destination)
|
||||
await this.#phase(task, 'destination-committed')
|
||||
task.processedEntries += 1
|
||||
task.currentSource = null
|
||||
task.currentDestination = null
|
||||
await this.#phase(task, 'mutating')
|
||||
continue
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EXDEV') throw error
|
||||
}
|
||||
}
|
||||
const staging = join(dirname(destination), `.dsh-file-task-${task.taskId}-${randomUUID()}.tmp`)
|
||||
task.staging.push(staging)
|
||||
task.currentPath = source.path
|
||||
task.currentSource = source.path
|
||||
task.currentDestination = destination
|
||||
await this.#persist(task)
|
||||
await copyEntry(source.path, staging)
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(staging, destination)
|
||||
task.staging = task.staging.filter(path => path !== staging)
|
||||
task.published.push(destination)
|
||||
await this.#phase(task, move ? 'destination-committed' : 'mutating')
|
||||
if (move) await rm(source.path, { recursive: true })
|
||||
task.processedEntries += 1
|
||||
task.processedBytes = Math.min(task.totalBytes, task.processedBytes + (await treeMetrics(destination)).bytes)
|
||||
task.currentSource = null
|
||||
task.currentDestination = null
|
||||
await this.#phase(task, 'mutating')
|
||||
}
|
||||
}
|
||||
|
||||
async #delete(task) {
|
||||
for (const source of task.sources) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const hidden = join(dirname(source.path), `.dsh-delete-${task.taskId}-${randomUUID()}.tmp`)
|
||||
await rename(source.path, hidden)
|
||||
task.hidden.push(hidden)
|
||||
await this.#phase(task, 'source-hidden')
|
||||
const removed = await treeMetrics(hidden)
|
||||
await rm(hidden, { recursive: true })
|
||||
task.hidden = task.hidden.filter(path => path !== hidden)
|
||||
task.processedEntries += removed.entries
|
||||
task.processedBytes = Math.min(task.totalBytes, task.processedBytes + removed.bytes)
|
||||
await this.#phase(task, 'cleaning')
|
||||
}
|
||||
}
|
||||
|
||||
async #attributes(task) {
|
||||
const [uid, gid] = await Promise.all([
|
||||
this.identity.userId(task.attributes.user),
|
||||
this.identity.groupId(task.attributes.group),
|
||||
])
|
||||
const mode = Number.parseInt(task.attributes.mode, 8)
|
||||
const pending = [task.sources[0].path]
|
||||
task.resolvedAttributes = { uid, gid, mode }
|
||||
while (pending.length > 0) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const path = pending.pop()
|
||||
const details = await lstat(path)
|
||||
if (task.attributes.recursive && details.isDirectory() && !details.isSymbolicLink()) {
|
||||
for (const name of await readdir(path)) pending.push(join(path, name))
|
||||
}
|
||||
await lchown(path, uid, gid)
|
||||
if (!details.isSymbolicLink()) await chmod(path, mode)
|
||||
assertAttributesApplied(path, await lstat(path), { uid, gid, mode }, details.isSymbolicLink())
|
||||
task.currentPath = path
|
||||
task.processedEntries += 1
|
||||
if (details.isFile()) task.processedBytes += details.size
|
||||
if (task.processedEntries % 100 === 0) await this.#phase(task, 'mutating')
|
||||
}
|
||||
task.currentPath = null
|
||||
}
|
||||
|
||||
async #archive(task) {
|
||||
const output = await uniqueDestination(task.destination, task.conflict)
|
||||
const sourceRoot = join(dirname(output), `.dsh-archive-input-${task.taskId}`)
|
||||
const staging = join(dirname(output), `.dsh-archive-${task.taskId}.tmp`)
|
||||
task.staging.push(sourceRoot, staging)
|
||||
await mkdir(sourceRoot)
|
||||
const names = new Set()
|
||||
for (const source of task.sources) {
|
||||
const name = cleanName(source.path)
|
||||
if (names.has(name)) throw new FileManagerError('archive source names must be unique', 409, 'FILE_EXISTS')
|
||||
names.add(name)
|
||||
task.currentPath = source.path
|
||||
await copyEntry(source.path, join(sourceRoot, name))
|
||||
task.processedEntries += 1
|
||||
await this.#phase(task, 'staging')
|
||||
}
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
const controller = new AbortController()
|
||||
task.abortController = controller
|
||||
try {
|
||||
await createArchive({ format: task.archiveFormat, sourceRoot, output: staging, signal: controller.signal })
|
||||
} finally {
|
||||
task.abortController = undefined
|
||||
}
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
if (task.conflict === 'overwrite' && await exists(output)) await rm(output, { recursive: true })
|
||||
await rename(staging, output)
|
||||
task.staging = task.staging.filter(path => path !== staging)
|
||||
task.published.push(output)
|
||||
task.processedBytes = task.totalBytes
|
||||
await rm(sourceRoot, { recursive: true, force: true })
|
||||
task.staging = task.staging.filter(path => path !== sourceRoot)
|
||||
task.currentPath = null
|
||||
await this.#phase(task, 'destination-committed')
|
||||
}
|
||||
|
||||
async #extract(task) {
|
||||
const archive = task.sources[0].path
|
||||
const staging = join(task.destination, `.dsh-extract-${task.taskId}.tmp`)
|
||||
task.staging.push(staging)
|
||||
task.currentPath = archive
|
||||
await this.#persist(task)
|
||||
const controller = new AbortController()
|
||||
task.abortController = controller
|
||||
let metrics
|
||||
try {
|
||||
metrics = await extractArchive({ format: task.archiveFormat, archive, output: staging, signal: controller.signal })
|
||||
} finally {
|
||||
task.abortController = undefined
|
||||
}
|
||||
task.totalEntries = metrics.entries
|
||||
task.totalBytes = metrics.bytes
|
||||
await this.#publishExtraction(task, staging)
|
||||
}
|
||||
|
||||
async #publishExtraction(task, staging) {
|
||||
const plan = []
|
||||
for (const name of await readdir(staging)) {
|
||||
plan.push({ source: join(staging, name), destination: await uniqueDestination(join(task.destination, name), task.conflict) })
|
||||
}
|
||||
for (const { source, destination } of plan) {
|
||||
if (task.cancelRequested) throw new Error('file task cancelled')
|
||||
if (task.conflict === 'overwrite' && await exists(destination)) await rm(destination, { recursive: true })
|
||||
await rename(source, destination)
|
||||
task.published.push(destination)
|
||||
const item = await treeMetrics(destination)
|
||||
task.processedEntries += item.entries
|
||||
task.processedBytes += item.bytes
|
||||
await this.#phase(task, 'publishing')
|
||||
}
|
||||
await rm(staging, { recursive: true, force: true })
|
||||
task.staging = task.staging.filter(path => path !== staging)
|
||||
task.currentPath = null
|
||||
}
|
||||
|
||||
async #cleanupStaging(task) {
|
||||
for (const path of task.staging) await rm(path, { recursive: true, force: true }).catch(() => {})
|
||||
task.staging = []
|
||||
}
|
||||
|
||||
async #recover(task) {
|
||||
if (task.operation === 'attributes') {
|
||||
try {
|
||||
task.cancelRequested = false
|
||||
task.processedEntries = 0
|
||||
task.processedBytes = 0
|
||||
await this.#phase(task, 'mutating')
|
||||
await this.#attributes(task)
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
await this.report('file-task.attributes.recovered', publicTask(task))
|
||||
} catch (error) {
|
||||
task.status = 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#phase(task, 'failed')
|
||||
await this.report('file-task.attributes.recovery-failed', { ...publicTask(task), error })
|
||||
}
|
||||
} else if (task.operation === 'archive' && task.phase === 'destination-committed') {
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else if (task.operation === 'extract' && task.phase === 'publishing' && task.staging?.length === 1 && await exists(task.staging[0])) {
|
||||
try {
|
||||
task.cancelRequested = false
|
||||
await this.#publishExtraction(task, task.staging[0])
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
await this.report('file-task.extract.recovered', publicTask(task))
|
||||
} catch (error) {
|
||||
task.status = 'failed'
|
||||
task.error = error instanceof Error ? error.message : String(error)
|
||||
task.errorCode = typeof error?.code === 'string' ? error.code : null
|
||||
await this.#phase(task, 'failed')
|
||||
await this.report('file-task.extract.recovery-failed', { ...publicTask(task), error })
|
||||
}
|
||||
} else if (task.operation === 'delete' || task.phase === 'source-hidden' || task.phase === 'cleaning') {
|
||||
for (const path of task.hidden ?? []) await rm(path, { recursive: true, force: true })
|
||||
task.hidden = []
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else if (['move', 'rename'].includes(task.operation) && task.phase === 'destination-committed') {
|
||||
if (task.currentSource !== null && await exists(task.currentSource)) await rm(task.currentSource, { recursive: true })
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else if (['move', 'rename'].includes(task.operation) && task.phase === 'move-prepared'
|
||||
&& task.currentSource !== null && task.currentDestination !== null
|
||||
&& !await exists(task.currentSource) && await exists(task.currentDestination)) {
|
||||
task.published ??= []
|
||||
if (!task.published.includes(task.currentDestination)) task.published.push(task.currentDestination)
|
||||
task.status = 'success'
|
||||
await this.#phase(task, 'completed')
|
||||
} else {
|
||||
await this.#cleanupStaging(task)
|
||||
task.status = 'interrupted'
|
||||
task.error = 'file operation was interrupted before a recoverable commit'
|
||||
await this.#phase(task, 'interrupted')
|
||||
}
|
||||
}
|
||||
|
||||
async #phase(task, phase) {
|
||||
task.phase = phase
|
||||
task.updatedAt = now()
|
||||
await this.#persist(task)
|
||||
this.#publish(task)
|
||||
}
|
||||
|
||||
async #terminal(task, status, phase) {
|
||||
task.status = status
|
||||
task.phase = phase
|
||||
task.updatedAt = now()
|
||||
await this.#persist(task)
|
||||
if (this.activeMutation === task) this.activeMutation = undefined
|
||||
this.queue = this.queue.filter(value => value !== task)
|
||||
this.#publishQueue()
|
||||
this.#publish(task)
|
||||
}
|
||||
|
||||
async #persist(task) {
|
||||
await mkdir(this.root, { recursive: true })
|
||||
const value = publicTask(task)
|
||||
await durableReplace(join(this.root, `${task.taskId}.json`), Buffer.from(`${JSON.stringify(value)}\n`))
|
||||
}
|
||||
|
||||
#publish(task) { this.onState(publicTask(task)) }
|
||||
|
||||
async #prune() {
|
||||
const completed = [...this.tasks.values()].filter(task => !['queued', 'running'].includes(task.status)).sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
|
||||
for (const task of completed.slice(100)) {
|
||||
this.tasks.delete(task.taskId)
|
||||
await rm(join(this.root, `${task.taskId}.json`), { force: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const fileTaskInternals = Object.freeze({ protectedDeleteRoots: DEFAULT_PROTECTED_DELETE_ROOTS, uniqueDestination, treeMetrics, normalizeAttributes, assertAttributesApplied })
|
||||
@@ -0,0 +1,205 @@
|
||||
import { createReadStream } from 'node:fs'
|
||||
import { link, lstat, mkdir, mkdtemp, open, readlink, rename, rm, stat } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { basename, dirname, extname, join } from 'node:path'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import { Readable } from 'node:stream'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import {
|
||||
FileManagerError, FileRevisionConflictError, fileError, fileManagerInternals,
|
||||
isManagedPath, normalizeAbsolutePath,
|
||||
} from './index.mjs'
|
||||
import { createArchive } from './archives.mjs'
|
||||
|
||||
function parseRange(value, size) {
|
||||
if (value === undefined) return { start: 0, end: Math.max(0, size - 1), partial: false }
|
||||
const match = /^bytes=(\d*)-(\d*)$/.exec(value)
|
||||
if (match === null || (match[1] === '' && match[2] === '')) throw new FileManagerError('range is invalid', 416, 'RANGE_INVALID')
|
||||
let start
|
||||
let end
|
||||
if (match[1] === '') {
|
||||
const suffix = Number(match[2])
|
||||
if (!Number.isSafeInteger(suffix) || suffix < 1) throw new FileManagerError('range is invalid', 416, 'RANGE_INVALID')
|
||||
start = Math.max(0, size - suffix)
|
||||
end = size - 1
|
||||
} else {
|
||||
start = Number(match[1])
|
||||
end = match[2] === '' ? size - 1 : Number(match[2])
|
||||
}
|
||||
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || start >= size || end < start) {
|
||||
throw new FileManagerError('range is not satisfiable', 416, 'RANGE_INVALID')
|
||||
}
|
||||
return { start, end: Math.min(end, size - 1), partial: true }
|
||||
}
|
||||
|
||||
function disposition(name) {
|
||||
const fallback = name.replace(/[^\x20-\x7e]/g, '_').replace(/["\\]/g, '_') || 'download'
|
||||
return `attachment; filename="${fallback}"; filename*=UTF-8''${encodeURIComponent(name)}`
|
||||
}
|
||||
|
||||
async function syncDirectory(path) {
|
||||
const handle = await open(path, 'r')
|
||||
try { await handle.sync() } finally { await handle.close() }
|
||||
}
|
||||
|
||||
async function existingMetadata(path, { requireFile = false } = {}) {
|
||||
try {
|
||||
const value = await stat(path, { bigint: true })
|
||||
if (requireFile && !value.isFile()) throw new FileManagerError('upload target is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
return value
|
||||
} catch (error) {
|
||||
if (error?.code === 'ENOENT') return undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function renamedPath(path, index) {
|
||||
const extension = extname(path)
|
||||
const stem = extension === '' ? path : path.slice(0, -extension.length)
|
||||
return `${stem} (${String(index)})${extension}`
|
||||
}
|
||||
|
||||
export class FileTransferManager {
|
||||
constructor({ isManaged = isManagedPath, stagingRoot = tmpdir() } = {}) {
|
||||
this.isManaged = isManaged
|
||||
this.stagingRoot = stagingRoot
|
||||
}
|
||||
|
||||
async openDownload(value, { revision, range, signal } = {}) {
|
||||
const path = normalizeAbsolutePath(value)
|
||||
let handle
|
||||
let cleanup
|
||||
try {
|
||||
const pathStats = await lstat(path, { bigint: true })
|
||||
if (pathStats.isDirectory()) {
|
||||
if (range !== undefined) throw new FileManagerError('directory downloads do not support ranges', 416, 'RANGE_INVALID')
|
||||
const actualRevision = fileManagerInternals.revisionFor(path, pathStats)
|
||||
if (revision !== undefined && revision !== '' && revision !== actualRevision) throw new FileRevisionConflictError('download revision changed')
|
||||
await mkdir(this.stagingRoot, { recursive: true })
|
||||
const staging = await mkdtemp(join(this.stagingRoot, '.dsh-directory-download-'))
|
||||
cleanup = () => rm(staging, { recursive: true, force: true })
|
||||
const name = basename(path) || 'root'
|
||||
const archive = join(staging, `${name}.zip`)
|
||||
await createArchive({ format: 'zip', sourceRoot: path === '/' ? path : dirname(path), output: archive, entries: [path === '/' ? '.' : name], signal })
|
||||
const download = await this.openDownload(archive, { signal })
|
||||
download.path = path
|
||||
download.revision = actualRevision
|
||||
download.headers['content-disposition'] = disposition(`${name}.zip`)
|
||||
download.cleanup = cleanup
|
||||
cleanup = undefined
|
||||
return download
|
||||
}
|
||||
handle = await open(path, 'r')
|
||||
const before = await handle.stat({ bigint: true })
|
||||
if (!before.isFile()) throw new FileManagerError('path is not a regular file', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const actualRevision = fileManagerInternals.revisionFor(path, before, await readlink(path).catch(() => ''))
|
||||
if (revision !== undefined && revision !== '' && revision !== actualRevision) throw new FileRevisionConflictError('download revision changed')
|
||||
const size = Number(before.size)
|
||||
if (!Number.isSafeInteger(size)) throw new FileManagerError('file is too large', 413, 'FILE_TOO_LARGE')
|
||||
if (size === 0 && range !== undefined) throw new FileManagerError('range is not satisfiable', 416, 'RANGE_INVALID')
|
||||
const selected = size === 0 ? { start: 0, end: -1, partial: false } : parseRange(range, size)
|
||||
const length = selected.end < selected.start ? 0 : selected.end - selected.start + 1
|
||||
const stream = length === 0 ? Readable.from([]) : createReadStream(path, {
|
||||
fd: handle.fd, autoClose: false, start: selected.start, end: selected.end,
|
||||
})
|
||||
return {
|
||||
path, handle, stream, revision: actualRevision,
|
||||
status: selected.partial ? 206 : 200,
|
||||
headers: {
|
||||
'accept-ranges': 'bytes',
|
||||
'cache-control': 'no-store',
|
||||
'content-disposition': disposition(basename(path)),
|
||||
'content-length': String(length),
|
||||
'content-type': 'application/octet-stream',
|
||||
etag: `"${actualRevision}"`,
|
||||
...(selected.partial ? { 'content-range': `bytes ${String(selected.start)}-${String(selected.end)}/${String(size)}` } : {}),
|
||||
},
|
||||
}
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {})
|
||||
await cleanup?.().catch(() => {})
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
|
||||
async sendDownload(response, download, { onProgress = () => {}, signal } = {}) {
|
||||
response.writeHead(download.status, download.headers)
|
||||
let sent = 0
|
||||
download.stream.on('data', chunk => {
|
||||
sent += chunk.byteLength
|
||||
onProgress(sent, Number(download.headers['content-length']))
|
||||
})
|
||||
try {
|
||||
await pipeline(download.stream, response, { signal })
|
||||
} finally {
|
||||
await download.handle.close().catch(() => {})
|
||||
await download.cleanup?.().catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
async upload(input, value, { conflict = 'reject', contentLength, onProgress = () => {}, signal } = {}) {
|
||||
const requestedPath = normalizeAbsolutePath(value)
|
||||
if (!['reject', 'overwrite', 'rename'].includes(conflict)) throw new FileManagerError('upload conflict mode is invalid')
|
||||
const parent = dirname(requestedPath)
|
||||
const parentStats = await stat(parent).catch(error => { throw fileError(error) })
|
||||
if (!parentStats.isDirectory()) throw new FileManagerError('upload parent is not a directory', 415, 'FILE_TYPE_UNSUPPORTED')
|
||||
const staging = join(parent, `.dsh-upload-${randomUUID()}.tmp`)
|
||||
let handle
|
||||
let received = 0
|
||||
try {
|
||||
const original = await existingMetadata(requestedPath, { requireFile: conflict === 'overwrite' })
|
||||
if (original !== undefined && conflict === 'reject') throw new FileManagerError('upload target already exists', 409, 'FILE_EXISTS')
|
||||
handle = await open(staging, 'wx', 0o600)
|
||||
for await (const chunk of input) {
|
||||
signal?.throwIfAborted()
|
||||
const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)
|
||||
received += bytes.byteLength
|
||||
await handle.write(bytes)
|
||||
onProgress(received, contentLength)
|
||||
}
|
||||
if (contentLength !== undefined && received !== contentLength) throw new FileManagerError('upload size does not match Content-Length')
|
||||
await handle.sync()
|
||||
if (original === undefined) await handle.chmod(0o644)
|
||||
else {
|
||||
await handle.chmod(Number(original.mode & 0o7777n))
|
||||
const current = await handle.stat({ bigint: true })
|
||||
if (current.uid !== original.uid || current.gid !== original.gid) await handle.chown(Number(original.uid), Number(original.gid))
|
||||
}
|
||||
await handle.close()
|
||||
handle = undefined
|
||||
let path = requestedPath
|
||||
if (conflict === 'overwrite') await rename(staging, path)
|
||||
else if (conflict === 'reject') {
|
||||
await link(staging, path)
|
||||
await rm(staging)
|
||||
} else {
|
||||
let index = 0
|
||||
for (;;) {
|
||||
path = index === 0 ? requestedPath : renamedPath(requestedPath, index)
|
||||
try {
|
||||
await link(staging, path)
|
||||
await rm(staging)
|
||||
break
|
||||
} catch (error) {
|
||||
if (error?.code !== 'EEXIST') throw error
|
||||
index += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
await syncDirectory(parent)
|
||||
const result = await stat(path, { bigint: true })
|
||||
return {
|
||||
path,
|
||||
size: received,
|
||||
revision: fileManagerInternals.revisionFor(path, result),
|
||||
managed: this.isManaged(path),
|
||||
}
|
||||
} catch (error) {
|
||||
await handle?.close().catch(() => {})
|
||||
await rm(staging, { force: true }).catch(() => {})
|
||||
throw fileError(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const fileTransferInternals = Object.freeze({ parseRange, disposition, renamedPath })
|
||||
Reference in new issue
Block a user