dsh/upstream/container/control-plane/services/gateway/test/availability.test.mjs

import assert from 'node:assert/strict'
import { createServer, request as httpRequest } from 'node:http'
import { connect as netConnect } from 'node:net'
import test from 'node:test'
import { DshAvailability, availabilityPage, language } from '../lib/availability.mjs'
import { closeGatewayServer, createGatewayServer as createGatewayServerBase, PLUGIN_FAILURE_PATH, READINESS_PATH, WAIT_PATH, safeReturnPath } from '../lib/proxy.mjs'
import { parseTrustedHosts } from '../lib/config.mjs'

const AUTHENTICATED_BROWSER = Object.freeze({
  status: async () => ({ state: 'initialized' }),
  validateDsh: async () => ({ authenticated: true }),
  validateManagement: async () => ({ authenticated: true }),
  enterManagement: async () => { throw new Error('unexpected Management login') },
  handle: async () => false,
})

const UNAUTHENTICATED_BROWSER = Object.freeze({
  status: async () => ({ state: 'initialized' }),
  validateDsh: async () => ({ authenticated: false }),
  validateManagement: async () => ({ authenticated: false }),
  enterManagement: async () => { throw new Error('unexpected Management login') },
  handle: async () => false,
})

function createGatewayServer(options) {
  return createGatewayServerBase({
    ...options,
    browserAuthentication: options.browserAuthentication ?? AUTHENTICATED_BROWSER,
  })
}

const ISOLATED_UNAUTHENTICATED_BROWSER = Object.freeze({
  ...UNAUTHENTICATED_BROWSER,
  status: async () => ({ state: 'initialized', account: {
    managementAccess: {
      mode: 'isolated',
      isolatedEntry: { kind: 'public', managementPublicOrigin: 'https://management.example' },
    },
  } }),
})

async function listen(server) {
  await new Promise((resolve, reject) => {
    server.once('error', reject)
    server.listen(0, '127.0.0.1', resolve)
  })
  return server.address().port
}

function request(port, path = '/', { method = 'GET', accept = 'text/html', headers = {} } = {}) {
  return new Promise((resolve, reject) => {
    const outgoing = httpRequest({
      hostname: '127.0.0.1', port, path, method,
      headers: { host: 'dsh.example', accept, ...headers },
    }, response => {
      const chunks = []
      response.on('data', chunk => chunks.push(chunk))
      response.on('end', () => resolve({
        status: response.statusCode,
        headers: response.headers,
        body: Buffer.concat(chunks).toString('utf8'),
      }))
    })
    outgoing.once('error', reject)
    outgoing.end()
  })
}

async function unavailableGateway({ platform = {}, availability = new DshAvailability(), probe = async () => false, report, browserAuthentication } = {}) {
  const placeholder = createServer()
  const unavailablePort = await listen(placeholder)
  await new Promise(resolve => placeholder.close(resolve))
  const gateway = createGatewayServer({
    trustedHosts: parseTrustedHosts({ DSH_TRUSTED_HOSTS: 'dsh.example' }),
    upstreamPort: unavailablePort,
    platformStatus: async () => platform,
    availability,
    probe,
    browserAuthentication,
    report,
    probeIntervalMs: 60_000,
  })
  return { gateway, port: await listen(gateway) }
}

test('authenticated DSH traffic waits for authoritative Bootstrap publication', async t => {
  let upstreamRequests = 0
  const upstream = createServer((_request, response) => {
    upstreamRequests += 1
    response.writeHead(200, { 'content-type': 'text/plain' })
    response.end('published')
  })
  const upstreamPort = await listen(upstream)
  t.after(() => upstream.close())
  let published = false
  const gateway = createGatewayServer({
    trustedHosts: parseTrustedHosts({ DSH_TRUSTED_HOSTS: 'dsh.example' }),
    upstreamPort,
    platformStatus: async () => ({ dshLifecycle: { state: 'running' } }),
    probe: async () => true,
    dshUpstreamAuthentication: {
      cookie: async () => null,
      publicReady: async () => published,
    },
    probeIntervalMs: 60_000,
  })
  const port = await listen(gateway)
  try {
    const socket = netConnect(port, '127.0.0.1')
    await new Promise((resolve, reject) => {
      socket.once('connect', resolve)
      socket.once('error', reject)
    })
    socket.write('GET /events HTTP/1.1\r\nHost: dsh.example\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n')
    const upgrade = await new Promise((resolve, reject) => {
      socket.once('data', data => resolve(data.toString('utf8')))
      socket.once('error', reject)
    })
    assert.match(upgrade, /^HTTP\/1\.1 503 Service Unavailable/)
    socket.destroy()
    const waiting = await request(port)
    assert.equal(waiting.status, 200)
    assert.match(waiting.body, /DeepSeek Harness is starting/)
    assert.equal(upstreamRequests, 0)
    assert.deepEqual(JSON.parse((await request(port, READINESS_PATH, { accept: 'application/json' })).body).ready, false)
    const dedicatedWaiting = await request(port, `${WAIT_PATH}?return=%2Fsessions%2Fcurrent`)
    assert.equal(dedicatedWaiting.status, 200)
    assert.match(dedicatedWaiting.body, /DeepSeek Harness is starting/)
    assert.match(dedicatedWaiting.body, /\/sessions\/current/)
    published = true
    const ready = await request(port)
    assert.equal(ready.status, 200)
    assert.equal(ready.body, 'published')
    assert.equal(upstreamRequests, 1)
    assert.equal(JSON.parse((await request(port, READINESS_PATH, { accept: 'application/json' })).body).ready, true)
  } finally {
    await closeGatewayServer(gateway)
  }
})

test('an intentional stopped state does not report an upstream proxy failure', async () => {
  const reports = []
  const context = await unavailableGateway({
    platform: { dshLifecycle: { state: 'stopped' } },
    report: (message, fields) => { reports.push({ message, fields }) },
  })
  try {
    const response = await request(context.port)
    assert.equal(response.status, 200)
    assert.match(response.body, /DeepSeek Harness is stopped/)
    await new Promise(resolve => setImmediate(resolve))
    assert.equal(reports.some(entry => entry.message === 'gateway.upstream.failed'), false)
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('an unauthenticated top-level navigation sees classified DSH recovery before the login page', async () => {
  const gateway = createGatewayServer({
    trustedHosts: parseTrustedHosts({ DSH_TRUSTED_HOSTS: 'dsh.example' }),
    upstreamPort: 1,
    platformStatus: async () => ({ dshLifecycle: { state: 'failed', error: 'private runtime detail' } }),
    probe: async () => false,
    browserAuthentication: UNAUTHENTICATED_BROWSER,
    probeIntervalMs: 60_000,
  })
  const port = await listen(gateway)
  try {
    const page = await request(port)
    assert.equal(page.status, 200)
    assert.match(page.body, /DeepSeek Harness could not start/)
    assert.match(page.body, /href="\/_dsh_platform\/console\/"/)
    assert.doesNotMatch(page.body, /private runtime detail|autocomplete="current-password"/)
    assert.equal((await request(port, '/api/sessions', { accept: 'application/json' })).status, 503)
  } finally {
    await closeGatewayServer(gateway)
  }
})

test('official-style holding page is self-contained and replaces the spinner with platform status', () => {
  const page = availabilityPage('switching', { 'accept-language': 'zh-CN' })
  assert.match(page, /HARNESS/)
  assert.match(page, /正在切换 DeepSeek Harness 运行版本/)
  assert.match(page, /href="\/_dsh_platform\/console\/"/)
  assert.match(page, /打开 DSH 管理中心进行检查和恢复/)
  assert.match(page, new RegExp(READINESS_PATH))
  assert.match(page, /prefers-color-scheme:light/)
  assert.match(page, /name="viewport"/)
  assert.match(page, /html\{height:100%\}body\{min-height:100%;margin:0;display:grid;place-items:center;/)
  assert.doesNotMatch(page, /100(?:d|s|l)?vh/)
  assert.match(page, /max-width:520px/)
  assert.match(page, /management\.href=value\.managementHref/)
  assert.doesNotMatch(page, /spinner|Loading plugins/)
  assert.doesNotMatch(page, /letter-spacing:-/)
})

test('persistent plugin failure page is terminal, localized, and links to Management', async () => {
  const page = availabilityPage('plugin-failed', { 'accept-language': 'zh-CN' }, { poll: false, allowRefresh: true })
  assert.match(page, /DeepSeek Harness 插件持续加载失败/)
  assert.match(page, /打开 DSH 管理中心进行检查和恢复/)
  assert.match(page, /刷新页面/)
  assert.doesNotMatch(page, /setTimeout\(check/)
  assert.doesNotMatch(availabilityPage('failed', {}, { poll: false }), /Reload page/)

  const context = await unavailableGateway()
  try {
    const response = await request(context.port, PLUGIN_FAILURE_PATH, {
      headers: { 'accept-language': 'zh-CN' },
    })
    assert.equal(response.status, 200)
    assert.match(response.body, /DeepSeek Harness 插件持续加载失败/)
    assert.equal((await request(context.port, PLUGIN_FAILURE_PATH, { method: 'POST' })).status, 405)
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('Gateway temporary DSH pages link to the configured isolated Management entry', async () => {
  const context = await unavailableGateway({ browserAuthentication: ISOLATED_UNAUTHENTICATED_BROWSER })
  try {
    const response = await request(context.port, PLUGIN_FAILURE_PATH)
    assert.equal(response.status, 200)
    assert.match(response.body, /href="https:\/\/management\.example\/"/)
    assert.doesNotMatch(response.body, /href="\/_dsh_platform\/console\/"/)
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('Gateway readiness publishes the current Management entry for an already-open temporary page', async () => {
  let access = { state: 'initialized', account: { managementAccess: { mode: 'compat' } } }
  const browserAuthentication = {
    ...UNAUTHENTICATED_BROWSER,
    status: async () => access,
  }
  const context = await unavailableGateway({ browserAuthentication })
  try {
    const compatible = JSON.parse((await request(context.port, READINESS_PATH)).body)
    assert.equal(compatible.managementHref, '/_dsh_platform/console/')

    access = await ISOLATED_UNAUTHENTICATED_BROWSER.status()
    const isolated = JSON.parse((await request(context.port, READINESS_PATH)).body)
    assert.equal(isolated.managementHref, 'https://management.example/')
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('holding pages prefer the last DSH locale cookie over browser language', () => {
  assert.equal(language({ cookie: 'theme=dark; dsh_locale=en', 'accept-language': 'zh-CN' }), 'en')
  assert.equal(language({ cookie: 'dsh_locale=zh', 'accept-language': 'en-US' }), 'zh')
  assert.equal(language({ cookie: 'dsh_locale=fr', 'accept-language': 'en-US' }), 'en')
  assert.equal(language({ 'accept-language': 'fr-FR, zh-CN;q=0.8, en;q=0.5' }), 'zh')
  assert.match(availabilityPage('recovering', { cookie: 'dsh_locale=zh', 'accept-language': 'en-US' }), /意外停止/)
  assert.match(availabilityPage('recovering', { cookie: 'dsh_locale=en', 'accept-language': 'zh-CN' }), /Open DSH Management Console for diagnostics and recovery/)
})

test('every classified outage page links to the standalone Platform Management console', () => {
  for (const state of ['starting', 'stopping', 'stopped', 'restarting', 'switching', 'runtime-recovering', 'recovering', 'failed', 'unavailable']) {
    const page = availabilityPage(state, { 'accept-language': 'en-US' })
    assert.match(page, /href="\/_dsh_platform\/console\/"/)
  }
})

test('intentional DSH restarts have explicit holding and failure states', () => {
  const availability = new DshAvailability()
  assert.equal(availability.classify({ operation: 'restarting' }), 'restarting')
  assert.equal(availability.classify({ operation: 'restart-failed' }), 'failed')
  assert.match(availabilityPage('restarting', { 'accept-language': 'zh-CN' }), /正在重新启动/)
  assert.equal(availability.classify({ dshLifecycle: { state: 'stopping' } }), 'stopping')
  assert.equal(availability.classify({ dshLifecycle: { state: 'stopped' } }), 'stopped')
  assert.equal(availability.classify({ dshLifecycle: { state: 'recovering', attempt: 2 } }), 'recovering')
  assert.match(availabilityPage('recovering', { 'accept-language': 'zh-CN' }, {
    lifecycle: { attempt: 2, maxAttempts: 3 },
  }), /第 2 \/ 3 次/)
})

test('holding-page return paths stay same-origin and reject ambiguous inputs', () => {
  assert.equal(safeReturnPath('/sessions/current?view=chat#latest'), '/sessions/current?view=chat#latest')
  for (const value of [null, '', '//example.com/path', '/\\example.com', 'https://example.com/', '/bad\npath']) {
    assert.equal(safeReturnPath(value), '/')
  }
  const page = availabilityPage('restarting', {}, { returnPath: '/</script><script>alert(1)</script>' })
  assert.doesNotMatch(page, /const returnPath="\/<\/script>/)
  assert.match(page, /\\u003c\/script>/)
})

test('cold start serves a holding page while API and WebSocket-shaped HTTP requests receive 503', async () => {
  const { gateway, port } = await unavailableGateway()
  try {
    const page = await request(port)
    assert.equal(page.status, 200)
    assert.match(page.body, /DeepSeek Harness is starting/)
    assert.equal(page.headers['cache-control'], 'no-store')
    assert.equal((await request(port, '/api/sessions', { accept: 'application/json' })).status, 503)
    assert.equal((await request(port, '/', { method: 'POST', accept: 'text/html' })).status, 503)
  } finally {
    await closeGatewayServer(gateway)
  }
})

test('an unresponsive management service does not block the cold-start holding page', async () => {
  const placeholder = createServer()
  const unavailablePort = await listen(placeholder)
  await new Promise(resolve => placeholder.close(resolve))
  const gateway = createGatewayServer({
    trustedHosts: parseTrustedHosts({ DSH_TRUSTED_HOSTS: 'dsh.example' }),
    upstreamPort: unavailablePort,
    platformStatus: () => new Promise(() => {}),
    probe: async () => false,
    probeIntervalMs: 60_000,
  })
  const port = await listen(gateway)
  try {
    const startedAt = Date.now()
    const page = await request(port)
    assert.equal(page.status, 200)
    assert.match(page.body, /DeepSeek Harness is starting/)
    assert.ok(Date.now() - startedAt < 1_000)
  } finally {
    await closeGatewayServer(gateway)
  }
})

test('cold-start management status failures remain diagnostic warnings', async () => {
  const reported = []
  const gateway = createGatewayServer({
    trustedHosts: parseTrustedHosts({ DSH_TRUSTED_HOSTS: 'dsh.example' }),
    upstreamPort: 1,
    platformStatus: async () => { throw Object.assign(new Error('socket is not ready'), { code: 'ENOENT' }) },
    probe: async () => false,
    probeIntervalMs: 60_000,
    report: async (message, fields) => reported.push({ message, fields }),
  })
  const port = await listen(gateway)
  try {
    assert.equal((await request(port)).status, 200)
    await new Promise(resolve => setImmediate(resolve))
    assert.equal(reported.length, 1)
    assert.equal(reported[0].message, 'gateway.platform-status.failed')
    assert.equal(reported[0].fields.level, 'warning')
    assert.equal(reported[0].fields.error.code, 'ENOENT')
  } finally {
    await closeGatewayServer(gateway)
  }
})

test('platform switching, recovery, and startup failure select distinct page messages', async () => {
  for (const [platform, expected] of [
    [{ operation: 'switching' }, /Switching the DeepSeek Harness runtime/],
    [{ operation: 'recovering' }, /Restoring the DeepSeek Harness runtime/],
    [{ recoveryMode: 'candidate failed' }, /DeepSeek Harness could not start/],
  ]) {
    const { gateway, port } = await unavailableGateway({ platform })
    try {
      const page = await request(port)
      assert.equal(page.status, 200)
      assert.match(page.body, expected)
      assert.equal((await request(port, '/api/status', { accept: 'application/json' })).status, 503)
    } finally {
      await closeGatewayServer(gateway)
    }
  }
})

test('dedicated holding route preserves a safe return path and redirects when ready', async () => {
  let ready = false
  const platform = { dshLifecycle: { state: 'stopping' } }
  const context = await unavailableGateway({
    platform,
    probe: async () => ready,
  })
  try {
    const target = '/sessions/current?view=chat#latest'
    const waiting = await request(context.port, `${WAIT_PATH}?return=${encodeURIComponent(target)}`)
    assert.equal(waiting.status, 200)
    assert.match(waiting.body, /DeepSeek Harness is stopping/)
    assert.match(waiting.body, /location\.replace\(returnPath\)/)
    assert.match(waiting.body, /\/sessions\/current\?view=chat#latest/)

    const invalid = await request(context.port, `${WAIT_PATH}?return=${encodeURIComponent('//example.com/')}`)
    assert.equal(invalid.status, 200)
    assert.match(invalid.body, /const returnPath="\/"/)

    ready = true
    const premature = await request(context.port, `${WAIT_PATH}?return=${encodeURIComponent(target)}`)
    assert.equal(premature.status, 200)
    assert.match(premature.body, /DeepSeek Harness is stopping/)
    const prematureReadiness = await request(context.port, READINESS_PATH)
    assert.equal(prematureReadiness.status, 503)
    assert.equal(JSON.parse(prematureReadiness.body).state, 'stopping')

    platform.dshLifecycle.state = 'running'
    const redirected = await request(context.port, `${WAIT_PATH}?return=${encodeURIComponent(target)}`)
    assert.equal(redirected.status, 302)
    assert.equal(redirected.headers.location, target)
    assert.deepEqual(JSON.parse((await request(context.port, READINESS_PATH)).body), {
      ready: true, state: 'ready', pluginRecoveryEligible: false,
      managementHref: '/_dsh_platform/console/',
    })
    assert.equal((await request(context.port, WAIT_PATH, { method: 'POST' })).status, 405)
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('WebSocket receives 503 during a classified DSH outage', async () => {
  const reports = []
  const { gateway, port } = await unavailableGateway({
    platform: { operation: 'switching' },
    report: (message, fields) => reports.push({ message, fields }),
  })
  const socket = netConnect(port, '127.0.0.1')
  try {
    await new Promise((resolve, reject) => {
      socket.once('connect', resolve)
      socket.once('error', reject)
    })
    socket.write('GET /events HTTP/1.1\r\nHost: dsh.example\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n')
    const response = await new Promise((resolve, reject) => {
      socket.once('data', data => resolve(data.toString('utf8')))
      socket.once('error', reject)
    })
    assert.match(response, /^HTTP\/1\.1 503 Service Unavailable/)
    await new Promise(resolve => setImmediate(resolve))
    const failure = reports.find(entry => entry.message === 'gateway.websocket.failed')
    assert.equal(failure?.fields.level, 'warning')
    assert.equal(failure?.fields.upstream, 'dsh')
  } finally {
    socket.destroy()
    await closeGatewayServer(gateway)
  }
})

test('one unexpected local failure remains 502 until repeated probes confirm unavailability', async () => {
  let now = 0
  const availability = new DshAvailability({ now: () => now, failures: 3, failureWindowMs: 1_500 })
  availability.observe(true)
  const context = await unavailableGateway({ availability })
  try {
    assert.equal((await request(context.port)).status, 502)
    availability.observe(false)
    now = 1_600
    availability.observe(false)
    const recovered = await request(context.port)
    assert.equal(recovered.status, 200)
    assert.match(recovered.body, /temporarily unavailable/)
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('a Bootstrap candidate handoff serves the switching page instead of a bare 502', async () => {
  const context = await unavailableGateway({ platform: { update: { status: 'building-candidate' } } })
  try {
    const response = await request(context.port)
    assert.equal(response.status, 200)
    assert.match(response.body, /Switching the DeepSeek Harness runtime/)
  } finally {
    await closeGatewayServer(context.gateway)
  }
})

test('readiness immediately reports ready and clears confirmed failure state', async () => {
  let ready = false
  let now = 0
  const availability = new DshAvailability({ now: () => now, failures: 1, failureWindowMs: 0 })
  availability.observe(true)
  availability.observe(false)
  const context = await unavailableGateway({ availability, probe: async () => ready })
  try {
    const unavailable = await request(context.port, READINESS_PATH, { accept: 'application/json' })
    assert.equal(unavailable.status, 503)
    assert.equal(JSON.parse(unavailable.body).state, 'unavailable')
    ready = true
    const healthy = await request(context.port, READINESS_PATH, { accept: 'application/json' })
    assert.equal(healthy.status, 200)
    assert.deepEqual(JSON.parse(healthy.body), {
      ready: true, state: 'ready', pluginRecoveryEligible: false,
      managementHref: '/_dsh_platform/console/',
    })
    assert.equal(availability.classify({}), 'unknown')
  } finally {
    await closeGatewayServer(context.gateway)
  }
})