test(web): run fixture browser cases through Host

This commit is contained in:
imccyu committed 2026-09-13 21:30:02 +08:00
1 parent 9b091919e2
commit eb2cb668a5
4 files changed
+143 -63

No files matched your search

+123 -41
View File
@@ -1,22 +1,29 @@
/**
* Opt-in browser stress reproduction for reasoning-stream renderer stalls.
* The fixture emits 100,000 individual chunks through the normal async
* carrier; the test measures event-loop and scheduled-interaction delay while
* the assembled React surface keeps a collapsed Think row live.
* A test-owned model adapter emits 100,000 individual chunks through the real
* Host, Gateway, and browser carriers; the test measures event-loop and
* scheduled-interaction delay while the assembled React surface keeps a
* collapsed Think row live.
*/
import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright'
import { expect, it, onTestFailed } from 'vitest'
import { LlmAdapter, type GenerateOptions, type StreamChunk } from '@deepseek-ai/dsh-llm'
import type {} from '@deepseek-ai/dsh-agent-default-model'
import { launchWebScaffold, watchConsole, type WebScaffold } from '../tests/scaffold.ts'
import { newEnglishPage, saveFailureShot } from '../tests/support.ts'
import {
connectFreshWorkspace, newEnglishPage, saveFailureShot, writeComposerDraft,
} from '../tests/support.ts'
const CHUNK_COUNT = 100_000
const CHUNKS_PER_INTERVAL = 128
const CHUNK_INTERVAL_MS = 16
const MAIN_THREAD_DELAY_BUDGET_MS = 250
const PROVIDER = 'reasoning-stress-test'
const MODEL = 'reasoning-storm'
const MARKER = `REASONING_STRESS_COMPLETE:${String(CHUNK_COUNT)}`
interface ReasoningChunkStormState {
sessionId: string
chunkCount: number
chunksPerInterval: number
intervalMs: number
@@ -25,6 +32,86 @@ interface ReasoningChunkStormState {
emitting: boolean
}
/** Externally paced model stream whose counters are observed outside the browser. */
class ReasoningStressAdapter extends LlmAdapter {
private finishStream!: () => void
private resolveStarted!: () => void
private resolveCompleted!: () => void
private releaseBatch: (() => void) | undefined
private resolveBatch: (() => void) | undefined
private batchInFlight = false
private finishRequested = false
private readonly finishGate = new Promise<void>((resolve) => { this.finishStream = resolve })
readonly started = new Promise<void>((resolve) => { this.resolveStarted = resolve })
readonly completed = new Promise<void>((resolve) => { this.resolveCompleted = resolve })
private emitted = 0
private emitting = false
finish(): void {
this.finishRequested = true
this.releaseBatch?.()
this.finishStream()
}
async emitNextBatch(): Promise<void> {
if (!this.emitting || this.releaseBatch === undefined) {
throw new Error('reasoning stress adapter has no batch awaiting release')
}
if (this.batchInFlight) throw new Error('reasoning stress adapter accepts one batch release at a time')
this.batchInFlight = true
const delivered = new Promise<void>((resolve) => { this.resolveBatch = resolve })
const release = this.releaseBatch
this.releaseBatch = undefined
release()
await delivered
}
state(): ReasoningChunkStormState {
return {
chunkCount: CHUNK_COUNT,
chunksPerInterval: CHUNKS_PER_INTERVAL,
intervalMs: CHUNK_INTERVAL_MS,
emitted: this.emitted,
marker: MARKER,
emitting: this.emitting,
}
}
override async *stream(options: GenerateOptions): AsyncIterable<StreamChunk> {
if (this.emitting || this.emitted !== 0) throw new Error('reasoning stress adapter accepts one model call')
this.emitting = true
const parts: string[] = []
yield { type: 'block-start', index: 0, blockType: 'reasoning' }
while (this.emitted < CHUNK_COUNT) {
await new Promise<void>((resolve) => {
this.releaseBatch = resolve
if (this.emitted === 0) this.resolveStarted()
if (this.finishRequested) resolve()
})
if (this.finishRequested) break
options.signal?.throwIfAborted()
const end = Math.min(this.emitted + CHUNKS_PER_INTERVAL, CHUNK_COUNT)
while (this.emitted < end) {
const text = this.emitted === CHUNK_COUNT - 1
? `\n${MARKER}`
: this.emitted % 64 === 63 ? '推理\n' : '推理'
parts.push(text)
yield { type: 'reasoning-delta', index: 0, text }
this.emitted += 1
}
this.batchInFlight = false
this.resolveBatch?.()
this.resolveBatch = undefined
}
this.emitting = false
if (this.emitted === CHUNK_COUNT) this.resolveCompleted()
await this.finishGate
options.signal?.throwIfAborted()
yield { type: 'block-end', index: 0, block: { type: 'reasoning', text: parts.join('') } }
yield { type: 'finish', reason: { kind: 'stop' } }
}
}
interface StressProbe {
intervalId: number
intervalMs: number
@@ -36,10 +123,6 @@ interface StressProbe {
}
interface StressWindow extends Window {
__fxTiming?: {
startReasoningChunkStorm(id: string, chunkCount: number, chunksPerInterval: number, intervalMs: number): string
reasoningChunkStormState(): ReasoningChunkStormState | null
}
__reasoningStressProbe?: StressProbe
}
@@ -47,23 +130,22 @@ it('keeps the browser responsive while rendering 100,000 reasoning chunks', asyn
let scaffold: WebScaffold | undefined
let browser: Browser | undefined
let page: Page | undefined
const adapter = new ReasoningStressAdapter()
try {
scaffold = await launchWebScaffold()
scaffold.ctx.effect(
() => scaffold!.ctx.llm.registerAdapter([PROVIDER], adapter),
'reasoning stress adapter',
)
await scaffold.ctx.agentDefaultModel.saveSelection({ provider: PROVIDER, model: MODEL })
browser = await chromium.launch({ headless: process.env.DSH_WEB_STRESS_HEADFUL !== '1' })
page = await newEnglishPage(browser)
const activePage = page
await activePage.addInitScript(() => {
localStorage.setItem('dsh.sessions.current', JSON.stringify({ sessionId: 'fx-alpha' }))
})
const tripwire = watchConsole(activePage)
onTestFailed(() => saveFailureShot(activePage, 'web-stress-reasoning-chunks'))
await activePage.goto(`${scaffold.baseUrl}?fixture`, { waitUntil: 'load' })
await activePage.goto(scaffold.authenticatedUrl, { waitUntil: 'load' })
await activePage.waitForSelector('[class*="frame"]', { timeout: 30_000 })
// Fixture settings deliberately reject writes, so its welcome notice
// cannot acknowledge. Hide only that test overlay; the assembled chat
// tree beneath it remains mounted and exercises the production renderer.
await activePage.addStyleTag({ content: '[class*="onboardingOverlay"] { display: none !important; }' })
await activePage.locator('[data-sample="bash"]').first().waitFor({ timeout: 30_000 })
await connectFreshWorkspace(activePage, scaffold.workspaceCwd)
await activePage.evaluate(() => {
const intervalMs = 50
@@ -92,45 +174,42 @@ it('keeps the browser responsive while rendering 100,000 reasoning chunks', asyn
;(window as StressWindow).__reasoningStressProbe = probe
})
const marker = await activePage.evaluate(({ chunkCount, chunksPerInterval, intervalMs }) => {
const hooks = (window as StressWindow).__fxTiming
if (hooks === undefined) throw new Error('reasoning stress fixture hooks unavailable')
return hooks.startReasoningChunkStorm('fx-alpha', chunkCount, chunksPerInterval, intervalMs)
}, {
chunkCount: CHUNK_COUNT,
chunksPerInterval: CHUNKS_PER_INTERVAL,
intervalMs: CHUNK_INTERVAL_MS,
})
const settled = scaffold.whenTurnSettled(540_000)
const input = activePage.locator('[data-composer-input]').first()
await writeComposerDraft(activePage, input, `Render ${String(CHUNK_COUNT)} reasoning chunks.`)
await input.press('Enter')
await adapter.started
await adapter.emitNextBatch()
const liveThink = activePage.locator('[data-variant="think"][data-state="running"]').last()
await liveThink.waitFor({ timeout: 60_000 })
await expect.poll(async () => await activePage.evaluate(() => {
const hooks = (window as StressWindow).__fxTiming
return hooks?.reasoningChunkStormState()?.emitted ?? 0
}), { timeout: 540_000, interval: 100 }).toBe(CHUNK_COUNT)
await expect.poll(() => liveThink.textContent(), { timeout: 60_000, interval: 100 }).toContain(marker)
await activePage.evaluate(intervalMs => new Promise<void>((resolve) => {
window.setTimeout(resolve, intervalMs)
}), CHUNK_INTERVAL_MS)
while (adapter.state().emitted < CHUNK_COUNT) {
await adapter.emitNextBatch()
await activePage.evaluate(intervalMs => new Promise<void>((resolve) => {
window.setTimeout(resolve, intervalMs)
}), CHUNK_INTERVAL_MS)
}
await adapter.completed
await expect.poll(() => liveThink.textContent(), { timeout: 60_000, interval: 100 }).toContain(MARKER)
const report = await activePage.evaluate(() => {
const browserReport = await activePage.evaluate(() => {
const win = window as StressWindow
const probe = win.__reasoningStressProbe
const state = win.__fxTiming?.reasoningChunkStormState()
if (probe === undefined || state === undefined || state === null) {
throw new Error('reasoning stress metrics unavailable')
}
if (probe === undefined) throw new Error('reasoning stress metrics unavailable')
window.clearInterval(probe.intervalId)
const interactionDelayMs = probe.interactionHandledAt === null
? null
: probe.interactionHandledAt - probe.interactionDueAt
return {
chunkCount: state.chunkCount,
chunksPerInterval: state.chunksPerInterval,
intervalMs: state.intervalMs,
emitted: state.emitted,
maxMainThreadDelayMs: Math.max(0, probe.maxDelayMs),
interactionDelayMs,
heartbeatSamples: probe.samples,
}
})
const report = { ...adapter.state(), ...browserReport }
process.stdout.write(`reasoning-chunk stress report: ${JSON.stringify(report)}\n`)
expect(report).toMatchObject({
@@ -146,7 +225,10 @@ it('keeps the browser responsive while rendering 100,000 reasoning chunks', asyn
expect(interactionDelayMs, JSON.stringify(report)).toBeLessThan(MAIN_THREAD_DELAY_BUDGET_MS)
expect(tripwire.pageErrors).toEqual([])
expect(tripwire.warnings).toEqual([])
adapter.finish()
await settled
} finally {
adapter.finish()
await browser?.close()
await scaffold?.close()
}
+15 -16
View File
@@ -1,18 +1,19 @@
// Keyless assembled-browser coverage for the goal bar over the shipped Web
// bundles and the fixture Connection RPC. The command creates a real projected
// goal in the fixture session; the golden pins the active strip, while the
// clear gesture proves the acknowledged tombstone leaves neither stale chrome
// nor a duplicate-mutation error.
// Keyless browser coverage for the goal bar over the shipped Web composition.
// The command creates a real projected goal in a real Host session. The
// goldens pin the active and disarmed strips, while the clear gesture proves
// the acknowledged tombstone leaves neither stale chrome nor a
// duplicate-mutation error.
import { fileURLToPath } from 'node:url'
import { join } from 'node:path'
import type { Browser, Page } from 'playwright'
import { chromium } from 'playwright'
import { afterAll, beforeAll, describe, expect, it, onTestFailed } from 'vitest'
import type {} from '@deepseek-ai/dsh-goal'
import {
assertFixtureInventory, captureStableAria, compareOrRefreshGolden,
launchWebScaffold, watchConsole, webSnapshotMode, type WebScaffold,
} from './scaffold.ts'
import { newEnglishPage, saveFailureShot } from './support.ts'
import { connectFreshWorkspace, newEnglishPage, saveFailureShot } from './support.ts'
const SNAPSHOT_DIR = fileURLToPath(new URL('./expected/goal-bar', import.meta.url))
const ACTIVE_EXPECTED = join(SNAPSHOT_DIR, 'active.expected.md')
@@ -27,14 +28,13 @@ describe('web e2e: goal bar clear convergence', () => {
let tripwire: ReturnType<typeof watchConsole>
beforeAll(async () => {
scaffold = await launchWebScaffold({ extraOverlayPath: OVERLAY, welcomeNoticePending: true })
scaffold = await launchWebScaffold({ extraOverlayPath: OVERLAY })
browser = await chromium.launch()
page = await newEnglishPage(browser)
tripwire = watchConsole(page)
const login = await page.context().request.get(scaffold.authenticatedUrl, { maxRedirects: 0 })
expect(login.status()).toBe(303)
await page.goto(`${scaffold.baseUrl}?fixture`, { waitUntil: 'load' })
await page.goto(scaffold.authenticatedUrl, { waitUntil: 'load' })
await page.waitForSelector('[class*="frame"]', { timeout: 30_000 })
await connectFreshWorkspace(page, scaffold.workspaceCwd)
}, 120_000)
afterAll(async () => {
@@ -44,8 +44,6 @@ describe('web e2e: goal bar clear convergence', () => {
it('renders one active goal and clears it without exposing a stale error', async () => {
onTestFailed(() => saveFailureShot(page, 'web-e2e-goal-bar-clear'))
// Startup reuses the fixture workspace's blank session, keeping this
// command independent of alpha's running replay and pending question.
const input = page.locator('[data-composer-input][data-placeholder="Describe what you want to build, / commands, @ files or sessions"]')
await input.waitFor({ timeout: 10_000 })
await input.fill('/goal guard rapid clear clicks')
@@ -53,15 +51,16 @@ describe('web e2e: goal bar clear convergence', () => {
const bar = page.locator('[data-goal-bar]')
await bar.waitFor({ timeout: 10_000 })
await expect.poll(() => bar.getByRole('button', { name: 'Pause goal' }).count(), {
const pause = bar.getByRole('button', { name: 'Pause goal' })
await expect.poll(() => pause.count(), {
timeout: 10_000,
}).toBe(1)
const snapshot = await captureStableAria(page, '[data-goal-bar]', scaffold.workspaceCwd)
await compareOrRefreshGolden(ACTIVE_EXPECTED, snapshot, MODE)
await page.evaluate(() => {
(globalThis as unknown as { __fxTiming?: { disarmOnlyGoal(): void } }).__fxTiming?.disarmOnlyGoal()
})
const agents = scaffold.ctx.agents.list()
expect(agents).toHaveLength(1)
scaffold.ctx.goals.disarm(agents[0]!)
await expect.poll(() => bar.getByRole('button', { name: 'Resume goal' }).count(), {
timeout: 10_000,
}).toBe(1)
+3 -4
View File
@@ -1,5 +1,4 @@
# The client-side FixtureApiClient intentionally rejects settings traffic, so
# this goal-only scenario omits the settings shell and the onboarding steps it
# would mount. Onboarding owns separate assembled-browser coverage.
- id: ui-settings-general
# This scenario owns Goal bar actions, not autonomous Goal rounds. Keeping the
# driver out prevents an active Goal from issuing an unrelated model request.
- id: goal-round-driver
disabled: true
+2 -2
View File
@@ -1,10 +1,10 @@
import tsconfigPaths from 'vite-tsconfig-paths'
import { defineConfig } from 'vitest/config'
import { vitestExecArgv } from './vitest.shared.ts'
import { standardDecoratorPlugin, vitestExecArgv } from './vitest.shared.ts'
/** Opt-in browser performance lane; no default Vitest config includes *.stress.ts. */
export default defineConfig({
plugins: [tsconfigPaths({ projects: ['./tsconfig.base.json'] })],
plugins: [tsconfigPaths({ projects: ['./tsconfig.base.json'] }), standardDecoratorPlugin()],
test: {
execArgv: vitestExecArgv,
include: ['apps/web/stress-tests/**/*.stress.ts'],