test(web): consolidate scenario Evidence promotion checks (#13160)

This commit is contained in:
morluto authored and GitHub committed 2026-11-21 17:22:41 +00:00
1 parent 0fa220c735
commit b0043f1fe4
1 file changed
+145 -181
@@ -10,9 +10,10 @@ import {
import { BrowserObservationError } from "../domain/browserObservationError.js";
import { parseEvidence } from "../domain/evidence.js";
import { browserScenarioInputSchema } from "../domain/browserScenario.js";
import type {
BrowserScenarioCapture,
BrowserScenarioPartialObservation,
import {
browserScenarioCaptureSchema,
type BrowserScenarioCapture,
type BrowserScenarioPartialObservation,
} from "../domain/browserScenarioCapture.js";
import { err, ok } from "../domain/result.js";
import { scriptScenarioFixture } from "../../tests/fixtures/webScriptCapture.js";
@@ -36,89 +37,71 @@ const scenarioIdentity = {
name: "Scenario fixture",
version: "1",
};
const runScenario = (
failure: AnalysisError,
id = { id: "fixture", name: "fixture", version: "1" },
) =>
const runScenario = (failure: AnalysisError) =>
captureBrowserScenario(
{
identity: () => id,
identity: () => scenarioIdentity,
captureScenario: async () => err(failure),
},
scenario,
);
const expectRetainedCapture = (
result: Awaited<ReturnType<typeof runScenario>>,
failure: AnalysisError,
) => {
if (result.ok) throw new Error("Expected the retained capture");
expect(result.error).toBe(failure);
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "capture_browser_scenario",
normalized_result: capture,
});
};
it("converts a retained raw scenario capture into partial Evidence", async () => {
const failure = new AnalysisCancelledError("capture_browser_scenario", {
partialObservation: {
kind: "browser-scenario-observation",
capture,
},
});
expectRetainedCapture(await runScenario(failure, scenarioIdentity), failure);
});
it("converts a raw scenario capture retained after cleanup failure", async () => {
const failure = new BrowserObservationError(
"capture_browser_scenario",
"cleanup_failed",
{
cleanup: {
reason: "browser close failed",
resources: ["browser process"],
},
partialObservation: {
kind: "browser-scenario-observation",
capture,
},
},
);
const result = await runScenario(failure, scenarioIdentity);
expectRetainedCapture(result, failure);
if (!result.ok) expect(result.error.cleanupIncomplete).toBe(true);
});
it.each([false, true])(
"retains completed scenario Evidence after cleanup resolves (wrapper facts: %s)",
async (wrapperFacts) => {
it.each([
{
kind: "direct cancellation",
cleanup: false,
wrapped: false,
wrapperFacts: false,
},
{
kind: "direct cleanup failure",
cleanup: true,
wrapped: false,
wrapperFacts: true,
},
{
kind: "cleanup primary",
cleanup: true,
wrapped: true,
wrapperFacts: false,
},
{ kind: "shared partial", cleanup: true, wrapped: true, wrapperFacts: true },
])(
"promotes completed partial captures to Evidence ($kind)",
async ({ cleanup, wrapped, wrapperFacts }) => {
const partial = { kind: "browser-scenario-observation" as const, capture };
const primary = new AnalysisCancelledError("capture_browser_scenario", {
partialObservation: partial,
});
const failure = new BrowserObservationError(
"capture_browser_scenario",
"cleanup_failed",
{
cleanupPrimary: primary,
cleanup: {
reason: "browser close failed",
resources: ["browser process"],
},
...(wrapperFacts ? { partialObservation: partial } : {}),
},
);
const result = await runScenario(failure, scenarioIdentity);
if (result.ok) throw new Error("Expected cleanup failure");
const failure = cleanup
? new BrowserObservationError(
"capture_browser_scenario",
"cleanup_failed",
{
...(wrapped ? { cleanupPrimary: primary } : {}),
cleanup: {
reason: "browser close failed",
resources: ["browser process"],
},
...(wrapperFacts ? { partialObservation: partial } : {}),
},
)
: primary;
const result = await runScenario(failure);
if (result.ok) throw new Error("Expected retained capture");
expect(result.error).toBe(failure);
releaseOwnedCleanup(failure);
expect(resolvedAnalysisError(failure)).toBe(primary);
expect(parseEvidence(primary.partialObservation)).toMatchObject({
expect(failure.cleanupIncomplete).toBe(cleanup);
if (wrapped) releaseOwnedCleanup(failure);
const owner = resolvedAnalysisError(failure);
expect(owner).toBe(wrapped ? primary : failure);
if (wrapped && wrapperFacts)
expect(failure.partialObservation).toBe(primary.partialObservation);
expect(parseEvidence(owner.partialObservation)).toMatchObject({
operation: "capture_browser_scenario",
provider: scenarioIdentity,
normalized_result: capture,
});
if (wrapperFacts)
expect(failure.partialObservation).toBe(primary.partialObservation);
},
);
@@ -149,62 +132,76 @@ it.each([
},
);
it("retains completed scenario Evidence when cancellation arrives during capture", async () => {
const caller = new AbortController();
const reason = { requested: "stop after capture" };
const provider: BrowserScenarioCapturePort = {
identity: () => scenarioIdentity,
captureScenario: async (received, options) => {
expect(received).toBe(scenario);
expect(options?.signal).toBe(caller.signal);
caller.abort(reason);
return ok(capture);
},
};
const result = await captureBrowserScenario(provider, scenario, {
signal: caller.signal,
});
if (result.ok) throw new Error("Expected late cancellation");
expect(result.error).toBeInstanceOf(AnalysisCancelledError);
expect(result.error.cause).toBe(reason);
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
operation: "capture_browser_scenario",
provider: scenarioIdentity,
confidence: "observed",
authority: "controlled-replay",
parameters: {
browser: scenario.browser,
start_url: scenario.start_url,
actions: scenario.actions,
},
normalized_result: capture,
});
});
it.each([false, true])(
"delivers completed Evidence without redundant source (cancelled during capture: %s)",
async (cancelled) => {
const caller = new AbortController();
const reason = { requested: "stop after capture" };
const provider: BrowserScenarioCapturePort = {
identity: () => scenarioIdentity,
captureScenario: async (received, options) => {
expect(received).toBe(scenario);
expect(options?.signal).toBe(caller.signal);
if (cancelled) caller.abort(reason);
return ok(capture);
},
};
const result = await captureBrowserScenario(provider, scenario, {
signal: caller.signal,
});
expect(result.ok).toBe(!cancelled);
if (!result.ok) {
expect(result.error).toBeInstanceOf(AnalysisCancelledError);
expect(result.error.cause).toBe(reason);
}
expect(
result.ok ? result.value : parseEvidence(result.error.partialObservation),
).toMatchObject({
operation: "capture_browser_scenario",
provider: scenarioIdentity,
confidence: "observed",
authority: "controlled-replay",
parameters: {
browser: scenario.browser,
start_url: scenario.start_url,
actions: scenario.actions,
},
normalized_result: capture,
raw_result: null,
});
},
);
it("keeps startup cancellation and its incomplete capture", async () => {
const reason = { requested: "stop during initial navigation" };
const completed = scriptScenarioFixture();
const {
completeness: _completeness,
steps: _steps,
browser,
...retained
} = completed;
const partial: BrowserScenarioPartialObservation = {
kind: "browser-scenario-observation",
capture: {
...retained,
browser: { ...browser, cleanup: "incomplete" },
steps: [],
},
};
const failure = new AnalysisCancelledError("capture_browser_scenario", {
cause: reason,
partialObservation: partial,
});
const result = await runScenario(failure);
expectKeptPartial(expectOwnerError(result, failure, reason), partial);
});
it.each(["startup", "failed step"] as const)(
"keeps the %s cancellation observation partial",
async (phase) => {
const { completeness: _completeness, ...retained } = capture;
const partial: BrowserScenarioPartialObservation = {
kind: "browser-scenario-observation",
capture:
phase === "startup"
? {
...retained,
browser: { ...capture.browser, cleanup: "incomplete" },
steps: [],
}
: browserScenarioCaptureSchema.parse({
...capture,
steps: [
capture.steps[0],
{ ...capture.steps[1], status: "failed", error: "step failed" },
],
}),
};
const reason = { phase };
const failure = new AnalysisCancelledError("capture_browser_scenario", {
cause: reason,
partialObservation: partial,
});
const result = await runScenario(failure);
expectKeptPartial(expectOwnerError(result, failure, reason), partial);
},
);
it("interprets only the retained observation's own JSON fields", async () => {
let reads = 0;
@@ -287,15 +284,28 @@ it("projects scenario own JSON before inspecting a hidden known property", async
});
it.each([false, true])(
"retains scenario nested own metadata (cleanup primary: %s)",
"retains sanitized source and its own metadata (cleanup primary: %s)",
async (cleanup) => {
const selected = browserScenarioInputSchema.parse({
...scenario,
secrets: [
{ secret_id: "selected", environment_variable: "DECLARED_VALUE" },
],
});
const secret = "explicit-source-secret";
const privacy = resolveScenarioSecrets(selected, {
DECLARED_VALUE: secret,
});
const sanitized = privacy.sanitizeUrl(
`https://transport-user:transport-password@fixture.test/?selected=${secret}&token=public-local-value`,
);
const source = {
...capture,
steps: capture.steps.map((step) => ({
...step,
before_url: {
...step.before_url,
provider_metadata: { source_id: "caller-source" },
...sanitized,
provider_metadata: { source_id: "harmless-source" },
},
})),
};
@@ -321,7 +331,7 @@ it.each([false, true])(
identity: () => scenarioIdentity,
captureScenario: async () => (cleanup ? err(failure) : ok(source)),
},
scenario,
selected,
);
const evidence = result.ok
? result.value
@@ -335,56 +345,10 @@ it.each([false, true])(
expect(evidence.normalized_result).not.toHaveProperty(
"steps.0.before_url.provider_metadata",
);
const recorded = JSON.stringify(evidence);
for (const value of [secret, "transport-user", "transport-password"])
expect(recorded).not.toContain(value);
expect(recorded).toContain("public-local-value");
expect(sanitized.redacted).toBe(true);
},
);
it("omits redundant raw scenario JSON for an unchanged capture", async () => {
const result = await captureBrowserScenario(
{
identity: () => scenarioIdentity,
captureScenario: async () => ok(capture),
},
scenario,
);
if (!result.ok) throw result.error;
expect(result.value.raw_result).toBeNull();
});
it("retains only post-redaction scenario source while preserving harmless metadata", async () => {
const selected = browserScenarioInputSchema.parse({
...scenario,
secrets: [
{ secret_id: "selected", environment_variable: "DECLARED_VALUE" },
],
});
const secret = "explicit-source-secret";
const privacy = resolveScenarioSecrets(selected, { DECLARED_VALUE: secret });
const sanitized = privacy.sanitizeUrl(
`https://transport-user:transport-password@fixture.test/?selected=${secret}&token=public-local-value`,
);
const source = {
...capture,
steps: capture.steps.map((step) => ({
...step,
before_url: {
...sanitized,
provider_metadata: { source_id: "harmless-source" },
},
})),
};
const result = await captureBrowserScenario(
{
identity: () => scenarioIdentity,
captureScenario: async () => ok(source),
},
selected,
);
if (!result.ok) throw result.error;
expect(result.value.raw_result).toEqual(source);
const recorded = JSON.stringify(result.value);
for (const privateValue of [secret, "transport-user", "transport-password"])
expect(recorded).not.toContain(privateValue);
expect(recorded).toContain("harmless-source");
expect(recorded).toContain("public-local-value");
expect(sanitized.redacted).toBe(true);
});