mirror of
https://github.com/morluto/rea.git
synced 2026-11-22 06:25:44 +00:00
test(web): consolidate scenario Evidence promotion checks (#13160)
This commit is contained in:
1 parent
0fa220c735
commit
b0043f1fe4
1 file changed
+145
-181
@@ -10,9 +10,10 @@ import {
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { parseEvidence } from "../domain/evidence.js";
|
||||
import { browserScenarioInputSchema } from "../domain/browserScenario.js";
|
||||
import type {
|
||||
BrowserScenarioCapture,
|
||||
BrowserScenarioPartialObservation,
|
||||
import {
|
||||
browserScenarioCaptureSchema,
|
||||
type BrowserScenarioCapture,
|
||||
type BrowserScenarioPartialObservation,
|
||||
} from "../domain/browserScenarioCapture.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import { scriptScenarioFixture } from "../../tests/fixtures/webScriptCapture.js";
|
||||
@@ -36,89 +37,71 @@ const scenarioIdentity = {
|
||||
name: "Scenario fixture",
|
||||
version: "1",
|
||||
};
|
||||
const runScenario = (
|
||||
failure: AnalysisError,
|
||||
id = { id: "fixture", name: "fixture", version: "1" },
|
||||
) =>
|
||||
const runScenario = (failure: AnalysisError) =>
|
||||
captureBrowserScenario(
|
||||
{
|
||||
identity: () => id,
|
||||
identity: () => scenarioIdentity,
|
||||
captureScenario: async () => err(failure),
|
||||
},
|
||||
scenario,
|
||||
);
|
||||
const expectRetainedCapture = (
|
||||
result: Awaited<ReturnType<typeof runScenario>>,
|
||||
failure: AnalysisError,
|
||||
) => {
|
||||
if (result.ok) throw new Error("Expected the retained capture");
|
||||
expect(result.error).toBe(failure);
|
||||
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
|
||||
operation: "capture_browser_scenario",
|
||||
normalized_result: capture,
|
||||
});
|
||||
};
|
||||
|
||||
it("converts a retained raw scenario capture into partial Evidence", async () => {
|
||||
const failure = new AnalysisCancelledError("capture_browser_scenario", {
|
||||
partialObservation: {
|
||||
kind: "browser-scenario-observation",
|
||||
capture,
|
||||
},
|
||||
});
|
||||
expectRetainedCapture(await runScenario(failure, scenarioIdentity), failure);
|
||||
});
|
||||
|
||||
it("converts a raw scenario capture retained after cleanup failure", async () => {
|
||||
const failure = new BrowserObservationError(
|
||||
"capture_browser_scenario",
|
||||
"cleanup_failed",
|
||||
{
|
||||
cleanup: {
|
||||
reason: "browser close failed",
|
||||
resources: ["browser process"],
|
||||
},
|
||||
partialObservation: {
|
||||
kind: "browser-scenario-observation",
|
||||
capture,
|
||||
},
|
||||
},
|
||||
);
|
||||
const result = await runScenario(failure, scenarioIdentity);
|
||||
expectRetainedCapture(result, failure);
|
||||
if (!result.ok) expect(result.error.cleanupIncomplete).toBe(true);
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"retains completed scenario Evidence after cleanup resolves (wrapper facts: %s)",
|
||||
async (wrapperFacts) => {
|
||||
it.each([
|
||||
{
|
||||
kind: "direct cancellation",
|
||||
cleanup: false,
|
||||
wrapped: false,
|
||||
wrapperFacts: false,
|
||||
},
|
||||
{
|
||||
kind: "direct cleanup failure",
|
||||
cleanup: true,
|
||||
wrapped: false,
|
||||
wrapperFacts: true,
|
||||
},
|
||||
{
|
||||
kind: "cleanup primary",
|
||||
cleanup: true,
|
||||
wrapped: true,
|
||||
wrapperFacts: false,
|
||||
},
|
||||
{ kind: "shared partial", cleanup: true, wrapped: true, wrapperFacts: true },
|
||||
])(
|
||||
"promotes completed partial captures to Evidence ($kind)",
|
||||
async ({ cleanup, wrapped, wrapperFacts }) => {
|
||||
const partial = { kind: "browser-scenario-observation" as const, capture };
|
||||
const primary = new AnalysisCancelledError("capture_browser_scenario", {
|
||||
partialObservation: partial,
|
||||
});
|
||||
const failure = new BrowserObservationError(
|
||||
"capture_browser_scenario",
|
||||
"cleanup_failed",
|
||||
{
|
||||
cleanupPrimary: primary,
|
||||
cleanup: {
|
||||
reason: "browser close failed",
|
||||
resources: ["browser process"],
|
||||
},
|
||||
...(wrapperFacts ? { partialObservation: partial } : {}),
|
||||
},
|
||||
);
|
||||
const result = await runScenario(failure, scenarioIdentity);
|
||||
if (result.ok) throw new Error("Expected cleanup failure");
|
||||
const failure = cleanup
|
||||
? new BrowserObservationError(
|
||||
"capture_browser_scenario",
|
||||
"cleanup_failed",
|
||||
{
|
||||
...(wrapped ? { cleanupPrimary: primary } : {}),
|
||||
cleanup: {
|
||||
reason: "browser close failed",
|
||||
resources: ["browser process"],
|
||||
},
|
||||
...(wrapperFacts ? { partialObservation: partial } : {}),
|
||||
},
|
||||
)
|
||||
: primary;
|
||||
|
||||
const result = await runScenario(failure);
|
||||
if (result.ok) throw new Error("Expected retained capture");
|
||||
expect(result.error).toBe(failure);
|
||||
releaseOwnedCleanup(failure);
|
||||
expect(resolvedAnalysisError(failure)).toBe(primary);
|
||||
expect(parseEvidence(primary.partialObservation)).toMatchObject({
|
||||
expect(failure.cleanupIncomplete).toBe(cleanup);
|
||||
if (wrapped) releaseOwnedCleanup(failure);
|
||||
const owner = resolvedAnalysisError(failure);
|
||||
expect(owner).toBe(wrapped ? primary : failure);
|
||||
if (wrapped && wrapperFacts)
|
||||
expect(failure.partialObservation).toBe(primary.partialObservation);
|
||||
expect(parseEvidence(owner.partialObservation)).toMatchObject({
|
||||
operation: "capture_browser_scenario",
|
||||
provider: scenarioIdentity,
|
||||
normalized_result: capture,
|
||||
});
|
||||
if (wrapperFacts)
|
||||
expect(failure.partialObservation).toBe(primary.partialObservation);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -149,62 +132,76 @@ it.each([
|
||||
},
|
||||
);
|
||||
|
||||
it("retains completed scenario Evidence when cancellation arrives during capture", async () => {
|
||||
const caller = new AbortController();
|
||||
const reason = { requested: "stop after capture" };
|
||||
const provider: BrowserScenarioCapturePort = {
|
||||
identity: () => scenarioIdentity,
|
||||
captureScenario: async (received, options) => {
|
||||
expect(received).toBe(scenario);
|
||||
expect(options?.signal).toBe(caller.signal);
|
||||
caller.abort(reason);
|
||||
return ok(capture);
|
||||
},
|
||||
};
|
||||
const result = await captureBrowserScenario(provider, scenario, {
|
||||
signal: caller.signal,
|
||||
});
|
||||
if (result.ok) throw new Error("Expected late cancellation");
|
||||
expect(result.error).toBeInstanceOf(AnalysisCancelledError);
|
||||
expect(result.error.cause).toBe(reason);
|
||||
expect(parseEvidence(result.error.partialObservation)).toMatchObject({
|
||||
operation: "capture_browser_scenario",
|
||||
provider: scenarioIdentity,
|
||||
confidence: "observed",
|
||||
authority: "controlled-replay",
|
||||
parameters: {
|
||||
browser: scenario.browser,
|
||||
start_url: scenario.start_url,
|
||||
actions: scenario.actions,
|
||||
},
|
||||
normalized_result: capture,
|
||||
});
|
||||
});
|
||||
it.each([false, true])(
|
||||
"delivers completed Evidence without redundant source (cancelled during capture: %s)",
|
||||
async (cancelled) => {
|
||||
const caller = new AbortController();
|
||||
const reason = { requested: "stop after capture" };
|
||||
const provider: BrowserScenarioCapturePort = {
|
||||
identity: () => scenarioIdentity,
|
||||
captureScenario: async (received, options) => {
|
||||
expect(received).toBe(scenario);
|
||||
expect(options?.signal).toBe(caller.signal);
|
||||
if (cancelled) caller.abort(reason);
|
||||
return ok(capture);
|
||||
},
|
||||
};
|
||||
const result = await captureBrowserScenario(provider, scenario, {
|
||||
signal: caller.signal,
|
||||
});
|
||||
expect(result.ok).toBe(!cancelled);
|
||||
if (!result.ok) {
|
||||
expect(result.error).toBeInstanceOf(AnalysisCancelledError);
|
||||
expect(result.error.cause).toBe(reason);
|
||||
}
|
||||
expect(
|
||||
result.ok ? result.value : parseEvidence(result.error.partialObservation),
|
||||
).toMatchObject({
|
||||
operation: "capture_browser_scenario",
|
||||
provider: scenarioIdentity,
|
||||
confidence: "observed",
|
||||
authority: "controlled-replay",
|
||||
parameters: {
|
||||
browser: scenario.browser,
|
||||
start_url: scenario.start_url,
|
||||
actions: scenario.actions,
|
||||
},
|
||||
normalized_result: capture,
|
||||
raw_result: null,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps startup cancellation and its incomplete capture", async () => {
|
||||
const reason = { requested: "stop during initial navigation" };
|
||||
const completed = scriptScenarioFixture();
|
||||
const {
|
||||
completeness: _completeness,
|
||||
steps: _steps,
|
||||
browser,
|
||||
...retained
|
||||
} = completed;
|
||||
const partial: BrowserScenarioPartialObservation = {
|
||||
kind: "browser-scenario-observation",
|
||||
capture: {
|
||||
...retained,
|
||||
browser: { ...browser, cleanup: "incomplete" },
|
||||
steps: [],
|
||||
},
|
||||
};
|
||||
const failure = new AnalysisCancelledError("capture_browser_scenario", {
|
||||
cause: reason,
|
||||
partialObservation: partial,
|
||||
});
|
||||
const result = await runScenario(failure);
|
||||
expectKeptPartial(expectOwnerError(result, failure, reason), partial);
|
||||
});
|
||||
it.each(["startup", "failed step"] as const)(
|
||||
"keeps the %s cancellation observation partial",
|
||||
async (phase) => {
|
||||
const { completeness: _completeness, ...retained } = capture;
|
||||
const partial: BrowserScenarioPartialObservation = {
|
||||
kind: "browser-scenario-observation",
|
||||
capture:
|
||||
phase === "startup"
|
||||
? {
|
||||
...retained,
|
||||
browser: { ...capture.browser, cleanup: "incomplete" },
|
||||
steps: [],
|
||||
}
|
||||
: browserScenarioCaptureSchema.parse({
|
||||
...capture,
|
||||
steps: [
|
||||
capture.steps[0],
|
||||
{ ...capture.steps[1], status: "failed", error: "step failed" },
|
||||
],
|
||||
}),
|
||||
};
|
||||
const reason = { phase };
|
||||
const failure = new AnalysisCancelledError("capture_browser_scenario", {
|
||||
cause: reason,
|
||||
partialObservation: partial,
|
||||
});
|
||||
const result = await runScenario(failure);
|
||||
expectKeptPartial(expectOwnerError(result, failure, reason), partial);
|
||||
},
|
||||
);
|
||||
|
||||
it("interprets only the retained observation's own JSON fields", async () => {
|
||||
let reads = 0;
|
||||
@@ -287,15 +284,28 @@ it("projects scenario own JSON before inspecting a hidden known property", async
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"retains scenario nested own metadata (cleanup primary: %s)",
|
||||
"retains sanitized source and its own metadata (cleanup primary: %s)",
|
||||
async (cleanup) => {
|
||||
const selected = browserScenarioInputSchema.parse({
|
||||
...scenario,
|
||||
secrets: [
|
||||
{ secret_id: "selected", environment_variable: "DECLARED_VALUE" },
|
||||
],
|
||||
});
|
||||
const secret = "explicit-source-secret";
|
||||
const privacy = resolveScenarioSecrets(selected, {
|
||||
DECLARED_VALUE: secret,
|
||||
});
|
||||
const sanitized = privacy.sanitizeUrl(
|
||||
`https://transport-user:transport-password@fixture.test/?selected=${secret}&token=public-local-value`,
|
||||
);
|
||||
const source = {
|
||||
...capture,
|
||||
steps: capture.steps.map((step) => ({
|
||||
...step,
|
||||
before_url: {
|
||||
...step.before_url,
|
||||
provider_metadata: { source_id: "caller-source" },
|
||||
...sanitized,
|
||||
provider_metadata: { source_id: "harmless-source" },
|
||||
},
|
||||
})),
|
||||
};
|
||||
@@ -321,7 +331,7 @@ it.each([false, true])(
|
||||
identity: () => scenarioIdentity,
|
||||
captureScenario: async () => (cleanup ? err(failure) : ok(source)),
|
||||
},
|
||||
scenario,
|
||||
selected,
|
||||
);
|
||||
const evidence = result.ok
|
||||
? result.value
|
||||
@@ -335,56 +345,10 @@ it.each([false, true])(
|
||||
expect(evidence.normalized_result).not.toHaveProperty(
|
||||
"steps.0.before_url.provider_metadata",
|
||||
);
|
||||
const recorded = JSON.stringify(evidence);
|
||||
for (const value of [secret, "transport-user", "transport-password"])
|
||||
expect(recorded).not.toContain(value);
|
||||
expect(recorded).toContain("public-local-value");
|
||||
expect(sanitized.redacted).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("omits redundant raw scenario JSON for an unchanged capture", async () => {
|
||||
const result = await captureBrowserScenario(
|
||||
{
|
||||
identity: () => scenarioIdentity,
|
||||
captureScenario: async () => ok(capture),
|
||||
},
|
||||
scenario,
|
||||
);
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.raw_result).toBeNull();
|
||||
});
|
||||
|
||||
it("retains only post-redaction scenario source while preserving harmless metadata", async () => {
|
||||
const selected = browserScenarioInputSchema.parse({
|
||||
...scenario,
|
||||
secrets: [
|
||||
{ secret_id: "selected", environment_variable: "DECLARED_VALUE" },
|
||||
],
|
||||
});
|
||||
const secret = "explicit-source-secret";
|
||||
const privacy = resolveScenarioSecrets(selected, { DECLARED_VALUE: secret });
|
||||
const sanitized = privacy.sanitizeUrl(
|
||||
`https://transport-user:transport-password@fixture.test/?selected=${secret}&token=public-local-value`,
|
||||
);
|
||||
const source = {
|
||||
...capture,
|
||||
steps: capture.steps.map((step) => ({
|
||||
...step,
|
||||
before_url: {
|
||||
...sanitized,
|
||||
provider_metadata: { source_id: "harmless-source" },
|
||||
},
|
||||
})),
|
||||
};
|
||||
const result = await captureBrowserScenario(
|
||||
{
|
||||
identity: () => scenarioIdentity,
|
||||
captureScenario: async () => ok(source),
|
||||
},
|
||||
selected,
|
||||
);
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.raw_result).toEqual(source);
|
||||
const recorded = JSON.stringify(result.value);
|
||||
for (const privateValue of [secret, "transport-user", "transport-password"])
|
||||
expect(recorded).not.toContain(privateValue);
|
||||
expect(recorded).toContain("harmless-source");
|
||||
expect(recorded).toContain("public-local-value");
|
||||
expect(sanitized.redacted).toBe(true);
|
||||
});
|
||||
Reference in new issue
Block a user