import assert from "node:assert/strict"; import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { isDeepStrictEqual, parseArgs } from "node:util"; import { sazRequest, verifySazRecords, verifySazOracleIdentity, sazEvidenceParameters, } from "./saz-entry.mjs"; const { values } = parseArgs({ options: { trial: { type: "string" }, oracle: { type: "string" }, "runtime-root": { type: "string" }, artifact: { type: "string" }, selection: { type: "string" }, }, }); if ( !values.trial || !values.oracle || !values["runtime-root"] || !["baseline", "focused"].includes(values.selection) ) throw new Error( "Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused", ); const load = async (file) => JSON.parse(await readFile(resolve(values.trial, file), "utf8")); const settings = await load("trial-settings.json"), summary = await load("summary.json"); const oracle = JSON.parse(await readFile(values.oracle, "utf8")); verifySazOracleIdentity(values.artifact, oracle); const runtime = (file) => pathToFileURL(resolve(values["runtime-root"], file)).href; const { parseEvidence } = await import(runtime("dist/domain/evidence.js")); const { inspectWebNetworkCaptureInputSchema, webNetworkCaptureSchema } = await import(runtime("dist/domain/webNetworkCapture.js")); const { inspectAnalysisViewInputSchema, analysisViewResultSchema } = await import(runtime("dist/domain/analysisView/analysisView.js")); const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8")) .split("\n") .filter(Boolean) .map(JSON.parse) .filter( (event) => event.type === "item.completed" && event.item?.type === "mcp_tool_call", ) .map((event) => event.item); const planned = inspectWebNetworkCaptureInputSchema.parse( sazRequest( values.artifact, settings.selected_artifact_path, values.selection, ), ); const reasons = [], workflowReasons = [], authenticated = new Map(), parents = new Set(), conflicts = new Set(); let transportRecovery = false; for (const call of calls) { try { assert.equal(call.server, "rea"); assert.equal(call.status, "completed"); assert.ok(!call.error, "RPC did not provide observations"); const text = call.result.content.filter((item) => item.type === "text"); assert.equal(text.length, 1, "Ambiguous Evidence text carrier"); let evidence = JSON.parse(text[0].text); if (call.result.isError) { const limits = evidence.error?.details?.reported_limits; assert.equal(call.tool, "inspect_web_network_capture"); assert.equal(evidence.error?.code, "resource_constraint"); assert.equal( evidence.error.details.operation, "inspect_web_network_capture", ); assert.equal(limits?.boundary, "mcp-response"); const input = inspectWebNetworkCaptureInputSchema.parse(call.arguments); assert.deepEqual(input, planned); const source = inspectAnalysisViewInputSchema.parse({ source: limits.evidence_reference, view: { kind: "summary" }, }).source; assert.equal(source.kind, "retained-evidence"); parents.add(source.evidence_id); transportRecovery = true; continue; } if (call.result.structuredContent !== undefined) assert.deepEqual(evidence, call.result.structuredContent); evidence = parseEvidence(evidence); assert.equal(evidence.subject.local_path, settings.selected_artifact_path); assert.equal(evidence.subject.digest.sha256, oracle.sha256); let records; if (call.tool === "inspect_web_network_capture") { const { detail, ...arguments_ } = call.arguments; const input = inspectWebNetworkCaptureInputSchema.parse(arguments_); assert.equal(input.capture_path, settings.selected_artifact_path); assert.equal(input.format, "saz"); if (!isDeepStrictEqual(input, planned)) workflowReasons.push( "Inspection selection differs from planned arguments", ); if (detail === "summary") { workflowReasons.push( "Summary delivery differs from the fixed complete-delivery plan", ); assert.equal(evidence.operation, "inspect_analysis_view"); const report = analysisViewResultSchema.parse( evidence.normalized_result, ); assert.equal(report.kind, "summary"); assert.equal(report.parent_operation, "inspect_web_network_capture"); assert.equal(report.summary.format, "saz"); assert.equal(report.summary.total_records, oracle.total_records); assert.deepEqual(evidence.parameters, { parent_evidence_id: report.parent_evidence_id, view: { kind: "summary" }, }); parents.add(report.parent_evidence_id); authenticated.set(evidence.evidence_id, []); continue; } assert.ok(detail === undefined || detail === "complete"); assert.equal(evidence.operation, "inspect_web_network_capture"); assert.deepEqual( evidence.parameters, sazEvidenceParameters(input, oracle.total_records), ); const report = webNetworkCaptureSchema.parse(evidence.normalized_result); assert.equal(report.total_records, oracle.total_records); assert.equal(report.format, "saz"); records = report.records; parents.add(evidence.evidence_id); } else { assert.equal(call.tool, "inspect_analysis_view"); const input = inspectAnalysisViewInputSchema.parse(call.arguments); assert.equal(input.source.kind, "retained-evidence"); assert.ok( parents.has(input.source.evidence_id), "Unknown retained parent", ); assert.equal(evidence.operation, "inspect_analysis_view"); assert.deepEqual(evidence.parameters, { parent_evidence_id: input.source.evidence_id, view: input.view, }); const page = analysisViewResultSchema.parse(evidence.normalized_result); assert.deepEqual(page.view, input.view); assert.equal(page.parent_evidence_id, input.source.evidence_id); assert.equal(page.parent_operation, "inspect_web_network_capture"); if (page.kind === "summary") { assert.equal(page.summary.total_records, oracle.total_records); assert.equal(page.summary.format, "saz"); records = []; } else { assert.equal(page.kind, "capture"); records = page.view.detail === "complete" ? page.items : []; } } const prior = authenticated.get(evidence.evidence_id); if (prior && !isDeepStrictEqual(prior, records)) { conflicts.add(evidence.evidence_id); authenticated.delete(evidence.evidence_id); parents.delete(evidence.evidence_id); throw new Error("Conflicting observations share an Evidence ID"); } if (!conflicts.has(evidence.evidence_id)) authenticated.set(evidence.evidence_id, records); } catch (cause) { workflowReasons.push(String(cause)); } } if ( calls[0]?.tool !== "inspect_web_network_capture" || calls.filter((call) => call.tool === "inspect_web_network_capture").length !== 1 || (!transportRecovery && calls.length !== 1) ) workflowReasons.push( "Completed call sequence differs from the fixed initial call/recovery plan", ); try { const raw = (await readFile(resolve(values.trial, "answer.txt"), "utf8")) .trim() .replace(/^```(?:json)?\s*|\s*```$/g, ""); const answer = JSON.parse(raw); assert.ok( Array.isArray(answer.evidence_ids) && answer.evidence_ids.length > 0, "Missing Evidence citations", ); const observed = new Map(); for (const id of answer.evidence_ids) { assert.ok(authenticated.has(id), "Unknown/failed cited Evidence"); for (const row of authenticated.get(id)) { const prior = observed.get(row.ordinal); assert.ok( !prior || isDeepStrictEqual(prior, row), "Conflicting cited record identities", ); observed.set(row.ordinal, row); } } const facts = verifySazRecords( oracle.record_ordinals.map((ordinal) => observed.get(ordinal)), oracle, ); assert.equal(answer.records.length, facts.length, "Answer coverage differs"); assert.deepEqual( [...answer.records].sort((a, b) => a.ordinal - b.ordinal), [...facts].sort((a, b) => a.ordinal - b.ordinal), "Answer facts differ from cited stored bytes", ); if ( !isDeepStrictEqual( answer.records.map((row) => row.ordinal), oracle.record_ordinals, ) ) workflowReasons.push("Final answer changes caller-selected order"); if ( !Array.isArray(answer.limitations) || answer.limitations.length === 0 || answer.limitations.some( (value) => typeof value !== "string" || !value.trim(), ) ) workflowReasons.push("Final answer omits limitations"); assert.equal(settings.selected_artifact_sha256_before, oracle.sha256); assert.equal(summary.selected_artifact_sha256_after, oracle.sha256); if ( summary.exit_code !== 0 || summary.timeout || summary.selected_artifact_bytes_unchanged !== true ) workflowReasons.push( "Trial did not complete with original bytes unchanged", ); } catch (cause) { reasons.push(String(cause)); } const report = { factual_correctness: reasons.length ? "failed" : "passed", workflow_correctness: reasons.length || workflowReasons.length ? "failed" : "passed", reasons, workflow_reasons: workflowReasons, completed_calls: calls.length, wall_seconds: summary.wall_seconds, usage: summary.usage, selected_artifact_sha256: oracle.sha256, scope: "Authenticated cited stored HTTP records and independent raw-byte checks; no live attribution, body decompression or certificate interpretation", limitation_prose: "Presence checked; wording requires transcript review", }; await writeFile( resolve(values.trial, "saz-review.json"), JSON.stringify(report, null, 2) + "\n", ); console.log(JSON.stringify(report)); if (report.workflow_correctness !== "passed") process.exitCode = 1;