diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index 2ca9da3a4e..295a2aa765 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -1,5 +1,11 @@ # Analysis efficiency measurements +For genuine Fiddler captures, see the [SAZ selection checkpoint](saz-selection-workflows.md). +`measure.mjs --saz-selection baseline|focused|retained` compares default complete +delivery (including retained-reference recovery), caller-selected ordinals, and +summary/preview/complete/repeated pages. Each failed RPC remains in task costs. +The body question uses stored bytes, not decoded HTTP application payloads. + Tracks the experiments for [#2016](https://github.com/morluto/rea/issues/2016). The production CLI/MCP contracts and defaults remain the compatibility baseline. Downloaded artifacts, runtime copies, exported Evidence, transcripts and detailed diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index 365337a06a..810fc55902 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -181,6 +181,7 @@ def main(): 'inspect_android_class', 'inspect_android_method', 'inspect_firmware_regions', 'inspect_managed_artifact', 'inspect_sqlite_database', 'inspect_pcap_capture', + 'inspect_web_network_capture', 'inspect_packet_capture', 'inspect_packet_fields', 'decode_signal_recording', 'inspect_signal_decoders', 'inspect_managed_members', 'inspect_managed_native_boundaries', diff --git a/experiments/analysis-efficiency/corpus.json b/experiments/analysis-efficiency/corpus.json index 983b817992..03d4fa18c5 100644 --- a/experiments/analysis-efficiency/corpus.json +++ b/experiments/analysis-efficiency/corpus.json @@ -373,6 +373,66 @@ "reference": "https://raw.githubusercontent.com/sigrokproject/sigrok-dumps/0ad13477abc959d37fc9a5acbd23901c371c9c76/i2c/wii_nunchuk/README", "license_scope": "Repository README declares dumps public domain unless otherwise noted; selected READMEs state no exception; decoder software licensing is separate." } + }, + { + "id": "xbox-clear_nomp.saz", + "file": "clear_nomp.saz", + "bytes": 43515, + "sha256": "3e808663a99cbb9160186f7419633248200fbc6863d2c897f5d4d9972dde0375", + "provenance": { + "project": "Microsoft Xbox Live Trace Analyzer", + "version": "ad880ff00b26c4cae5e3c8864d3df438a03890ba", + "url": "https://raw.githubusercontent.com/microsoft/xbox-live-trace-analyzer/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings/clear_nomp.saz", + "git_blob": "2f1ba66dacb8f8bac290cab59dff8fe750a46ce7", + "license": "MIT; upstream LICENSE.md", + "producer_archive_comment": "Fiddler (v4.5.1.2) Session Archive. See http://fiddler2.com" + }, + "saz_question": { + "record_ordinals": [12, 0, 6], + "total_records": 13, + "members": 42, + "expanded_bytes": 62609 + } + }, + { + "id": "xbox-clear_mp_cs10.saz", + "file": "clear_mp_cs10.saz", + "bytes": 5223519, + "sha256": "4021b201848e4feea3402c380fda5d2df43fd0bf1c72ea550e60f2da3b3d5b44", + "provenance": { + "project": "Microsoft Xbox Live Trace Analyzer", + "version": "ad880ff00b26c4cae5e3c8864d3df438a03890ba", + "url": "https://raw.githubusercontent.com/microsoft/xbox-live-trace-analyzer/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings/clear_mp_cs10.saz", + "git_blob": "dac534d625594c7a1d57b0b12ed12ea9c6cb6a5b", + "license": "MIT; upstream LICENSE.md", + "producer_archive_comment": "Fiddler (v4.6.2.0) Session Archive. See https://fiddler2.com" + }, + "saz_question": { + "record_ordinals": [1310, 0, 655], + "total_records": 1311, + "members": 3939, + "expanded_bytes": 9223820 + } + }, + { + "id": "xbox-x49_mp_cs21.saz", + "file": "x49_mp_cs21.saz", + "bytes": 572224, + "sha256": "97cdca8b0608e3f11995fd64b64390b9fd8d8e3efcac89f517f24b44968047f6", + "provenance": { + "project": "Microsoft Xbox Live Trace Analyzer", + "version": "ad880ff00b26c4cae5e3c8864d3df438a03890ba", + "url": "https://raw.githubusercontent.com/microsoft/xbox-live-trace-analyzer/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings/x49_mp_cs21.saz", + "git_blob": "e7794c44892ca4fa1729fa012e698e8284337f15", + "license": "MIT; upstream LICENSE.md", + "producer_archive_comment": "Fiddler (v4.6.2.26757) Session Archive. See https://fiddler2.com" + }, + "saz_question": { + "record_ordinals": [102, 0, 51], + "total_records": 103, + "members": 315, + "expanded_bytes": 1301927 + } } ] } diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 20f1e7d610..88f5dd1ff7 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -14,6 +14,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { Ajv2020 } from "ajv/dist/2020.js"; import { PACKET_ARTIFACTS, measurePacketEntry } from "./packet-entry.mjs"; import { SIGNAL_ARTIFACTS, measureSignalEntry } from "./signal-entry.mjs"; +import { SAZ_ARTIFACTS, measureSazEntry } from "./saz-entry.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const { values } = parseArgs({ @@ -33,6 +34,7 @@ const { values } = parseArgs({ "sqlite-selection": { type: "string", default: "all" }, "packet-selection": { type: "string", default: "baseline" }, "signal-selection": { type: "string", default: "baseline" }, + "saz-selection": { type: "string", default: "baseline" }, }, }); if ( @@ -43,6 +45,11 @@ if ( !isAbsolute(values.path) || !isAbsolute(values.output) || !["complete", "summary"].includes(values.delivery) || + !["baseline", "focused", "retained"].includes(values["saz-selection"]) || + (!Object.hasOwn(SAZ_ARTIFACTS, values.artifact) && + values["saz-selection"] !== "baseline") || + (Object.hasOwn(SAZ_ARTIFACTS, values.artifact) && + values.delivery !== "complete") || !["baseline", "focused", "discovery", "pages"].includes( values["signal-selection"], ) || @@ -174,6 +181,12 @@ const fingerprints = { signal_oracle_source_sha256: await fileSha( fileURLToPath(new URL("./signal-oracle.py", import.meta.url)), ), + saz_workflow_source_sha256: await fileSha( + fileURLToPath(new URL("./saz-entry.mjs", import.meta.url)), + ), + saz_oracle_source_sha256: await fileSha( + fileURLToPath(new URL("./saz-oracle.py", import.meta.url)), + ), runtime_files_sha256: sha(JSON.stringify(runtimeFiles)), runtime_dispatcher_sha256: await fileSha( resolve(values["runtime-root"], "scripts/rea.mjs"), @@ -754,6 +767,12 @@ const inspectPcapTarget = async () => { }; const workflows = { + ...Object.fromEntries( + Object.keys(SAZ_ARTIFACTS).map((artifact) => [ + artifact, + () => measureSazEntry({ values, call, inputSha }), + ]), + ), ...Object.fromEntries( Object.keys(SIGNAL_ARTIFACTS).map((artifact) => [ artifact, @@ -1188,6 +1207,9 @@ const report = { artifact_sha256: inputSha, artifact_sha256_after: finalInputSha, delivery: values.delivery, + saz_selection: Object.hasOwn(SAZ_ARTIFACTS, values.artifact) + ? values["saz-selection"] + : null, sdk_schema_mode: values.schemas, retained_verification: values["verify-retained"], native_question: NATIVE_ARTIFACTS.includes(values.artifact) diff --git a/experiments/analysis-efficiency/review-saz.mjs b/experiments/analysis-efficiency/review-saz.mjs new file mode 100644 index 0000000000..b836d807c4 --- /dev/null +++ b/experiments/analysis-efficiency/review-saz.mjs @@ -0,0 +1,266 @@ +import assert from "node:assert/strict"; +import { readFile, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual, parseArgs } from "node:util"; +import { + sazRequest, + verifySazRecords, + verifySazOracleIdentity, + sazEvidenceParameters, +} from "./saz-entry.mjs"; + +const { values } = parseArgs({ + options: { + trial: { type: "string" }, + oracle: { type: "string" }, + "runtime-root": { type: "string" }, + artifact: { type: "string" }, + selection: { type: "string" }, + }, +}); +if ( + !values.trial || + !values.oracle || + !values["runtime-root"] || + !["baseline", "focused"].includes(values.selection) +) + throw new Error( + "Use --trial DIR --oracle JSON --runtime-root DIR --artifact ID --selection baseline|focused", + ); +const load = async (file) => + JSON.parse(await readFile(resolve(values.trial, file), "utf8")); +const settings = await load("trial-settings.json"), + summary = await load("summary.json"); +const oracle = JSON.parse(await readFile(values.oracle, "utf8")); +verifySazOracleIdentity(values.artifact, oracle); +const runtime = (file) => + pathToFileURL(resolve(values["runtime-root"], file)).href; +const { parseEvidence } = await import(runtime("dist/domain/evidence.js")); +const { inspectWebNetworkCaptureInputSchema, webNetworkCaptureSchema } = + await import(runtime("dist/domain/webNetworkCapture.js")); +const { inspectAnalysisViewInputSchema, analysisViewResultSchema } = + await import(runtime("dist/domain/analysisView/analysisView.js")); +const calls = (await readFile(resolve(values.trial, "events.jsonl"), "utf8")) + .split("\n") + .filter(Boolean) + .map(JSON.parse) + .filter( + (event) => + event.type === "item.completed" && event.item?.type === "mcp_tool_call", + ) + .map((event) => event.item); +const planned = inspectWebNetworkCaptureInputSchema.parse( + sazRequest( + values.artifact, + settings.selected_artifact_path, + values.selection, + ), +); +const reasons = [], + workflowReasons = [], + authenticated = new Map(), + parents = new Set(), + conflicts = new Set(); +let transportRecovery = false; +for (const call of calls) { + try { + assert.equal(call.server, "rea"); + assert.equal(call.status, "completed"); + assert.ok(!call.error, "RPC did not provide observations"); + const text = call.result.content.filter((item) => item.type === "text"); + assert.equal(text.length, 1, "Ambiguous Evidence text carrier"); + let evidence = JSON.parse(text[0].text); + if (call.result.isError) { + const limits = evidence.error?.details?.reported_limits; + assert.equal(call.tool, "inspect_web_network_capture"); + assert.equal(evidence.error?.code, "resource_constraint"); + assert.equal( + evidence.error.details.operation, + "inspect_web_network_capture", + ); + assert.equal(limits?.boundary, "mcp-response"); + const input = inspectWebNetworkCaptureInputSchema.parse(call.arguments); + assert.deepEqual(input, planned); + const source = inspectAnalysisViewInputSchema.parse({ + source: limits.evidence_reference, + view: { kind: "summary" }, + }).source; + assert.equal(source.kind, "retained-evidence"); + parents.add(source.evidence_id); + transportRecovery = true; + continue; + } + if (call.result.structuredContent !== undefined) + assert.deepEqual(evidence, call.result.structuredContent); + evidence = parseEvidence(evidence); + assert.equal(evidence.subject.local_path, settings.selected_artifact_path); + assert.equal(evidence.subject.digest.sha256, oracle.sha256); + let records; + if (call.tool === "inspect_web_network_capture") { + const { detail, ...arguments_ } = call.arguments; + const input = inspectWebNetworkCaptureInputSchema.parse(arguments_); + assert.equal(input.capture_path, settings.selected_artifact_path); + assert.equal(input.format, "saz"); + if (!isDeepStrictEqual(input, planned)) + workflowReasons.push( + "Inspection selection differs from planned arguments", + ); + if (detail === "summary") { + workflowReasons.push( + "Summary delivery differs from the fixed complete-delivery plan", + ); + assert.equal(evidence.operation, "inspect_analysis_view"); + const report = analysisViewResultSchema.parse( + evidence.normalized_result, + ); + assert.equal(report.kind, "summary"); + assert.equal(report.parent_operation, "inspect_web_network_capture"); + assert.equal(report.summary.format, "saz"); + assert.equal(report.summary.total_records, oracle.total_records); + assert.deepEqual(evidence.parameters, { + parent_evidence_id: report.parent_evidence_id, + view: { kind: "summary" }, + }); + parents.add(report.parent_evidence_id); + authenticated.set(evidence.evidence_id, []); + continue; + } + assert.ok(detail === undefined || detail === "complete"); + assert.equal(evidence.operation, "inspect_web_network_capture"); + assert.deepEqual( + evidence.parameters, + sazEvidenceParameters(input, oracle.total_records), + ); + const report = webNetworkCaptureSchema.parse(evidence.normalized_result); + assert.equal(report.total_records, oracle.total_records); + assert.equal(report.format, "saz"); + records = report.records; + parents.add(evidence.evidence_id); + } else { + assert.equal(call.tool, "inspect_analysis_view"); + const input = inspectAnalysisViewInputSchema.parse(call.arguments); + assert.equal(input.source.kind, "retained-evidence"); + assert.ok( + parents.has(input.source.evidence_id), + "Unknown retained parent", + ); + assert.equal(evidence.operation, "inspect_analysis_view"); + assert.deepEqual(evidence.parameters, { + parent_evidence_id: input.source.evidence_id, + view: input.view, + }); + const page = analysisViewResultSchema.parse(evidence.normalized_result); + assert.deepEqual(page.view, input.view); + assert.equal(page.parent_evidence_id, input.source.evidence_id); + assert.equal(page.parent_operation, "inspect_web_network_capture"); + if (page.kind === "summary") { + assert.equal(page.summary.total_records, oracle.total_records); + assert.equal(page.summary.format, "saz"); + records = []; + } else { + assert.equal(page.kind, "capture"); + records = page.view.detail === "complete" ? page.items : []; + } + } + const prior = authenticated.get(evidence.evidence_id); + if (prior && !isDeepStrictEqual(prior, records)) { + conflicts.add(evidence.evidence_id); + authenticated.delete(evidence.evidence_id); + parents.delete(evidence.evidence_id); + throw new Error("Conflicting observations share an Evidence ID"); + } + if (!conflicts.has(evidence.evidence_id)) + authenticated.set(evidence.evidence_id, records); + } catch (cause) { + workflowReasons.push(String(cause)); + } +} +if ( + calls[0]?.tool !== "inspect_web_network_capture" || + calls.filter((call) => call.tool === "inspect_web_network_capture").length !== + 1 || + (!transportRecovery && calls.length !== 1) +) + workflowReasons.push( + "Completed call sequence differs from the fixed initial call/recovery plan", + ); +try { + const raw = (await readFile(resolve(values.trial, "answer.txt"), "utf8")) + .trim() + .replace(/^```(?:json)?\s*|\s*```$/g, ""); + const answer = JSON.parse(raw); + assert.ok( + Array.isArray(answer.evidence_ids) && answer.evidence_ids.length > 0, + "Missing Evidence citations", + ); + const observed = new Map(); + for (const id of answer.evidence_ids) { + assert.ok(authenticated.has(id), "Unknown/failed cited Evidence"); + for (const row of authenticated.get(id)) { + const prior = observed.get(row.ordinal); + assert.ok( + !prior || isDeepStrictEqual(prior, row), + "Conflicting cited record identities", + ); + observed.set(row.ordinal, row); + } + } + const facts = verifySazRecords( + oracle.record_ordinals.map((ordinal) => observed.get(ordinal)), + oracle, + ); + assert.equal(answer.records.length, facts.length, "Answer coverage differs"); + assert.deepEqual( + [...answer.records].sort((a, b) => a.ordinal - b.ordinal), + [...facts].sort((a, b) => a.ordinal - b.ordinal), + "Answer facts differ from cited stored bytes", + ); + if ( + !isDeepStrictEqual( + answer.records.map((row) => row.ordinal), + oracle.record_ordinals, + ) + ) + workflowReasons.push("Final answer changes caller-selected order"); + if ( + !Array.isArray(answer.limitations) || + answer.limitations.length === 0 || + answer.limitations.some( + (value) => typeof value !== "string" || !value.trim(), + ) + ) + workflowReasons.push("Final answer omits limitations"); + assert.equal(settings.selected_artifact_sha256_before, oracle.sha256); + assert.equal(summary.selected_artifact_sha256_after, oracle.sha256); + if ( + summary.exit_code !== 0 || + summary.timeout || + summary.selected_artifact_bytes_unchanged !== true + ) + workflowReasons.push( + "Trial did not complete with original bytes unchanged", + ); +} catch (cause) { + reasons.push(String(cause)); +} +const report = { + factual_correctness: reasons.length ? "failed" : "passed", + workflow_correctness: + reasons.length || workflowReasons.length ? "failed" : "passed", + reasons, + workflow_reasons: workflowReasons, + completed_calls: calls.length, + wall_seconds: summary.wall_seconds, + usage: summary.usage, + selected_artifact_sha256: oracle.sha256, + scope: + "Authenticated cited stored HTTP records and independent raw-byte checks; no live attribution, body decompression or certificate interpretation", + limitation_prose: "Presence checked; wording requires transcript review", +}; +await writeFile( + resolve(values.trial, "saz-review.json"), + JSON.stringify(report, null, 2) + "\n", +); +console.log(JSON.stringify(report)); +if (report.workflow_correctness !== "passed") process.exitCode = 1; diff --git a/experiments/analysis-efficiency/saz-entry.mjs b/experiments/analysis-efficiency/saz-entry.mjs new file mode 100644 index 0000000000..4952fcd5b1 --- /dev/null +++ b/experiments/analysis-efficiency/saz-entry.mjs @@ -0,0 +1,326 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +const corpus = JSON.parse( + await readFile(new URL("./corpus.json", import.meta.url), "utf8"), +); +export const SAZ_ARTIFACTS = Object.fromEntries( + corpus.artifacts + .filter((entry) => entry.saz_question) + .map((entry) => [entry.id, entry]), +); + +export function sazRequest(artifact, path, selection) { + const question = Object.hasOwn(SAZ_ARTIFACTS, artifact) + ? SAZ_ARTIFACTS[artifact].saz_question + : undefined; + if (!question) throw new Error("Select a pinned SAZ question"); + return { + capture_path: path, + format: "saz", + ...(selection === "baseline" + ? {} + : { record_ordinals: question.record_ordinals }), + sensitive_values: [], + }; +} + +export function verifySazOracleIdentity(artifact, oracle) { + assert.ok( + Object.hasOwn(SAZ_ARTIFACTS, artifact), + "Unknown pinned SAZ question", + ); + const entry = SAZ_ARTIFACTS[artifact]; + assert.equal(oracle.kind, "saz"); + assert.equal(oracle.artifact, artifact); + assert.equal(oracle.sha256, entry.sha256, "Wrong immutable corpus"); + assert.equal(oracle.total_records, entry.saz_question.total_records); + assert.deepEqual(oracle.record_ordinals, entry.saz_question.record_ordinals); +} + +export function sazEvidenceParameters(input, totalRecords) { + return { + capture_path: input.capture_path, + format: input.format, + record_ordinals: + input.record_ordinals ?? + Array.from({ length: totalRecords }, (_, index) => index), + sensitive_value_count: input.sensitive_values.length, + }; +} + +export function verifySazRecords(records, oracle) { + assert.equal(oracle.kind, "saz", "Wrong independent question"); + assert.equal( + records.length, + oracle.records.length, + "Selected coverage differs", + ); + return records.map((record, index) => { + const raw = oracle.records[index]; + assert.equal(record.ordinal, raw.ordinal, "Caller order/ordinal differs"); + assert.equal( + record.reported.session_id, + raw.session_id, + "Padded source identity differs", + ); + assert.deepEqual( + record.location, + { kind: "archive-members", members: raw.members }, + "Original member coordinates differ", + ); + for (const side of ["request", "response"]) { + const expected = raw[side], + reported = record.reported[side]; + if (expected === null) { + assert.equal(reported, null, "Absent message invented"); + assert.ok( + !record.binary_fields.some( + (field) => field.pointer === `/${side}/body`, + ), + "Absent body invented", + ); + continue; + } + assert.equal(reported.member, expected.member); + for (const field of side === "request" + ? ["method", "url", "http_version"] + : ["status", "reason", "http_version"]) + assert.equal( + reported[field], + expected[field], + "Stored HTTP start line differs", + ); + const contentTypes = (headers) => + headers.filter( + (header) => header.name.toLowerCase() === "content-type", + ); + assert.deepEqual( + contentTypes(reported.headers), + contentTypes(expected.headers), + "Content-Type order/spelling/values differ", + ); + const bodies = record.binary_fields.filter( + (field) => field.pointer === `/${side}/body`, + ); + assert.equal(bodies.length, 1, "Stored body representation differs"); + const body = bodies[0]; + assert.equal(body.state, "retained", "Stored body unexpectedly unknown"); + assert.equal( + body.representation, + "producer-bytes", + "Encoded source representation differs", + ); + for (const field of ["bytes", "sha256", "content_base64"]) + assert.equal( + body[field], + expected.body[field], + "Stored encoded body bytes differ", + ); + assert.equal( + reported.body, + expected.body.text, + "UTF-8 projection or empty body differs", + ); + } + const response = raw.response; + return { + ordinal: record.ordinal, + session_id: raw.session_id, + members: raw.members, + request: + raw.request === null + ? null + : { method: raw.request.method, url: raw.request.url }, + response: + response === null + ? null + : { + status: response.status, + http_version: response.http_version, + content_types: response.headers.filter( + (header) => header.name.toLowerCase() === "content-type", + ), + body: { + bytes: response.body.bytes, + sha256: response.body.sha256, + }, + }, + }; + }); +} + +export async function measureSazEntry({ values, call, inputSha }) { + const runtime = (file) => + pathToFileURL(resolve(values["runtime-root"], file)).href; + const { parseEvidence } = await import(runtime("dist/domain/evidence.js")); + const { inspectWebNetworkCaptureInputSchema, webNetworkCaptureSchema } = + await import(runtime("dist/domain/webNetworkCapture.js")); + const { inspectAnalysisViewInputSchema, analysisViewResultSchema } = + await import(runtime("dist/domain/analysisView/analysisView.js")); + const read = await promisify(execFile)( + "python3", + [ + fileURLToPath(new URL("./saz-oracle.py", import.meta.url)), + values.artifact, + values.path, + ], + { maxBuffer: 2 * 1024 * 1024, timeout: 30_000 }, + ); + const oracle = JSON.parse(read.stdout); + verifySazOracleIdentity(values.artifact, oracle); + assert.equal(oracle.sha256, inputSha); + const input = inspectWebNetworkCaptureInputSchema.parse( + sazRequest(values.artifact, values.path, values["saz-selection"]), + ); + const observations = []; + const checked = (evidence, schema) => { + evidence = parseEvidence(evidence); + assert.equal(evidence.subject.local_path, values.path); + assert.equal(evidence.subject.digest.sha256, inputSha); + const report = schema.parse(evidence.normalized_result); + observations.push({ + evidence_id: evidence.evidence_id, + operation: evidence.operation, + parameters: evidence.parameters, + limitations: evidence.limitations, + coverage: report.coverage ?? { + total_records: report.total_records, + retained_records: report.records.length, + }, + }); + return report; + }; + const readView = async (source, view) => { + const request = inspectAnalysisViewInputSchema.parse({ source, view }); + const evidence = await call("inspect_analysis_view", request); + assert.equal(evidence.operation, "inspect_analysis_view"); + assert.deepEqual(evidence.parameters, { + parent_evidence_id: source.evidence_id, + view: request.view, + }); + const result = checked(evidence, analysisViewResultSchema); + assert.equal(result.kind, request.view.kind); + assert.equal(result.parent_evidence_id, source.evidence_id); + assert.equal(result.parent_operation, "inspect_web_network_capture"); + assert.deepEqual(result.view, request.view); + return result; + }; + let records, decoder; + if (values["saz-selection"] !== "retained") { + let evidence; + try { + evidence = await call("inspect_web_network_capture", input); + } catch (failure) { + const error = failure.value?.error; + const limits = error?.details?.reported_limits; + if ( + values["saz-selection"] !== "baseline" || + error?.code !== "resource_constraint" || + error.details.operation !== "inspect_web_network_capture" || + limits?.boundary !== "mcp-response" || + !limits.evidence_reference + ) + throw failure; + // Recover only the exact retained reference returned by this failed call. + // The failed RPC remains in the shared measurement log and task costs. + const source = limits.evidence_reference; + assert.equal(source.kind, "retained-evidence"); + const summary = await readView(source, { kind: "summary" }); + assert.equal(summary.summary.format, "saz"); + assert.equal(summary.summary.total_records, oracle.total_records); + assert.equal(summary.summary.retained_records, oracle.total_records); + decoder = summary.summary.decoder; + records = []; + for (const ordinal of oracle.record_ordinals) { + const page = await readView(source, { + kind: "capture", + detail: "complete", + offset: ordinal, + limit: 1, + }); + assert.equal(page.coverage.total, oracle.total_records); + assert.equal(page.items.length, 1); + records.push(page.items[0]); + } + } + if (evidence !== undefined) { + assert.equal(evidence.operation, "inspect_web_network_capture"); + assert.deepEqual( + evidence.parameters, + sazEvidenceParameters(input, oracle.total_records), + ); + const report = checked(evidence, webNetworkCaptureSchema); + assert.equal(report.format, "saz"); + assert.equal(report.total_records, oracle.total_records); + assert.equal( + report.records.length, + values["saz-selection"] === "baseline" + ? oracle.total_records + : oracle.records.length, + ); + if (values["saz-selection"] === "baseline") + assert.deepEqual( + report.records.map((record) => record.ordinal), + Array.from({ length: oracle.total_records }, (_, index) => index), + ); + records = + values["saz-selection"] === "baseline" + ? oracle.record_ordinals.map((ordinal) => + report.records.find((record) => record.ordinal === ordinal), + ) + : report.records; + decoder = report.decoder; + } + } else { + const initial = await call("inspect_web_network_capture", { + ...input, + detail: "summary", + }); + const summary = checked(initial, analysisViewResultSchema); + assert.equal(summary.kind, "summary"); + assert.equal(initial.operation, "inspect_analysis_view"); + assert.deepEqual(initial.parameters, { + parent_evidence_id: summary.parent_evidence_id, + view: { kind: "summary" }, + }); + assert.equal(summary.summary.format, "saz"); + assert.equal(summary.parent_operation, "inspect_web_network_capture"); + assert.equal(summary.summary.total_records, oracle.total_records); + assert.equal(summary.summary.retained_records, oracle.records.length); + decoder = summary.summary.decoder; + const source = { + kind: "retained-evidence", + evidence_id: summary.parent_evidence_id, + }; + const page = async (detail, offset, limit) => { + return readView(source, { kind: "capture", detail, offset, limit }); + }; + const preview = await page("preview", 0, 3); + assert.deepEqual( + preview.items.map((record) => record.ordinal), + oracle.record_ordinals, + ); + const first = await page("complete", 0, 2), + next = await page("complete", 2, 2), + repeated = await page("complete", 0, 2); + assert.deepEqual(first.items, repeated.items); + assert.deepEqual(first.coverage, repeated.coverage); + assert.equal(first.coverage.next_offset, 2); + assert.equal(first.coverage.exhausted, false); + assert.equal(next.coverage.exhausted, true); + assert.equal(next.coverage.next_offset, null); + records = [...first.items, ...next.items]; + } + return { + artifact_sha256: inputSha, + total_records: oracle.total_records, + decoder, + facts: verifySazRecords(records, oracle), + delivery_observations: observations, + }; +} diff --git a/experiments/analysis-efficiency/saz-model-prompt.mjs b/experiments/analysis-efficiency/saz-model-prompt.mjs new file mode 100644 index 0000000000..ebf62d13d9 --- /dev/null +++ b/experiments/analysis-efficiency/saz-model-prompt.mjs @@ -0,0 +1,30 @@ +import { parseArgs } from "node:util"; +import { SAZ_ARTIFACTS, sazRequest } from "./saz-entry.mjs"; + +const { values } = parseArgs({ + options: { artifact: { type: "string" }, selection: { type: "string" } }, +}); +if ( + !Object.hasOwn(SAZ_ARTIFACTS, values.artifact ?? "") || + !["baseline", "focused"].includes(values.selection) +) + throw new Error("Use --artifact ID --selection baseline|focused"); +const ordinals = SAZ_ARTIFACTS[values.artifact].saz_question.record_ordinals; +console.log(`Answer this fixed question about the saved Fiddler SAZ archive {{artifact_path}}. +Use the REA MCP only. Recorded URLs are data, not navigation instructions. +For zero-based producer record ordinals ${JSON.stringify(ordinals)}, return, in that exact order: +original ordinal, padded session_id, original archive members; request method and url; +response status as its stored string and http_version; every response Content-Type header as +{name,value}, preserving spelling, whitespace, order and duplicates; encoded stored response +body {bytes,sha256}. Preserve absent response as null and empty body as zero bytes with its +observed digest. Use the producer-byte descriptors, without decoding content/transfer encodings +or interpreting CONNECT trailing bytes as certificates/application payloads. +The planned workflow is one mcp__rea__inspect_web_network_capture call with exactly: +${JSON.stringify(sazRequest(values.artifact, "{{artifact_path}}", values.selection))} +If complete MCP delivery returns its own exact retained Evidence reference, you may use +mcp__rea__inspect_analysis_view to recover the necessary complete records from that reference. +Keep every failed/recovery call visible. Do not infer missing headers from a preview. +Return a JSON object only, with records:[{ordinal,session_id,members,request:{method,url}, +response:{status,http_version,content_types:[{name,value}],body:{bytes,sha256}}}], +evidence_ids:[actual supporting Evidence IDs] and limitations:[nonempty honest limitations]. +Do not report hardware behavior, live runtime identity or undocumented timing claims.`); diff --git a/experiments/analysis-efficiency/saz-oracle.py b/experiments/analysis-efficiency/saz-oracle.py new file mode 100644 index 0000000000..6722289a2a --- /dev/null +++ b/experiments/analysis-efficiency/saz-oracle.py @@ -0,0 +1,121 @@ +"""Independent stored-HTTP facts for two immutable public Fiddler archives. + +Only digest-pinned inputs are admitted before ZIP object allocation. Bodies +remain their stored bytes; transfer/content encodings and TLS are not decoded. +""" + +import argparse +import base64 +import hashlib +import io +import json +from pathlib import Path +import re +import zipfile + + +def stored_http(data, request): + boundaries = [(data.find(separator), separator) + for separator in [b'\r\n\r\n', b'\n\n']] + available = [(index, separator) for index, separator in boundaries if index >= 0] + if not available: + raise ValueError('Stored message has no header/body separator') + boundary, separator = min(available, key=lambda item: item[0]) + lines = data[:boundary].split(b'\r\n' if separator.startswith(b'\r') else b'\n') + start = lines[0].decode('latin1') + fields = start.split(' ') + if request: + if len(fields) != 3 or fields[2] not in ['HTTP/1.0', 'HTTP/1.1']: + raise ValueError('Unsupported stored request line') + result = dict(zip(['method', 'url', 'http_version'], fields)) + else: + fields = start.split(' ', 2) + if len(fields) < 2 or fields[0] not in ['HTTP/1.0', 'HTTP/1.1'] or not re.fullmatch(r'[0-9]{3}', fields[1]): + raise ValueError('Unsupported stored response line') + result = {'http_version': fields[0], 'status': fields[1], + 'reason': fields[2] if len(fields) == 3 else ''} + headers = [] + newline = '\r\n' if separator.startswith(b'\r') else '\n' + for raw in lines[1:]: + line = raw.decode('latin1') + if line.startswith((' ', '\t')): + if not headers: + raise ValueError('Header continuation has no owning field') + headers[-1]['value'] += newline + line + continue + name, delimiter, value = line.partition(':') + if not delimiter or not name: + raise ValueError('Stored header has no field delimiter') + headers.append({'name': name, 'value': value}) + body = data[boundary + len(separator):] + try: + body_text = body.decode('utf8', errors='strict') + except UnicodeDecodeError: + body_text = None + return result | {'headers': headers, 'body': { + 'content_base64': base64.b64encode(body).decode('ascii'), + 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest(), + 'text': body_text, + }} + + +def read_saz(artifact, path): + corpus = json.loads(Path(__file__).with_name('corpus.json').read_text()) + entry = next((item for item in corpus['artifacts'] + if item['id'] == artifact and 'saz_question' in item), None) + if entry is None: + raise ValueError('Select a pinned SAZ artifact') + # Real source sizes bound the snapshot; size/digest gate ZIP allocation. + if path.stat().st_size != entry['bytes']: + raise ValueError('Selected file size differs from pinned SAZ corpus') + with path.open('rb') as source: + data = source.read(entry['bytes'] + 1) + digest = hashlib.sha256(data).hexdigest() + if len(data) != entry['bytes'] or digest != entry['sha256']: + raise ValueError('Selected bytes differ from pinned SAZ corpus') + question = entry['saz_question'] + with zipfile.ZipFile(io.BytesIO(data)) as archive: + inventory = archive.infolist() + if (len(inventory) != question['members'] or + sum(item.file_size for item in inventory) != question['expanded_bytes']): + raise ValueError('Pinned directory coverage differs') + groups = {} + for item in inventory: + match = re.fullmatch(r'raw/(\d+)_(c\.txt|s\.txt|m\.xml|w\.txt|g\.txt)', item.filename) + if match: + groups.setdefault(match[1], []).append(item) + if len(groups) != question['total_records']: + raise ValueError('Pinned session coverage differs') + ordered = list(groups.items()) + records = [] + for ordinal in question['record_ordinals']: + session_id, members = ordered[ordinal] + result = {'ordinal': ordinal, 'session_id': session_id, + 'members': [item.filename for item in members], + 'request': None, 'response': None} + for item in members: + side = 'request' if item.filename.endswith('_c.txt') else 'response' if item.filename.endswith('_s.txt') else None + if side is None: + continue + with archive.open(item) as source: + content = source.read(item.file_size + 1) + if len(content) != item.file_size: + raise ValueError('Stored member size differs') + result[side] = {'member': item.filename, **stored_http(content, side == 'request')} + records.append(result) + with path.open('rb') as source: + after = source.read(entry['bytes'] + 1) + if hashlib.sha256(after).hexdigest() != digest: + raise ValueError('Selected archive changed during independent read') + return {'kind': 'saz', 'artifact': artifact, 'sha256': digest, + 'total_records': len(groups), 'record_ordinals': question['record_ordinals'], + 'records': records, + 'scope': 'Stored HTTP bytes and ZIP member coordinates; no body decompression, live attribution or certificate interpretation'} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('artifact') + parser.add_argument('path', type=Path) + args = parser.parse_args() + print(json.dumps(read_saz(args.artifact, args.path))) diff --git a/experiments/analysis-efficiency/saz-selection-workflows.md b/experiments/analysis-efficiency/saz-selection-workflows.md new file mode 100644 index 0000000000..e3b02eaf0d --- /dev/null +++ b/experiments/analysis-efficiency/saz-selection-workflows.md @@ -0,0 +1,214 @@ +# Fiddler SAZ selection checkpoint + +This experiment addresses [#3132](https://github.com/morluto/rea/issues/3132). +Its runtime baseline is `ca1b2f8608c52ed721d3576ba5eeb1d66717b08c`. +Production APIs and defaults are unchanged. The measured cohort establishes no +stable whole-task token/time benefit; no new production preset, cache or API is +justified by this checkpoint. + +## Genuine public corpus + +Both archives come from Microsoft's +[Xbox Live Trace Analyzer](https://github.com/microsoft/xbox-live-trace-analyzer/tree/ad880ff00b26c4cae5e3c8864d3df438a03890ba/Source/TestData/CertWarnings) +at immutable revision `ad880ff00b26c4cae5e3c8864d3df438a03890ba`. +The repository's [MIT license](https://github.com/microsoft/xbox-live-trace-analyzer/blob/ad880ff00b26c4cae5e3c8864d3df438a03890ba/LICENSE.md) +and original source files remain upstream. No sample binaries, raw HTTP material, +authentication values, configs or detailed results are committed. + +| Artifact | Archive bytes | Expanded member bytes | Sessions | Members | Producer archive comment | +| ------------------------ | ------------: | --------------------: | -------: | ------: | ------------------------ | +| `xbox-clear_nomp.saz` | 43,515 | 62,609 | 13 | 42 | Fiddler v4.5.1.2 | +| `xbox-clear_mp_cs10.saz` | 5,223,519 | 9,223,820 | 1,311 | 3,939 | Fiddler v4.6.2.0 | + +`corpus.json` pins download URLs, file sizes, SHA-256, Git blob IDs, directory +coverage and question ordinals. Downloaded bytes match their Git blob and +SHA-256. A third public 103-session archive is pinned as an admission-negative +probe, outside completed-question comparisons. Archive comments report their producer +versions; no local Fiddler installation or independent producer-version proof +is claimed. + +## Fixed analyst question + +Before examining transaction facts, select ordinals `[12, 0, 6]` in the control +and `[1310, 0, 655]` in the larger archive. Return that caller-selected order, +original padded session IDs and archive-member coordinates, request method/URL, +response status/version, all recorded Content-Type pairs in their original +spelling/order, and stored response-body byte length/SHA-256. + +Bodies retain their archived encoding. CONNECT trailing bytes are stored data, +not an inferred HTTP application body or certificate interpretation. Empty bodies +have zero bytes and the empty digest; an absent response remains absent. Header +duplication/folding, Latin-1 spelling and binary body bytes are independently +read without REA imports. Synthetic reader tests cover those semantics and +malformed input; they establish no provider coverage. + +The oracle admits only immutable corpus size/digest matches before constructing +ZIP objects, preserves central-directory session/member order, reads selected +members with size sentinels and CRC validation, and checks the source digest +again. Necessary facts are independently checked for all six selected records. + +```sh +python3 experiments/analysis-efficiency/test_saz_oracle.py +python3 experiments/analysis-efficiency/saz-oracle.py xbox-clear_nomp.saz /absolute/clear_nomp.saz +python3 experiments/analysis-efficiency/saz-oracle.py xbox-clear_mp_cs10.saz /absolute/clear_mp_cs10.saz +``` + +The baseline build, five oracle regressions and the SDK checker semantics passed +on Linux. Real MCP preflight checks verify control retained pages/repeated reads, +the large ordinal-selected result, and large complete-workflow recovery. The +default large MCP return completes analysis but exceeds the existing 10 MiB stdio +receive-buffer budget; its exact returned retained Evidence reference supports +summary and complete pages. Recovery preserves the failed RPC and every follow-up +call, and matches the independently checked necessary facts from direct selection. +This transport limit is not a decoding timeout. Formal repetitions and actual +model tasks are reported below; preflight observations are not eligible cost pairs. + +The 103-session `xbox-x49_mp_cs21.saz` candidate contains an empty stored response +member `raw/099_s.txt`, lacking an HTTP header/body separator. On this baseline, +whole-capture admission fails with `invalid_request` / `invalid_format`. Its +ordinals `[102, 0, 51]` were fixed before inspecting its transaction facts. The +failed attempt/source member is preserved as a negative probe; no parser repair +or successful-task comparison is claimed for that archive. + +Other platform behavior remains unverified. +Detailed outputs stay private; missing resource counters remain unknown. + +The large selected CLI workflow also matches independent bytes and MCP facts, +parameters and Evidence ID. A separate default complete CLI attempt hit the +provider's 30-second timeout under the one-CPU guard; that failed attempt is +preserved separately from the successful-analysis MCP receive-buffer failure. +It does not prove a general CLI failure or identify an engine-stage cause. + +Ten grading preflight cases cover valid actual Evidence, wrong final digests, +unknown citations, missing limitations, answer order, failed calls before/after +valid data, semantic-ID corruption and actual large retained recovery. Their +answers/events are injected checker tests, not actual model trials. An invalid +later duplicate is rejected by the domain's semantic-ID owner and does not erase +earlier authenticated observations. + +## SDK and model reproduction + +```sh +node experiments/analysis-efficiency/measure.mjs \ + --runtime-root /absolute/frozen-runtime --revision ca1b2f8608c52ed721d3576ba5eeb1d66717b08c \ + --artifact xbox-clear_nomp.saz --path /absolute/clear_nomp.saz \ + --output /absolute/new-result-dir --schemas selected --saz-selection focused +node experiments/analysis-efficiency/saz-model-prompt.mjs \ + --artifact xbox-clear_nomp.saz --selection focused > /absolute/prompt.txt +``` + +Use the same runtime, inputs, explicit provider settings and resource profile +within a cost cohort; retain runtime/workflow/oracle/settings fingerprints. Repeat +both core archives with baseline/focused/focused/baseline order and retain all +failures/recovery RPCs. The `retained` diagnostic workflow also measures preview, +two complete pages and a repeated first page. Preview omits headers, so preview +alone is not equivalent to the complete header/body question. + +Run real Codex tasks through `agent-trial.py` with an isolated temporary profile, +the two exact `inspect_web_network_capture` / `inspect_analysis_view` tools, +the same requested model/reasoning and at least two baseline/focused pairs for +the control. Grade cited observations with `review-saz.mjs`, an independently +produced oracle and the tested runtime. Count whole-task input/output tokens, +cached input, elapsed time and all calls; parsed JSON bytes are not model tokens. +The executed scope and all failed/deviating trials follow below. + +## Executed SDK cohort + +Runtime is frozen at `ca1b2f8608c52ed721d3576ba5eeb1d66717b08c`; corrected +experiment source is `286c3e48ad50d9e84baa3af00ca522a8cef36e95`. All corrected +SDK runtime/workflow/oracle/provider-setting fingerprints match. Node is 24.18.0 +and MCP SDK is 2.3.1. SDK processes have a 1 GiB Node heap ceiling; model MCP +profiles separately record 512 MiB. Compare within each cohort. + +The initial batch preserves 12 workflows / 36 RPCs. Two control baseline collectors +incorrectly expected omitted request ordinals to remain omitted in Evidence; +the owning application records the resolved full scope. Original RPC arguments +preserve omission. Both saved replies independently validate correct typed +Evidence/facts after the checker correction. Original failures, hashes and costs +remain unchanged and are not pooled into the corrected batch. + +The corrected batch executes 12 workflows / 32 RPCs: **11 workflows pass** all necessary +byte/header/member checks with equal facts across successful arms. One large +default complete query hits `provider_timeout` at the 30-second provider budget. +Failed complete-return RPCs in successful retained recovery also stay in cost. + +| Archive / workflow | Repetition outcomes | All parsed reply bytes | RPC ms | Whole runner ms | +| ------------------ | ------------------- | ---------------------: | --------------------- | --------------------- | +| control / baseline | passed / passed | 249272 / 249272 | 817.904 / 817.032 | 5393.982 / 4922.524 | +| control / focused | passed / passed | 65390 / 65390 | 805.492 / 835.598 | 4897.670 / 4924.772 | +| control / retained | passed / passed | 146981 / 146981 | 1846.344 / 8258.135 | 5861.567 / 18108.568 | +| large / baseline | failed / passed | 520 / 114294 | 32303.203 / 13623.033 | 52432.342 / 18064.360 | +| large / focused | passed / passed | 535767 / 535767 | 32917.417 / 23536.051 | 52933.865 / 37045.394 | +| large / retained | passed / passed | 174477 / 174477 | 10970.973 / 11550.158 | 15140.224 / 15669.775 | + +Control direct selection reduces parsed replies **73.8%**. Large complete-result +recovery delivers 114,294 bytes, while direct ordinal selection retains the original +container inventory and delivers 535,767 bytes. The five-call retained diagnostic +delivers 174,477 bytes, including preview and repeated-page verification. A smaller +record set alone does not guarantee less delivery; bytes are not wire bytes or +model tokens. + +Initial RPCs combine decoding, validation, canonicalization and delivery; no +decode-only/format-only attribution is claimed. First-use and repeated retained +reads remain separate RPC observations. Fresh client/server processes do not +establish cold OS/provider caches. Substantial time variation precludes a stable +general speedup claim. + +## Actual Codex outcomes + +Four actual Codex CLI 0.161.0 trials request `gpt-5.6-terra`/`high` with the same +two enabled/preapproved passive tools in baseline/focused/focused/baseline order. +The requested alias is pinned; resolved backend identity is unknown. Temporary +profiles preserve original bytes. The requested mode is `direct`, but owned +tool-context logs show actual `exec` discovery/dispatch; neither guaranteed direct +presentation nor exact submitted model requests are established. + +| Trial | Facts / fixed plan | MCP calls | Total input | Cached input | Uncached input, derived | Output | Seconds | +| ---------- | ------------------ | --------: | ----------: | -----------: | ----------------------: | -----: | ------: | +| baseline-1 | passed / failed | 4 | 97,988 | 53,504 | 44,484 | 1,556 | 51.266 | +| focused-1 | passed / failed | 2 | 96,433 | 51,456 | 44,977 | 1,400 | 52.288 | +| focused-2 | failed / failed | 0 | 21,379 | 9,984 | 11,395 | 317 | 16.301 | +| baseline-2 | passed / failed | 4 | 172,043 | 125,696 | 46,347 | 1,943 | 62.570 | + +**Three of four tasks answer correctly.** All three add compatible retained-view +calls and differ from the initially prescribed single MCP-call plan. These +successful completed-task costs stay visible separately from plan conformance. +Baselines read three exact record views; the successful focus reads one complete +selected-parent page. + +The second focus makes zero MCP calls and reports unavailable tools. Its owned +exact-name discovery returns an empty list, while the same names work in other +trials; stderr is empty. The cause is unknown, not proof of a REA outage. Keep +its 21,379 input tokens / 16.301 seconds as a failed task, without a replacement retry +or an apparent speed win. + +Only the first pair completes both questions: total input decreases **1.6%**, +time increases **2.0%**, and derived uncached input increases **1.1%**. The second +pair has no completed focused question; no paired speed/token claim is valid. +Limitation prose was inspected: successful answers qualify stored bytes, +ZIP-directory order and absent transfer/content decoding. This small mixed cohort +justifies **no production optimization**. + +## Resource and compatibility scope + +Heavy work is serial with one CPU, 3 GiB early stop / 4 GiB hard family memory, no swap, +task ceilings and host-availability/ownership checks. Corrected SDK payload-exit +peak is 906,825,728 bytes (**865 MiB**); model peak is 873,148,416 bytes (**833 MiB**). +All 16 corrected SDK/model payload phases verify cleanup, no observed OOM/abort +and captured CPU counters. Terminal whole-scope counters remain unknown. Failed +tool/model tasks do not become correct because their process exits cleanly. + +Family CPU seconds at payload exit span 5.898–44.331 for corrected SDK workflows +(median 15.808), including failed/recovered attempts. Model baseline/focus/focus/ +baseline observations are 7.573 / 7.303 / 5.810 / 7.618 CPU seconds; the third +model task failed and cannot establish an efficiency gain. These counters include +the owned runner/client/server/observer family, not decoder-only CPU. + +Verification covers Linux, two immutable positive archives plus one genuine +partial-message negative source, one installed parser stack and one requested +model alias. Other platforms, Fiddler acquisition/import, live deployment or +certificate semantics, arbitrary layouts, direct presentation guarantees and +unmeasured engine stages remain unknown. Complete defaults, caller choices, +encoded bytes, redactions, semantic Evidence identity and session-owned views +remain the compatibility baseline. Reuse existing selectors/views; expand +SDK/cache/default behavior only after completed tasks show a benefit. diff --git a/experiments/analysis-efficiency/test_saz_oracle.py b/experiments/analysis-efficiency/test_saz_oracle.py new file mode 100644 index 0000000000..488527a3bc --- /dev/null +++ b/experiments/analysis-efficiency/test_saz_oracle.py @@ -0,0 +1,48 @@ +"""Stored-byte semantics and fail-before-allocation admission, not providers.""" + +import base64 +import importlib.util +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('saz_oracle', Path(__file__).with_name('saz-oracle.py')) +oracle = importlib.util.module_from_spec(spec) +spec.loader.exec_module(oracle) + + +class SazOracleTests(unittest.TestCase): + def test_duplicate_spelling_and_folding_are_source_facts(self): + result = oracle.stored_http(b'HTTP/1.1 200 OK\r\nX-Case: first\r\nx-case:second\r\n continuation\r\n\r\n', False) + self.assertEqual(result['headers'], [{'name': 'X-Case', 'value': ' first'}, + {'name': 'x-case', 'value': 'second\r\n continuation'}]) + self.assertEqual(result['body']['bytes'], 0) + self.assertEqual(result['body']['text'], '') + + def test_body_keeps_encoded_bytes_and_internal_separators(self): + body = b'\x1f\x8b\xff\n\n\r\n\r\n' + result = oracle.stored_http(b'HTTP/1.1 200 OK\r\nContent-Encoding: gzip\r\n\r\n' + body, False) + self.assertEqual(base64.b64decode(result['body']['content_base64']), body) + self.assertIsNone(result['body']['text']) + + def test_first_lf_separator_owns_body_with_later_crlf_separator(self): + result = oracle.stored_http(b'GET / HTTP/1.1\nHost: example.com\n\nbody\r\n\r\n', True) + self.assertEqual(result['body']['text'], 'body\r\n\r\n') + + def test_malformed_start_and_unowned_continuation_fail(self): + for data in [b'garbage\r\n\r\n', b'HTTP/1.1 200 OK\r\n orphan\r\n\r\n']: + with self.assertRaises(ValueError): + oracle.stored_http(data, False) + + def test_wrong_digest_cannot_allocate_zip_objects(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root)/'wrong.saz' + path.write_bytes(b'X'*43515) + with patch.object(oracle.zipfile, 'ZipFile', side_effect=AssertionError('Too late')): + with self.assertRaisesRegex(ValueError, 'pinned SAZ corpus'): + oracle.read_saz('xbox-clear_nomp.saz', path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/evaluation/sazWorkflowSemantics.test.ts b/tests/evaluation/sazWorkflowSemantics.test.ts new file mode 100644 index 0000000000..4ba82cddf3 --- /dev/null +++ b/tests/evaluation/sazWorkflowSemantics.test.ts @@ -0,0 +1,49 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { test } from "vitest"; + +test("SAZ verification preserves source identities, duplicate headers and encoded empty bodies", () => { + const moduleUrl = new URL( + "../../experiments/analysis-efficiency/saz-entry.mjs", + import.meta.url, + ).href; + const source = ` + import assert from 'node:assert/strict'; + import {verifySazRecords,verifySazOracleIdentity,sazRequest,sazEvidenceParameters} from ${JSON.stringify(moduleUrl)}; + const omitted=sazRequest('xbox-clear_nomp.saz','/tmp/sample.saz','baseline'); + assert.equal(Object.hasOwn(omitted,'record_ordinals'),false); + assert.deepEqual(sazEvidenceParameters(omitted,3),{capture_path:'/tmp/sample.saz',format:'saz',record_ordinals:[0,1,2],sensitive_value_count:0}); + assert.deepEqual(sazEvidenceParameters(sazRequest('xbox-clear_nomp.saz','/tmp/sample.saz','focused'),13).record_ordinals,[12,0,6]); + const empty={bytes:0,sha256:'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',content_base64:'',text:''}; + const raw=[12,0,6].map(ordinal=>{ + const session_id=String(ordinal+1).padStart(2,'0'); + return {ordinal,session_id,members:['raw/'+session_id+'_c.txt','raw/'+session_id+'_s.txt'], + request:{member:'raw/'+session_id+'_c.txt',method:'GET',url:'/',http_version:'HTTP/1.1',headers:[],body:empty}, + response:{member:'raw/'+session_id+'_s.txt',status:'200',reason:'OK',http_version:'HTTP/1.1',headers:[{name:'Content-Type',value:' text/plain'},{name:'content-type',value:''}],body:empty}}; + }); + const oracle={kind:'saz',artifact:'xbox-clear_nomp.saz',sha256:'3e808663a99cbb9160186f7419633248200fbc6863d2c897f5d4d9972dde0375',total_records:13,record_ordinals:[12,0,6],records:raw}; + const rows=raw.map(row=>({ordinal:row.ordinal,location:{kind:'archive-members',members:row.members}, + reported:{session_id:row.session_id,request:{...row.request,body:''},response:{...row.response,body:''}}, + binary_fields:['request','response'].map(side=>({pointer:'/'+side+'/body',representation:'producer-bytes',state:'retained',...empty}))})); + verifySazOracleIdentity('xbox-clear_nomp.saz',oracle); + const facts=verifySazRecords(rows,oracle);assert.deepEqual(facts.map(row=>row.ordinal),[12,0,6]); + assert.equal(facts[0].response.content_types.length,2); + assert.throws(()=>verifySazRecords(rows.slice(1),oracle),/coverage/); + assert.throws(()=>verifySazRecords(rows.toReversed(),oracle),/order/); + const changed=mutate=>{const copy=structuredClone(rows);mutate(copy);return copy;}; + assert.throws(()=>verifySazRecords(changed(copy=>copy[1].reported.session_id='1'),oracle),/identity/); + assert.throws(()=>verifySazRecords(changed(copy=>copy[0].reported.response.headers.pop()),oracle),/Content-Type/); + assert.throws(()=>verifySazRecords(changed(copy=>copy[0].binary_fields[1].content_base64='WA=='),oracle),/encoded body/); + assert.throws(()=>verifySazRecords(changed(copy=>copy[0].reported.response=null),oracle)); + assert.throws(()=>verifySazRecords(changed(copy=>copy[0].location.members=['raw/other_s.txt']),oracle),/coordinates/); + assert.throws(()=>verifySazOracleIdentity('xbox-clear_nomp.saz',{...oracle,sha256:'a'.repeat(64)}),/corpus/); + assert.throws(()=>sazRequest('__proto__','/tmp/sample.saz','focused'),/pinned/); + `; + // Checker-only synthesized rows; actual-provider verification is separate. + const result = spawnSync(process.execPath, ["--input-type=module", "-"], { + input: source, + encoding: "utf8", + }); + assert.ifError(result.error); + assert.equal(result.status, 0, result.stderr); +});