test: require independently observed verifier outcomes

scripts/verify-il2cpp-metadata.mjs

@@ -79,6 +79,24 @@ await withVerifierCleanup(async () => {
       assert.equal(error.code, "resource_constraint");
       assert.equal(error.details.reported_limits.boundary, "mcp-response");
       assert.match(error.details.reported_limits.evidence_id, /^ev_/u);
+      const retained = (
+        await call("inspect_analysis_view", {
+          source: {
+            kind: "retained-evidence",
+            evidence_id: error.details.reported_limits.evidence_id,
+          },
+          view: { kind: "il2cpp-metadata", collection: "methods", limit: 16 },
+        })
+      ).normalized_result;
+      assert.ok(
+        report.methods.length > 0,
+        "Overflow retention needs a nonempty method oracle",
+      );
+      assert.deepEqual(
+        retained.items,
+        report.methods.slice(0, 16).map((row, index) => ({ ...row, index })),
+        "Failed complete delivery must retain its exact Evidence before reanalysis",
+      );
       console.log(
         `[ok] MCP complete delivery refused at the stdio budget (${error.details.reported_limits.response_bytes_at_least} bytes) with retained Evidence`,
       );

scripts/verify-native-ui.mjs

@@ -165,7 +165,9 @@ try {
       scenario.value.steps.some((step) => step.outcome !== "completed") ||
       !scenario.value.steps
         .at(-1)
-        ?.after?.nodes.some((node) => node.title === "REA fixture incremented")
+        ?.after?.nodes.some(
+          (node) => node.title === "REA fixture incremented 1",
+        )
     )
       throw new Error(
         `Source-owned AX scenario failed: ${JSON.stringify(scenario.value.steps.map(({ outcome, reason }) => ({ outcome, reason })))}`,
@@ -195,7 +197,12 @@ try {
       evidence.normalized_result?.steps?.some(
         (step) => step.outcome !== "completed",
       ) ||
-      evidence.normalized_result?.steps?.length !== 2
+      evidence.normalized_result?.steps?.length !== 2 ||
+      !evidence.normalized_result?.steps
+        ?.at(-1)
+        ?.after?.nodes.some(
+          (node) => node.title === "REA fixture incremented 2",
+        )
     )
       throw new Error(
         `Native UI CLI did not preserve the selected scenario: ${JSON.stringify(evidence.error ?? evidence.normalized_result?.steps?.map(({ outcome, reason }) => ({ outcome, reason })))}`,
@@ -232,7 +239,7 @@ try {
         !captured.steps
           .at(-1)
           ?.after?.nodes.some(
-            (node) => node.title === "REA fixture incremented",
+            (node) => node.title === "REA fixture incremented 3",
           )
       )
         throw new Error("MCP did not observe the selected fixture's AX action");

scripts/verify-qwen-client.mjs

@@ -310,7 +310,6 @@ await withVerifierCleanup(async () => {
             role: "assistant",
             content: "REA_CLIENT_COMPATIBILITY_OK",
           };
-          finalSeen = true;
         } else if (skill && !skillRequested) {
           skillRequested = true;
           message = call(skill.name, { skill: "reverse-engineer-anything" });
@@ -401,11 +400,13 @@ await withVerifierCleanup(async () => {
         stderr: join(lab, `${mode}.stderr`),
       },
     );
-    const initialized = response.stdout
+    const events = response.stdout
       .split("\n")
       .filter(Boolean)
-      .map((line) => JSON.parse(line))
-      .find((v) => v.type === "system" && v.subtype === "init");
+      .map((line) => JSON.parse(line));
+    const initialized = events.find(
+      (v) => v.type === "system" && v.subtype === "init",
+    );
     assert(initialized);
     const observed = initialized.tools
       .filter((n) => n.startsWith("mcp__rea__"))
@@ -417,7 +418,15 @@ await withVerifierCleanup(async () => {
       "Qwen must discover the complete REA catalog",
     );
     assert.equal(fixture.failure, undefined);
-    assert(finalSeen);
+    const result = events.findLast((event) => event.type === "result");
+    assert.equal(
+      result?.subtype,
+      "success",
+      "Qwen must emit a successful terminal result",
+    );
+    assert.equal(result.is_error, false);
+    assert.equal(result.result, "REA_CLIENT_COMPATIBILITY_OK");
+    finalSeen = true;
     if (mode === "call") {
       assert(evidenceSeen);
       assert(selectedSkillLoaded);

scripts/verify/managed/oracles.mjs

@@ -32,50 +32,61 @@ async function runManagedAppManifestSelfTest(context) {
   const members = inspectManagedMembersBytes(sample.bytes, sample.target);
   const method = members.methods[0];
   assert.ok(method);
-  const summary = await context.verifyManagedAppManifest(
-    {
-      label: "source-owned manifest verifier self-test",
-      target: {
-        path: sample.path,
-        sha256: sample.target.sha256,
-        mvid: artifact.module?.mvid,
-        assembly_name: artifact.assembly?.name,
-        runtime_family: artifact.classification.runtime_family,
-        managed_architecture: artifact.classification.managed_architecture,
+  const manifest = {
+    label: "source-owned manifest verifier self-test",
+    target: {
+      path: sample.path,
+      sha256: sample.target.sha256,
+      mvid: artifact.module?.mvid,
+      assembly_name: artifact.assembly?.name,
+      runtime_family: artifact.classification.runtime_family,
+      managed_architecture: artifact.classification.managed_architecture,
+    },
+    methods: [
+      {
+        label: "pinned-semantic-slice",
+        token: method.token,
+        signature_sha256: method.signature.raw_sha256,
+        il_size: method.body.il_size,
+        il_sha256: method.body.il_sha256,
+        normalized_il_sha256: method.body.normalized_il_sha256,
       },
-      methods: [
+    ],
+    application_graph: {
+      expected_node_kinds: [
+        "artifact",
+        "managed-assembly",
+        "managed-module",
+        "managed-type",
+        "managed-method",
+      ],
+      feature_traces: [
         {
-          label: "pinned-semantic-slice",
-          token: method.token,
-          signature_sha256: method.signature.raw_sha256,
-          il_size: method.body.il_size,
-          il_sha256: method.body.il_sha256,
-          normalized_il_sha256: method.body.normalized_il_sha256,
+          label: "trace pinned method by name",
+          method_token: method.token,
+          seed: "PinnedSemanticSlice",
+          match: "exact",
+          min_matched_seeds: 1,
         },
       ],
-      application_graph: {
-        expected_node_kinds: [
-          "artifact",
-          "managed-assembly",
-          "managed-module",
-          "managed-type",
-          "managed-method",
-        ],
-        feature_traces: [
-          {
-            label: "trace pinned method by name",
-            method_token: method.token,
-            seed: "PinnedSemanticSlice",
-            match: "exact",
-            min_matched_seeds: 1,
-          },
-        ],
-      },
     },
-    join(context.workspace, "source-owned-managed-app-manifest.json"),
+  };
+  const manifestPath = join(
+    context.workspace,
+    "source-owned-managed-app-manifest.json",
+  );
+  const summary = await context.verifyManagedAppManifest(
+    manifest,
+    manifestPath,
+  );
+  const rejectedMismatches = await verifyManifestRejections(
+    context,
+    manifest,
+    manifestPath,
   );
   return {
     verified: true,
+    rejected_mismatches: rejectedMismatches,
     assertions: summary.assertions,
     methods: summary.methods.length,
     applicationGraph: summary.application_graph ?? null,
@@ -83,6 +94,60 @@ async function runManagedAppManifestSelfTest(context) {
   };
 }
 
+async function verifyManifestRejections(context, manifest, manifestPath) {
+  const refusals = [
+    [
+      "target sha256",
+      ["target", "sha256"],
+      "0".repeat(64),
+      /target sha256 mismatch/u,
+    ],
+    [
+      "method token",
+      ["methods", 0, "token"],
+      "0x0600ffff",
+      /method 0x0600ffff was not returned/u,
+    ],
+    [
+      "signature sha256",
+      ["methods", 0, "signature_sha256"],
+      "0".repeat(64),
+      /signature sha256 mismatch/u,
+    ],
+    ["IL size", ["methods", 0, "il_size"], 999, /IL size mismatch/u],
+    [
+      "IL sha256",
+      ["methods", 0, "il_sha256"],
+      "0".repeat(64),
+      /IL sha256 mismatch/u,
+    ],
+    [
+      "normalized IL sha256",
+      ["methods", 0, "normalized_il_sha256"],
+      "0".repeat(64),
+      /normalized IL sha256 mismatch/u,
+    ],
+  ];
+  for (const [label, path, wrong, expected] of refusals) {
+    const rejected = structuredClone(manifest);
+    let owner = rejected;
+    for (const key of path.slice(0, -1)) owner = owner[key];
+    const key = path.at(-1);
+    assert.notEqual(
+      owner[key],
+      wrong,
+      `${label} control must differ from the valid manifest`,
+    );
+    owner[key] = wrong;
+    await assert.rejects(
+      () => context.verifyManagedAppManifest(rejected, manifestPath),
+      expected,
+      `Manifest verifier accepted an authored wrong ${label}`,
+    );
+  }
+  return refusals.map(([label]) => label);
+}
+
 async function runOptionalManagedAppManifest(context) {
   const manifestPath = process.env.REA_MANAGED_APP_MANIFEST_PATH;
   if (manifestPath === undefined) return null;