From cf7eb709e19f61bc5e1b23a2b905f2c31b2c3305 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 23 Oct 2026 16:48:07 -0700 Subject: [PATCH] test(environment): build the real-agentenv lane's bundle with environment-build The lane now runs the command users run: it builds the Grafana-shaped HAR with environment-build and the packed checkout, checks the report, then runs the bundle with agent-env as before. Co-Authored-By: Claude Opus 5.5 --- docs/testing.md | 16 +++-- scripts/verify-agentenv-environment.mjs | 92 ++++++++++--------------- 2 files changed, 45 insertions(+), 63 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 946db07bf1..72b1bb8115 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1900,13 +1900,15 @@ would remain after the pause, the verifier waits 90 seconds and runs once more, recording `attempts: 2` and `retried_for` in the receipt. Any other failure ends the lane at once. -The verifier writes the Grafana-shaped HAR the view tests own, inspects it and -projects the `environment-manifest` view with all three template rules through -the built CLI, packs the checkout with `npm pack`, exports the bundle with -`--package-tarball` so the image installs this checkout rather than the -registry package, and runs `agent-env run` on the bundle. It checks that the -export reports the planned file set with the inputs copied byte-exact and a -bundle served from the derived state, which serves fewer tools than the +The verifier writes the Grafana-shaped HAR the view tests own, packs the +checkout with `npm pack`, builds the bundle from the HAR with +`rea environment-build --package-tarball` through the built CLI, the command +users run, so the image installs this checkout rather than the registry +package, and runs `agent-env run` on the bundle. It checks that the build +reports the 28-record capture, the default template rules, the planned file set +with the shipped manifest naming the reported capture, a coverage summary whose +read and served write tools add up to the served tool count, and a bundle +served from the derived state, which serves fewer tools than the manifest because write tools the capture did not confirm are not served, that the runner exits `0` with `tasks/smoke.json` reported as passed and the verifier's description naming that served tool count, and that no container exists diff --git a/scripts/verify-agentenv-environment.mjs b/scripts/verify-agentenv-environment.mjs index d5d4b31954..60acf99f40 100644 --- a/scripts/verify-agentenv-environment.mjs +++ b/scripts/verify-agentenv-environment.mjs @@ -29,11 +29,6 @@ const QUOTA_RETRY_PAUSE_MS = 90_000; const RUN_BUDGET_MS = 22 * 60_000; const FIRST_ATTEMPT_MS = 15 * 60_000; const MIN_RETRY_MS = 5 * 60_000; -const TEMPLATE_RULES = [ - "after-uid-literal", - "opaque-token", - "sibling-variance", -]; const execFileAsync = promisify(execFile); // Both tools are caller-supplied absolute paths, as every real-* lane names its upstream. @@ -86,43 +81,6 @@ try { ); const harPath = join(runtime.path, "capture.har"); await writeFile(harPath, JSON.stringify(environmentManifestHar())); - const capture = await cli( - "inspect-web-network-capture", - harPath, - "har", - "--format", - "json", - ); - assert.equal(capture.operation, "inspect_web_network_capture"); - assert.equal(capture.normalized_result.total_records, 28); - cases++; - const viewPath = join(runtime.path, "view.json"); - await writeFile( - viewPath, - JSON.stringify({ - source: { kind: "inline", evidence: capture }, - view: { kind: "environment-manifest", template_rules: TEMPLATE_RULES }, - }), - ); - const manifest = await cli( - "inspect-analysis-view", - viewPath, - "--format", - "json", - ); - assert.equal(manifest.normalized_result.kind, "environment-manifest"); - assert.equal( - manifest.normalized_result.parent_evidence_id, - capture.evidence_id, - ); - assert.equal(manifest.normalized_result.name, "grafana-example"); - const toolCount = manifest.normalized_result.tools.length; - assert.ok(toolCount > 0); - cases++; - const capturePath = join(runtime.path, "capture.json"); - const manifestPath = join(runtime.path, "manifest.json"); - await writeFile(capturePath, JSON.stringify(capture)); - await writeFile(manifestPath, JSON.stringify(manifest)); // The lane proves this checkout, not the registry package: pack it and install the tarball in the image. const { stdout: packed } = await execNpm( ["pack", "--ignore-scripts", "--json", "--pack-destination", runtime.path], @@ -130,25 +88,43 @@ try { ); const tarball = join(runtime.path, packedFilename(JSON.parse(packed))); await access(tarball); + // The one-step build users run: it inspects the HAR, projects the manifest and writes the bundle. const bundle = join(runtime.path, "grafana-example"); - const exported = await cli( - "environment-export", - manifestPath, - "--evidence", - capturePath, + const built = await cli( + "environment-build", + harPath, bundle, "--package-tarball", tarball, ); - assert.equal(exported.status, "exported"); - assert.equal(exported.name, "grafana-example"); + assert.equal(built.status, "built"); + assert.equal(built.capture.operation, "inspect_web_network_capture"); + assert.equal(built.capture.records, 28); + cases++; + assert.equal(built.name, "grafana-example"); + const toolCount = built.manifest.tools; + assert.ok(toolCount > 0); + assert.deepEqual(built.manifest.template_rules, [ + "after-uid-literal", + "opaque-token", + "sibling-variance", + "prefix-variance", + ]); + cases++; // The bundle serves the state its server derives from the capture, which leaves the - // manifest's write tools unserved; the export reports the count the bundle serves. - assert.equal(exported.state.served, true); - const servedCount = exported.tool_count; + // manifest's write tools unserved; the build reports the count the bundle serves. + assert.equal(built.state.served, true); + const servedCount = built.tool_count; assert.ok(servedCount > 0 && servedCount < toolCount); - assert.equal(exported.package_source.kind, "tarball"); - assert.deepEqual(exported.files.map((file) => file.path).sort(), [ + // Every read tool and every served write tool is in the served catalog. + assert.equal( + built.coverage.reads.state + + built.coverage.reads.recorded + + built.coverage.writes.served.length, + servedCount, + ); + assert.equal(built.package_source.kind, "tarball"); + assert.deepEqual(built.files.map((file) => file.path).sort(), [ "README.md", "artifacts/check_grafana-example/verify.py", "envs/grafana-example/Dockerfile", @@ -158,9 +134,13 @@ try { "envs/grafana-example/rea-agents.tgz", "tasks/smoke.json", ]); - assert.equal( + const shipped = JSON.parse( await readFile(join(bundle, "envs/grafana-example/manifest.json"), "utf8"), - await readFile(manifestPath, "utf8"), + ); + assert.equal(shipped.evidence_id, built.manifest.evidence_id); + assert.equal( + shipped.normalized_result.parent_evidence_id, + built.capture.evidence_id, ); const dockerfile = await readFile( join(bundle, "envs/grafana-example/Dockerfile"),