test(environment): build the real-agentenv lane's bundle with environment-build

The lane now runs the command users run: it builds the Grafana-shaped HAR
with environment-build and the packed checkout, checks the report, then
runs the bundle with agent-env as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Edgar ArakelyanandClaude Opus 5.5 committed 2026-10-23 16:48:07 -07:00
1 parent 045f14a3ae
commit cf7eb709e1
2 files changed
+45 -63

No files matched your search

+9 -7
View File
@@ -1900,13 +1900,15 @@ would remain after the pause, the verifier waits 90 seconds and runs once more,
recording `attempts: 2` and `retried_for` in the receipt. Any other failure
ends the lane at once.
The verifier writes the Grafana-shaped HAR the view tests own, inspects it and
projects the `environment-manifest` view with all three template rules through
the built CLI, packs the checkout with `npm pack`, exports the bundle with
`--package-tarball` so the image installs this checkout rather than the
registry package, and runs `agent-env run` on the bundle. It checks that the
export reports the planned file set with the inputs copied byte-exact and a
bundle served from the derived state, which serves fewer tools than the
The verifier writes the Grafana-shaped HAR the view tests own, packs the
checkout with `npm pack`, builds the bundle from the HAR with
`rea environment-build --package-tarball` through the built CLI, the command
users run, so the image installs this checkout rather than the registry
package, and runs `agent-env run` on the bundle. It checks that the build
reports the 28-record capture, the default template rules, the planned file set
with the shipped manifest naming the reported capture, a coverage summary whose
read and served write tools add up to the served tool count, and a bundle
served from the derived state, which serves fewer tools than the
manifest because write tools the capture did not confirm are not served, that the runner exits `0` with
`tasks/smoke.json` reported as passed and the verifier's description naming
that served tool count, and that no container exists
+36 -56
View File
@@ -29,11 +29,6 @@ const QUOTA_RETRY_PAUSE_MS = 90_000;
const RUN_BUDGET_MS = 22 * 60_000;
const FIRST_ATTEMPT_MS = 15 * 60_000;
const MIN_RETRY_MS = 5 * 60_000;
const TEMPLATE_RULES = [
"after-uid-literal",
"opaque-token",
"sibling-variance",
];
const execFileAsync = promisify(execFile);
// Both tools are caller-supplied absolute paths, as every real-* lane names its upstream.
@@ -86,43 +81,6 @@ try {
);
const harPath = join(runtime.path, "capture.har");
await writeFile(harPath, JSON.stringify(environmentManifestHar()));
const capture = await cli(
"inspect-web-network-capture",
harPath,
"har",
"--format",
"json",
);
assert.equal(capture.operation, "inspect_web_network_capture");
assert.equal(capture.normalized_result.total_records, 28);
cases++;
const viewPath = join(runtime.path, "view.json");
await writeFile(
viewPath,
JSON.stringify({
source: { kind: "inline", evidence: capture },
view: { kind: "environment-manifest", template_rules: TEMPLATE_RULES },
}),
);
const manifest = await cli(
"inspect-analysis-view",
viewPath,
"--format",
"json",
);
assert.equal(manifest.normalized_result.kind, "environment-manifest");
assert.equal(
manifest.normalized_result.parent_evidence_id,
capture.evidence_id,
);
assert.equal(manifest.normalized_result.name, "grafana-example");
const toolCount = manifest.normalized_result.tools.length;
assert.ok(toolCount > 0);
cases++;
const capturePath = join(runtime.path, "capture.json");
const manifestPath = join(runtime.path, "manifest.json");
await writeFile(capturePath, JSON.stringify(capture));
await writeFile(manifestPath, JSON.stringify(manifest));
// The lane proves this checkout, not the registry package: pack it and install the tarball in the image.
const { stdout: packed } = await execNpm(
["pack", "--ignore-scripts", "--json", "--pack-destination", runtime.path],
@@ -130,25 +88,43 @@ try {
);
const tarball = join(runtime.path, packedFilename(JSON.parse(packed)));
await access(tarball);
// The one-step build users run: it inspects the HAR, projects the manifest and writes the bundle.
const bundle = join(runtime.path, "grafana-example");
const exported = await cli(
"environment-export",
manifestPath,
"--evidence",
capturePath,
const built = await cli(
"environment-build",
harPath,
bundle,
"--package-tarball",
tarball,
);
assert.equal(exported.status, "exported");
assert.equal(exported.name, "grafana-example");
assert.equal(built.status, "built");
assert.equal(built.capture.operation, "inspect_web_network_capture");
assert.equal(built.capture.records, 28);
cases++;
assert.equal(built.name, "grafana-example");
const toolCount = built.manifest.tools;
assert.ok(toolCount > 0);
assert.deepEqual(built.manifest.template_rules, [
"after-uid-literal",
"opaque-token",
"sibling-variance",
"prefix-variance",
]);
cases++;
// The bundle serves the state its server derives from the capture, which leaves the
// manifest's write tools unserved; the export reports the count the bundle serves.
assert.equal(exported.state.served, true);
const servedCount = exported.tool_count;
// manifest's write tools unserved; the build reports the count the bundle serves.
assert.equal(built.state.served, true);
const servedCount = built.tool_count;
assert.ok(servedCount > 0 && servedCount < toolCount);
assert.equal(exported.package_source.kind, "tarball");
assert.deepEqual(exported.files.map((file) => file.path).sort(), [
// Every read tool and every served write tool is in the served catalog.
assert.equal(
built.coverage.reads.state +
built.coverage.reads.recorded +
built.coverage.writes.served.length,
servedCount,
);
assert.equal(built.package_source.kind, "tarball");
assert.deepEqual(built.files.map((file) => file.path).sort(), [
"README.md",
"artifacts/check_grafana-example/verify.py",
"envs/grafana-example/Dockerfile",
@@ -158,9 +134,13 @@ try {
"envs/grafana-example/rea-agents.tgz",
"tasks/smoke.json",
]);
assert.equal(
const shipped = JSON.parse(
await readFile(join(bundle, "envs/grafana-example/manifest.json"), "utf8"),
await readFile(manifestPath, "utf8"),
);
assert.equal(shipped.evidence_id, built.manifest.evidence_id);
assert.equal(
shipped.normalized_result.parent_evidence_id,
built.capture.evidence_id,
);
const dockerfile = await readFile(
join(bundle, "envs/grafana-example/Dockerfile"),