From c6c2a64af3ce17365580f3a437fc50dd4ea144ff Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 18 Oct 2026 01:25:29 -0700 Subject: [PATCH] feat(environment): export an environment manifest as an agent environment bundle Add rea environment-export --evidence [--package-tarball ] [--name ]: it reads and authenticates the manifest and capture pair once and writes an agent environment bundle folder (README with provenance and a rights note, envs/ with a Dockerfile that pins this exact rea version or installs a given tarball, the manifest and the Evidence as byte-identical copies, a Python verifier that checks replay through the MCP endpoint, and a smoke task) that the agentenv framework runs with agent-env run. Refuses a non-empty directory and a mismatched pair before writing. Adds the real-agentenv hosted lane, which packs the checkout, exports a bundle from the source-owned capture fixture and runs it on the runner's Docker with a pinned agentenv-framework, wired into the CI planner and its guard tests. Refs morluto/rea#4933 Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 12 +- .github/workflows/real-agentenv.yml | 52 +++ docs/agent-environments.md | 73 +++- docs/cli.md | 35 ++ .../environment-export/plan.md | 25 ++ .../environment-export/spec.md | 113 +++++++ .../environment-export/tasks.md | 12 + .../environment-export/verification.md | 226 +++++++++++++ docs/testing.md | 42 ++- package.json | 1 + scripts/ci/scopes.mjs | 19 ++ scripts/verify-agentenv-environment.mjs | 249 ++++++++++++++ .../environment/environmentBundle.test.ts | 182 ++++++++++ .../environment/environmentBundle.ts | 306 +++++++++++++++++ .../environment/environmentExport.test.ts | 224 ++++++++++++ .../environment/environmentExport.ts | 318 ++++++++++++++++++ .../environment/environmentSource.ts | 34 +- src/cli/environmentCommands.ts | 127 ++++++- src/cliCommandNames.ts | 1 + .../boundary/cli/environmentExportCli.test.ts | 162 +++++++++ .../boundary/filesystem/ciChangeScope.test.ts | 21 +- .../filesystem/ciRequiredLanes.test.ts | 2 + .../environmentBundlePackagePin.test.ts | 40 +++ 23 files changed, 2250 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/real-agentenv.yml create mode 100644 docs/specs/agent-environments/environment-export/plan.md create mode 100644 docs/specs/agent-environments/environment-export/spec.md create mode 100644 docs/specs/agent-environments/environment-export/tasks.md create mode 100644 docs/specs/agent-environments/environment-export/verification.md create mode 100644 scripts/verify-agentenv-environment.mjs create mode 100644 src/application/environment/environmentBundle.test.ts create mode 100644 src/application/environment/environmentBundle.ts create mode 100644 src/application/environment/environmentExport.test.ts create mode 100644 src/application/environment/environmentExport.ts create mode 100644 tests/boundary/cli/environmentExportCli.test.ts create mode 100644 tests/conformance/package/environmentBundlePackagePin.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 384e54d303..fffe2de4e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: web_network: ${{ steps.plan.outputs.web_network }} web_runtime: ${{ steps.plan.outputs.web_runtime }} web_source_map: ${{ steps.plan.outputs.web_source_map }} + agentenv: ${{ steps.plan.outputs.agentenv }} nativeaot: ${{ steps.plan.outputs.nativeaot }} ghidra_bridge: ${{ steps.plan.outputs.ghidra_bridge }} website: ${{ steps.plan.outputs.website }} @@ -302,7 +303,8 @@ jobs: needs.changes.outputs.recorded_crash == 'true' || needs.changes.outputs.web_network == 'true' || needs.changes.outputs.web_runtime == 'true' || - needs.changes.outputs.web_source_map == 'true' + needs.changes.outputs.web_source_map == 'true' || + needs.changes.outputs.agentenv == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -909,6 +911,13 @@ jobs: with: build_cache_artifact: rea-build-cache + agentenv: + needs: [changes, static, build] + if: needs.changes.outputs.agentenv == 'true' + uses: ./.github/workflows/real-agentenv.yml + with: + build_cache_artifact: rea-build-cache + nativeaot: needs: [changes, static] if: needs.changes.outputs.nativeaot == 'true' @@ -953,6 +962,7 @@ jobs: web_network, web_runtime, web_source_map, + agentenv, nativeaot, ghidra_bridge, website, diff --git a/.github/workflows/real-agentenv.yml b/.github/workflows/real-agentenv.yml new file mode 100644 index 0000000000..5a37629c95 --- /dev/null +++ b/.github/workflows/real-agentenv.yml @@ -0,0 +1,52 @@ +name: Real agentenv environment bundle + +on: + workflow_dispatch: + workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" + +permissions: + contents: read + +env: + HUSKY: "0" + NODE_OPTIONS: --max-old-space-size=768 + PIP_DISABLE_PIP_VERSION_CHECK: "1" + +jobs: + real-agentenv: + name: Export and run an environment bundle with the agentenv runner + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache + - run: npm ci + - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + - name: Acquire the pinned agentenv runner in isolated runner storage and check host Docker + shell: bash + run: | + runner_root="${RUNNER_TEMP}/rea-agentenv-runner" + python -m venv "${runner_root}" + "${runner_root}/bin/python" -m pip install --require-virtualenv --no-cache-dir 'agentenv-framework==0.9.1334' + "${runner_root}/bin/agent-env" --version + docker version --format '{{.Server.Version}}' + echo "REA_AGENT_ENV_COMMAND=${runner_root}/bin/agent-env" >> "${GITHUB_ENV}" + echo "REA_DOCKER_COMMAND=$(command -v docker)" >> "${GITHUB_ENV}" + - name: Export a bundle from the source-owned capture and run it with the agentenv runner + run: npm run verify:agentenv:environment diff --git a/docs/agent-environments.md b/docs/agent-environments.md index c88300a689..1e7b5309b9 100644 --- a/docs/agent-environments.md +++ b/docs/agent-environments.md @@ -4,7 +4,8 @@ Use `rea environment-serve` to serve an environment manifest as an agent environment: an MCP server whose tools answer from the records of one retained web network capture, plus the environment card the agentenv framework reads to discover it. Nothing is fetched from the recorded application; every answer is -retained Evidence. +retained Evidence. `rea environment-export` writes the same pair as a bundle +folder `agent-env run` builds and runs; see [Export a bundle](#export-a-bundle). ```sh rea inspect-web-network-capture /absolute/session.har har --format json > capture.json @@ -170,6 +171,76 @@ schema does not carry. `[]` and `null`: a replay environment declares no extensions, no data-plane operations and no children. +## Export a bundle + +`rea environment-export` writes the same pair as a bundle folder the agentenv +framework runs directly, shaped like its own echo example: + +```sh +rea environment-export manifest.json --evidence capture.json ./grafana-example --json +agent-env run ./grafana-example +``` + +```text +grafana-example/ + README.md provenance, replay limits, run command, rights note + envs/grafana-example/Dockerfile pinned image: node:24.18.0-bookworm-slim + rea-agents@ + envs/grafana-example/env.toml environment_name = "grafana-example" + envs/grafana-example/manifest.json byte-exact copy of manifest.json + envs/grafana-example/evidence.json byte-exact copy of capture.json + artifacts/check_grafana-example/verify.py the replay verifier + tasks/smoke.json deploy_env (local sandbox), then env_outcome_verifier +``` + +The command reads each file once, then authenticates and cross-checks the +pair as `environment-serve` does, so a manifest projected from another capture +is refused with the same `invalid_request` before anything is written, and the +bytes it copies are the bytes it authenticated. The bundle directory must be +absent or empty; an occupied directory is refused and nothing in it changes. +`--name` replaces the manifest's kebab-case name as the env folder and env id; +`--package-tarball ` copies a packed `rea-agents` tarball into the env +folder and makes the Dockerfile install it instead of the registry package and +remove it in the same layer, which is how a checkout rather than a release is +run. The command prints the +written files with their sizes and the pinned package source. + +The Dockerfile sets `MCP_HOST=0.0.0.0` and `MCP_PORT=18765`, the port the +framework maps for an MCP server env; the framework's own deploy supplies +`MCP_HOST`, `MCP_PORT` and `ENVIRONMENT_NAME`, so the served card carries the +env's name. It installs Debian's `python3` because the framework's local +sandbox probes a container's readiness with `python3` inside it, which the slim +Node image lacks. The image runs as the image's non-root `node` user and starts +`rea environment-serve /env/manifest.json --evidence /env/evidence.json`. The +`env.toml` names the environment because the framework otherwise reads the +name from Python sources the image does not have. + +`verify.py` is the framework's `async def verify(mcp_url)` shape over its +Python MCP client: it lists the tools and checks the count equals the +manifest's, calls the first `GET` tool without path parameters with no +arguments and checks the replayed `status` equals the recorded status of that +tool's first cited record, and returns one `replay` result whose description +reports both checks. A manifest with no such tool yields a verifier that checks +the count only and says so. `tasks/smoke.json` deploys the env on the local +sandbox and runs that verifier against the env's MCP endpoint; the first run +also builds the framework's own infrastructure, and the run tears everything +down when it ends. + +The README written into the bundle records the capture artifact's SHA-256, the +capture and manifest Evidence ids, the manifest view digest, the tool count and +the package source, and states that the recorded content belongs to the +captured application and must not be published without the rights to it. + +## Compatibility verification + +`npm run verify:agentenv:environment` is the hosted `real-agentenv` lane +described in [testing](testing.md#exported-environment-bundles). It projects +the Grafana-shaped source-owned capture through the built CLI, packs the +checkout, exports a bundle with `--package-tarball`, runs it with the pinned +`agentenv-framework` release on the runner's Docker and requires the smoke task +to pass with no container left behind. It proves that one exported bundle runs +on one framework release; it does not claim any hosted platform or other +framework versions. + ## Resource and lifecycle boundaries Both Evidence files are read once at startup through the bounded JSON input diff --git a/docs/cli.md b/docs/cli.md index 9dd12b8d1b..141c405fd9 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -355,6 +355,33 @@ like a blank `MCP_HOST`. The bound URL is written to stderr as one JSON log line [agent environments](agent-environments.md) for what replay returns and does not establish. +## Export an environment bundle + +`rea environment-export --evidence +[--package-tarball ] [--name ]` writes the same manifest and +capture pair as a bundle folder the agentenv framework runs with +`agent-env run `. It reads each file once, authenticates and cross-checks +the pair with the same checks as `environment-serve`, copies the bytes it +authenticated, refuses to write into a directory that exists and is not empty, +and never overwrites a file. + +```sh +rea environment-export manifest.json --evidence capture.json ./grafana-example --json +``` + +The folder holds a `README.md` with the bundle's provenance, an env folder +`envs//` with a `Dockerfile` that installs this exact `rea-agents` +version and serves the copied `manifest.json` and `evidence.json` with +`environment-serve`, a Python verifier under `artifacts/check_/` and the +smoke task `tasks/smoke.json`. `--package-tarball` copies a packed `rea-agents` +tarball into the env folder and installs it instead of the registry package, +which is how the hosted lane proves a checkout rather than a release. `--name` +replaces the manifest's kebab-case name as the env folder and env id. The +command prints one JSON document listing the written files with their sizes +and the pinned package source. See +[agent environments](agent-environments.md#export-a-bundle) for the folder +layout and what the smoke task checks. + ## Output and exit status `rea mcp doctor --json` checks production MCP startup and reports child logs in @@ -400,6 +427,14 @@ and exits `130` or `143` through the shared cancellation handler below. Refusals before binding (unreadable files, non-Evidence JSON, a manifest whose parent is not the supplied evidence, an unbindable port) print a structured error and exit `1`, as does a listener that fails after binding. +`rea environment-export` is one-shot: it prints the written files and exits +`0`, or prints a structured `invalid_request` error and exits `1` for the same +pair refusals, a non-empty or file-occupied directory, a missing package +tarball or a name that is not a kebab-case label, writing nothing in each case. +A write that fails after the first file (a permission or disk-space failure +on a later path) propagates as the host's failure and leaves the partial +folder in place for inspection; remove it before retrying, since the next +attempt refuses the non-empty directory. A `cancelled` result does not by itself determine the exit status; it depends on the command's signal handling. Commands routed through the shared diff --git a/docs/specs/agent-environments/environment-export/plan.md b/docs/specs/agent-environments/environment-export/plan.md new file mode 100644 index 0000000000..9e927e128d --- /dev/null +++ b/docs/specs/agent-environments/environment-export/plan.md @@ -0,0 +1,25 @@ +# Implementation plan + +1. Extend `src/application/environment/environmentSource.ts` so + `loadEnvironmentSource` takes the reporting operation and retains the + capture artifact's SHA-256, letting the export reuse the serve command's + authentication and cross-check unchanged. +2. Add `environmentBundle.ts` to plan the bundle from the source alone: the + name rule, the Dockerfile for the registry and tarball package sources, the + verifier, the smoke task, the README and the copy targets. +3. Add `environmentExport.ts` to read each input once through the + regular-file reader, refuse an occupied directory, a missing tarball or an + unreadable input before writing, then write the planned files with + exclusive creation and copy the authenticated bytes and the tarball + byte-exact. +4. Register `environment-export` in `src/cli/environmentCommands.ts` beside + `environment-serve`, sharing the argument descriptions, and in + `CLI_COMMANDS`. +5. Add the `agentenv` scope, `real-agentenv.yml`, the package script and + `scripts/verify-agentenv-environment.mjs`, wire `ci.yml` and extend the + `ciRequiredLanes` and `ciChangeScope` tables. +6. Add services, conformance and cli-boundary tests; document the command in + `docs/cli.md`, the bundle and lane in `docs/agent-environments.md` and + `docs/testing.md`; record the spec. + +No new dependency is introduced; the CLI inventory gains one command. diff --git a/docs/specs/agent-environments/environment-export/spec.md b/docs/specs/agent-environments/environment-export/spec.md new file mode 100644 index 0000000000..09c79b7115 --- /dev/null +++ b/docs/specs/agent-environments/environment-export/spec.md @@ -0,0 +1,113 @@ +# Export an environment manifest as an agent environment bundle + +Issue: https://github.com/morluto/rea/issues/4933 + +## Outcome + +`rea environment-export --evidence +[--package-tarball ] [--name ]` writes the manifest and +capture pair that `environment-serve` serves as a bundle folder the agentenv +framework runs with `agent-env run `: a README with provenance, an env +folder whose Dockerfile installs this exact `rea-agents` version and serves the +copied pair, a Python replay verifier and a smoke task. The hosted +`real-agentenv` lane exports such a bundle from the source-owned Grafana-shaped +capture through the built CLI, packs the checkout into the image and runs it +with the pinned framework release on the runner's Docker. This increment +completes the agent-environment work of the issue; serving came in the +previous increment. + +## Requirements + +- Read each of the two files once through the regular-file reader, parse it + from those bytes and authenticate and cross-check the pair with + `loadEnvironmentSource` as `environment-serve` does, so the bytes copied into + the bundle are the bytes that were authenticated; every refusal reports + operation `environment-export`, and a manifest projected from a different + capture is refused before anything is written. +- Refuse a bundle directory that exists and is not empty, or that names a file, + with an `invalid_request` whose issue path is `directory`; an absent or empty + directory is accepted. Write every file with the exclusive-create flag so + nothing is ever overwritten. +- Write `README.md`, `envs//Dockerfile`, `envs//env.toml`, + `envs//manifest.json` and `envs//evidence.json` as byte-exact + copies of the inputs, `artifacts/check_/verify.py` and + `tasks/smoke.json`, where `` defaults to the manifest's kebab-case name + and `--name` must be a kebab-case label of at most 64 characters. +- The Dockerfile reads `FROM node:24.18.0-bookworm-slim`, sets + `MCP_HOST=0.0.0.0` and `MCP_PORT=18765`, installs Debian's `python3` for the + framework's readiness probe, installs + `rea-agents@` from the registry or, with + `--package-tarball`, copies the tarball as `rea-agents.tgz`, installs it + from `/tmp/` and removes it in the same layer, copies the pair to `/env/`, + switches to the image's `node` user + and runs `rea environment-serve /env/manifest.json --evidence + /env/evidence.json`. +- `verify.py` mirrors the framework's echo example: `async def verify(mcp_url)` + over `mcp.client.streamable_http`, lists the tools and checks the count + equals the manifest's, calls the first `GET` tool without path parameters + with no arguments and checks the replayed `status` equals the recorded status + of that tool's first cited record, prints the description and returns + `[{"id": "replay", "result": , "description": }]`; with no + such tool it checks the count only and says so. +- `tasks/smoke.json` is `deploy_env` of `` on the local sandbox followed + by `env_outcome_verifier` with `file_artifact_id` `check_` and + `verifier_id` `replay`. The README records the capture artifact SHA-256, both + Evidence ids, the manifest view digest, the tool count and the package + source, states what replay does and does not establish, names the run + command as `agent-env run .` from the bundle root (the folder it sits in, + whatever the caller named it), and carries the rights note. +- Print one JSON document with `status: "exported"`, the directory, name, tool + count, both Evidence ids, the pinned `package_source` and the written files + with sizes. Register the flat `environment-export` command in the CLI + inventory; no MCP tool is added. +- Add the `agentenv` CI scope, the reusable `real-agentenv.yml` workflow + (`workflow_dispatch` and `workflow_call` with `build_cache_artifact`, + `ubuntu-24.04`, SHA-pinned actions, `setup-python` 3.12, a virtual + environment with `agentenv-framework==0.9.1334`, the runner's Docker, 30 + minutes), the `verify:agentenv:environment` script and + `scripts/verify-agentenv-environment.mjs`, which requires absolute + `REA_AGENT_ENV_COMMAND` and `REA_DOCKER_COMMAND` like every other lane + verifier, runs `agent-env run --sandbox local` bounded to 15 minutes + and resolves nothing from `PATH`; wire `ci.yml` at the four spots and extend + the two CI guard tests. + +## Acceptance and limits + +Services tests cover the planned file set, the Dockerfile for both package +sources, the verifier text, the smoke task, the README, the name override and +its refusals, writing into an absent and an empty directory with byte-exact +copies and correct sizes, the tarball copy, and the refusals of a non-empty +directory, a file path, a missing tarball and an inline manifest. The +conformance test pins the registry install to the `package.json` version. The +cli-boundary test writes the folder from the fixture pair through the built +CLI, refuses a non-empty directory leaving it unchanged, and refuses a +mismatched pair without creating the directory. The CI guard tests enumerate +the new consumer, route and classification cases. The local end-to-end run +recorded in `verification.md` is the proof that one exported bundle runs under +`agent-env run` with the smoke task passed and no container left behind. + +The export copies bytes and never inspects the tarball; it does not build or +run the image itself. Its writes are not transactional: a host failure after +the first file leaves the partial folder for inspection, as `docs/cli.md` +states. The verifier checks one replayed status and the tool count, not every +tool. The lane passes `--sandbox local` as designed although +`tasks/smoke.json` already pins `sandbox_type: local` on its deploy step, so +the flag restates what the task fixes. The lane verifier imports the +Grafana-shaped HAR builder from `tests/fixtures/historicalHar.ts` through +Node's type stripping rather than carrying a second copy under `scripts/`; the +`agentenv` scope lists that fixture so a change to it runs the lane. The lane +proves one framework release on one hosted runner and claims nothing about +any hosted platform. Three additions to the +designed Dockerfile and file set were needed by the framework, each found by +the local run: `MCP_PORT=18765` as the listener default for the port the +framework maps, Debian's `python3` because the framework's local sandbox +probes readiness with `python3 -c "import socket; …"` inside the container and +the slim Node image has none, and `env.toml` naming the environment, because +the framework otherwise reads the name from Python sources the Node image does +not have. + +## Deferred issue criteria + +This increment closes the agent-environment scope of #4933 together with the +previous three. Running the lane on the maintainer's CI and any follow-up on +hosted platforms are outside this PR. diff --git a/docs/specs/agent-environments/environment-export/tasks.md b/docs/specs/agent-environments/environment-export/tasks.md new file mode 100644 index 0000000000..f1a0a4a211 --- /dev/null +++ b/docs/specs/agent-environments/environment-export/tasks.md @@ -0,0 +1,12 @@ +# Tasks + +- [x] Record the export contract, requirements and the lane. +- [x] Thread the operation name and capture digest through the source loader. +- [x] Plan the bundle: name rule, Dockerfile for both package sources, verifier, smoke task, README. +- [x] Write the bundle with directory, tarball and input refusals and exclusive creation. +- [x] Register the flat CLI command with described arguments and options. +- [x] Add the agentenv scope, workflow, package script and verifier; wire ci.yml; extend the guard tests. +- [x] Add services, conformance and cli-boundary coverage. +- [x] Document the command, the bundle and the lane; record the spec. +- [x] Run static, formatting, duplication, documentation and test gates and the local end-to-end run. +- [x] Repair the review findings (README run command, absolute tool paths, single read, tarball cleanup) and re-run the gates and the end-to-end run. diff --git a/docs/specs/agent-environments/environment-export/verification.md b/docs/specs/agent-environments/environment-export/verification.md new file mode 100644 index 0000000000..243d9a9b0c --- /dev/null +++ b/docs/specs/agent-environments/environment-export/verification.md @@ -0,0 +1,226 @@ +# Verification + +Verified base: 24574576e4fa74ff56eda965ae287555b7552c03, the tip of the environment-serve increment on fetched main b83b9caf (this increment builds on it). + +The worktree carries the manifest-view, streamable-HTTP and serve increments +as commits on upstream main b83b9caf; this increment's changes are uncommitted +edits on top. Every check below ran as written on this tree after the review +repairs; nothing is reported that did not run. Timings are macOS +`/usr/bin/time -l` command accounting (elapsed, user, system seconds and +maximum resident set size of the command and the children it waited for) on +macOS 26.5.1 with Node 22.23.1, npm 12.0.2, Docker 29.5.3 and a Python 3.12.12 +virtual environment holding `agentenv-framework 0.9.1334`. + +## Review findings addressed + +An adversarial review of the first pass found two should-fix findings and +seven nits; each was repaired at its owning boundary before the gates below +ran. + +- The bundle README told the reader to run `agent-env run `, which only + works from the bundle's parent when the folder happens to carry the env's + name. The README now says `agent-env run .` from the bundle root and names + the folder's path for any other working directory; the services test + asserts the new block and the absence of the old command. +- The lane verifier resolved `agent-env` and `docker` from `PATH` when the + `REA_*` variables were unset. It now requires absolute + `REA_AGENT_ENV_COMMAND` and `REA_DOCKER_COMMAND`, throwing the standard + actionable message naming the npm script and that nothing is installed, and + checks both paths exist; the workflow exports `REA_DOCKER_COMMAND` from + `command -v docker` beside the virtual environment's `agent-env`; + `docs/testing.md` names both variables and drops the `PATH` default. No + Linux preflight was added, so the lane stays runnable on a macOS host. +- The brittle `assert.doesNotMatch(stdout, /failed/)` is gone; exit code `0` + and the `tasks/smoke.json v: passed` summary already decide the case. +- The inner `agent-env run` timeout is 15 minutes inside the 30-minute job, + so a hung run ends with the verifier's failed receipt rather than the + runner's kill; `docs/testing.md` says so. +- The pair was read twice: parsed once for authentication and re-read from + disk for the byte-exact copies, so a file replaced between the reads was + copied unchecked. `readEnvironmentExportInputs` now reads each file once + through the regular-file reader, parses the value from those bytes with the + shared UTF-8 JSON parser, and the command passes the same buffers to the + export, which copies them; the refusals of inline JSON, a directory and a + file that is not JSON are typed by role, and the spec, plan and both guides + state the single read. +- The tarball Dockerfile left the 6 MB `rea-agents.tgz` in `/tmp` of the + image; the install layer now removes it (`&& rm /tmp/rea-agents.tgz`), + confirmed by listing `/tmp` in the built image below. +- The verifier passes `--sandbox local` as the design specifies; the spec + records that the task's `sandbox_type: local` already pins the provider and + the flag restates it. +- The verifier's import of the HAR builder from `tests/fixtures/` through + Node's type stripping is kept and recorded in the spec's limits with the + reason (no second copy under `scripts/`; the scope lists the fixture). +- Writes are not transactional; `docs/cli.md` now states that a host failure + after the first file leaves the partial folder for inspection and that the + next attempt refuses the non-empty directory, and the spec's limits say so. + +## Setup + +`npm ci` completed (exit 0; 373 top-level entries). `/usr/bin/time -l npm run +build` passed with 3 tasks successful in 7.60 s elapsed (14.81 s user, 2.69 s +sys, 3013410816 bytes); `/usr/bin/time -l npm run test:prepare` passed with 6 +tasks successful, 3 cached, in 1.63 s (3.47 s user, 0.81 s sys, 400752640 +bytes). + +## Static gates + +Every gate ran on the final tree after the last source edit, sequentially: +`npm run check:fast` (typecheck and lint with module boundaries; 2 tasks +successful; 5.28 s elapsed, 23.22 s user, 2.93 s sys, 4564680704 bytes); +`npm run format` rewrote nothing outside this increment's files ("Finished in +635ms on 4077 files"), and `npm run format:check` passed ("All matched files +use the correct format"; 0.87 s, 4.30 s user, 0.64 s sys, 224051200 bytes); +`npx knip` reported no findings (3.49 s, 5.04 s user, 0.72 s sys, 805191680 +bytes); `npm run verify:test-discovery` reported "Discovered all 1597 test +files exactly once" (0.55 s, 0.38 s user, 0.14 s sys, 141492224 bytes); +`npm run check:ci:tasks` (typecheck, lint:built, format:check, knip, +metadata:check, jscpd, scan:todos) passed with 8 tasks successful, lint "3764 +files, 0 errors, 1690 warnings" and jscpd "3171 files, 2587 clones, 3.34% +duplicated lines" under the 3.5% threshold (4.89 s elapsed, 15.19 s user, +3.86 s sys, 874627072 bytes); `npm run docs:check` passed including the README +consistency test (6 tasks, 5 tests; 4.00 s elapsed, 3.86 s user, 0.94 s sys, +290111488 bytes). `actionlint 1.7.7` (`-shellcheck= -pyflakes=`, from the +review's scratch virtual environment) reported no findings on +`.github/workflows/real-agentenv.yml` and `.github/workflows/ci.yml`. A +case-insensitive grep for the forbidden words and marker comments over the 23 +new or modified files matched only the `scan:todos` script name in +`package.json` and this file. + +## Test projects + +Source group: `/usr/bin/time -l npx vitest run --project services --project +conformance` passed 122 files, 1524 tests in 11.08 s elapsed, 23.10 s user, +2.24 s sys, 1631240192 bytes. The increment's files are +`environmentBundle.test.ts` (7 tests: the planned file set and copy targets, +the registry Dockerfile pinned to `PACKAGE_METADATA.version` with the +readiness-probe layer, the tarball Dockerfile installing and removing +`/tmp/rea-agents.tgz`, the verifier text naming `get_annotations` as the +first parameterless `GET` tool with recorded status 200 of record 25, the +smoke task, the README's provenance, `agent-env run .` block and rights note, +the name override and six refused names), `environmentExport.test.ts` (6 +tests: the reader returning each file's bytes and parsed value, its refusals +of inline JSON, a directory and malformed JSON by role, writing into an absent +directory with the read bytes copied unchanged and reported sizes, the +tarball copy, the refusals of a non-empty directory, a file path and a missing +tarball with nothing written, and an existing empty directory accepted), +`environmentSource.test.ts` (4 tests; the loader takes the reporting +operation) and the conformance test `environmentBundlePackagePin.test.ts` (1 +test: the registry install line ends with `rea-agents@` +and never uses `latest` or a range). + +Built group after `npm run test:prepare`: `/usr/bin/time -l npx vitest run +--project cli-boundary --project boundary` over `environmentExportCli` (3), +`ciChangeScope` (228), `ciRequiredLanes` (17), `workflowActionPinning` (5), +`ciCachePolicy` (6), `cliToolCoverage` (1), `cliStartupImports` (5) and +`ciTestProjects` (11) passed 8 files, 276 tests in 432.80 s elapsed, 303.47 s +user, 92.63 s sys, 422412288 bytes; `tests/boundary/filesystem/productCatalog` +passed separately (10 tests, 5.32 s). The cli-boundary file writes the folder +from the fixture pair through the built CLI and checks the seven written +paths, the byte-exact copies and the Dockerfile's first line and `CMD`; +refuses a non-empty directory leaving its one file in place; and refuses a +mismatched pair without creating the directory. + +## Local end-to-end run + +With the built CLI in this worktree, from a fresh scratch directory: +`inspect-web-network-capture grafana-like.har har --format json --json` wrote +capture Evidence `ev_4ee4759b…` (28 records); `inspect-analysis-view` with the +inline source and all three template rules wrote manifest Evidence +`ev_6553a8e5…` naming that parent, `grafana-example`, 19 tools, view digest +`8d1eb3ee…`. `npm pack --ignore-scripts` of the checkout produced +`rea-agents-6.6.0.tgz` (6039238 bytes). `environment-export manifest.json +--evidence capture.json /grafana-example --package-tarball --json` +exited 0 and printed `status: exported`, `tool_count: 19`, `package_source: +{kind: tarball}` and eight files: `README.md` (2076 bytes), +`envs/grafana-example/env.toml` (37), `envs/grafana-example/Dockerfile` (441), +`artifacts/check_grafana-example/verify.py` (1534), `tasks/smoke.json` (291), +`envs/grafana-example/manifest.json` (15701), +`envs/grafana-example/evidence.json` (58296) and +`envs/grafana-example/rea-agents.tgz` (6039238); `cmp` confirmed the two +Evidence copies and the tarball byte-identical to the inputs. The Dockerfile's +install line reads `RUN npm install -g --ignore-scripts --no-fund --no-audit +/tmp/rea-agents.tgz && rm /tmp/rea-agents.tgz`, and the README's run section +is the `agent-env run .` block. + +`/usr/bin/time -l agent-env run /grafana-example` exited 0 in 43.66 s +elapsed (5.19 s user, 3.26 s sys, 129318912 bytes), output + +```text +[tasks/smoke.json] step 1/2 env done in 25.1s +[tasks/smoke.json] step 2/2 replay (env_outcome_verifier) +replay: listed 19 tools, expected 19; get_annotations replayed status 200, recorded 200 +[tasks/smoke.json] passed in 25.1s + +Tasks: + tasks/smoke.json v1: passed (replay: 1), 25.1s, instance @local/…/grafana-example/smoke-nwvvojs7 + +Tore down 1 sandbox. +``` + +`docker ps -aq` before and after the run listed the same three pre-existing +containers (a Grafana container, an exited container and the framework's +registry from earlier sessions, unrelated to this run); no container was +added, and the image count rose from 19 to 20 for the built env image +`5a5b3bf62aae`. `docker run --rm --entrypoint sh 5a5b3bf62aae -c 'ls -la +/tmp; rea --version; id -un'` listed only `node-compile-cache` in `/tmp` (no +`rea-agents.tgz`), printed `6.6.0` and ran as `node`. + +`REA_AGENT_ENV_COMMAND=/bin/agent-env REA_DOCKER_COMMAND=$(command -v +docker) /usr/bin/time -l node scripts/verify-agentenv-environment.mjs` exited +0 in 37.24 s elapsed (10.42 s user, 4.45 s sys, 268533760 bytes) and printed +the receipt `status: passed`, `public_cases: 5`, `upstream: agentenv-framework +0.9.1334`, `runner: darwin arm64; host Docker`, `docker_before: {containers: +3, images: 20}`, `agent_env_run: {exit_code: 0, seconds: 32}` with the same +passed summary in its stdout tail, `docker_after: {containers: 3, images: 21, +new_containers: []}` and `process_lineage.status: verified` with no +descendants. Without `REA_DOCKER_COMMAND` the verifier exited 1 before +touching anything with "verify:agentenv:environment requires absolute +REA_DOCKER_COMMAND; see docs/testing.md. No dependencies are installed by this +verifier."; with a relative `REA_AGENT_ENV_COMMAND=agent-env` it exited 1 with +the same sentence naming that variable. + +Refusals through the built CLI: exporting into the already written bundle +directory exited 1 with `invalid_request`, issue path `["directory"]` and the +message "is not empty (4 entries); choose an absent or empty directory"; +exporting the manifest against the capture of one selected record +(`--record 0`, `ev_3249d233…`) exited 1 with `invalid_request`, issue path +`["evidence", "evidence_id"]` and "The manifest was projected from a +different capture than the supplied evidence file.", creating no directory; +a truncated `broken.json` as the evidence exited 1 with `invalid_request`, +issue path `["evidence"]`, reason `invalid_format` and "broken.json is not +valid JSON: Unexpected end of JSON input", creating no directory. An export +without `--package-tarball` wrote a Dockerfile whose install line is +`RUN npm install -g --ignore-scripts --no-fund --no-audit rea-agents@6.6.0`, +the `package.json` version, with no tarball beside it; the tarball variant +copies `rea-agents.tgz` and installs and removes `/tmp/rea-agents.tgz`. + +## Independent review + +The adversarial review of the first pass is the independent review of this +increment; its findings and their repairs are listed above. One further +observation surfaced while repairing them: the single-read helper routes the +export's input refusals through the export module rather than the CLI's +shared JSON-input parser, so the inline-JSON refusal is now a typed +`invalid_request` on the `manifest` or `evidence` issue path instead of the +parser's `input_path`/`input_reason` envelope; the cli-boundary tests did not +depend on that envelope. + +## Scope and measurement limits + +The framework readiness probe, the port variable default and the `env.toml` +name were found by running the bundle in the first pass, not by reading a +published contract; they are recorded in the spec as the three additions to +the designed file set. The local run and the verifier used the host's Docker +with the framework's gateway and service-database images already built from +earlier sessions, so the "first run also builds the infrastructure" path was +not timed here. The verifier's container check compares `docker ps -aq` +before and after; images built by the run remain by design and are only +counted. `npm pack --json` has the npm 12 object shape on this host and the +npm 11 array shape on the hosted runner; the verifier accepts both but only +the former was exercised locally. The hosted `real-agentenv` workflow was not +dispatched from this tree. The built projects ran on this host's Node 22.23.1 +rather than the pinned 24.18.0. The `/usr/bin/time -l` figures account for +each command and the children it waited for; Docker's own work runs in the +daemon and is outside them. Windows was not run. diff --git a/docs/testing.md b/docs/testing.md index 6a0b7984a9..75b3b05b44 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1168,6 +1168,16 @@ identifiable; unknown or shared script changes select everything. Android build-script edits retain the Linux suite and workflow validation; real Android verification stays manual. Renames account for both paths. +The `agentenv` scope owns the environment application and server modules, the +`environment-serve` and `environment-export` command registration, the shared +streamable HTTP listener, `scripts/verify-agentenv-environment.mjs` and its +workflow `real-agentenv.yml`. The lane exports a bundle from the source-owned +Grafana-shaped capture through the built CLI, packs the checkout into the +bundle's image and runs `agent-env run` on the hosted runner's Docker with the +pinned `agentenv-framework` release; it passes when the smoke task passes and +no container remains. It claims nothing about any hosted platform or another +framework version. + This narrower selection trades PR-time coverage for runner capacity. A change to unowned shared runtime code that fails only on arm64, macOS, Windows, the Apple artifact lane, or a provider lane that does not own the file is not caught on @@ -1316,9 +1326,9 @@ Portable owner-based test selection remains observational. Its reports compare proposed omissions with the full run; incomplete or skipped observations do not establish that fewer tests would preserve regression detection. -The eight automatic browser, binary-layout, EVM, JavaScript-recovery, -recorded-crash, and web provider lanes download `rea-build-cache` from their own -run after the build and static jobs succeed. Their existing build commands still +The nine automatic browser, binary-layout, EVM, JavaScript-recovery, +recorded-crash, web and environment-bundle provider lanes download +`rea-build-cache` from their own run after the build and static jobs succeed. Their existing build commands still validate task hashes and rebuild mismatched inputs; provider observations always run. Provider-workflow-only edits also admit the build without enabling source test shards. Direct provider dispatches retain the cross-run restore-only cache. @@ -1545,6 +1555,32 @@ This is fixture evidence, not a sandbox claim. Inputs remain unchanged and the sentinel must stay alive; owned cleanup and empty verifier descendants are required. Pass an installed package entrypoint as the script's first argument for package coverage. +### Exported environment bundles + +`npm run verify:agentenv:environment` is the `real-agentenv` lane. It requires +the host's Docker CLI named by absolute `REA_DOCKER_COMMAND`, Python 3.12 with +`agentenv-framework==0.9.1334` installed in a virtual environment whose +`agent-env` binary is named by absolute `REA_AGENT_ENV_COMMAND`, and outbound +network for `pip` on the runner and for `npm` inside the image build. Both +variables are required; the verifier resolves nothing from `PATH` and installs +nothing. Its disposable CI runner creates the virtual environment in runner +storage, exports both paths and uses the runner's own Docker; no developer host +configuration changes. The `agent-env run` itself is bounded to 15 minutes +inside the 30-minute job, so a hung run ends with the verifier's failed receipt +rather than the runner's kill. + +The verifier writes the Grafana-shaped HAR the view tests own, inspects it and +projects the `environment-manifest` view with all three template rules through +the built CLI, packs the checkout with `npm pack`, exports the bundle with +`--package-tarball` so the image installs this checkout rather than the +registry package, and runs `agent-env run` on the bundle. It checks that the +export reports the planned file set with the inputs copied byte-exact, that +the runner exits `0` with `tasks/smoke.json` reported as passed and the +verifier's description naming the tool count, and that no container exists +after the run that did not exist before; images built by the run remain and are +counted in the receipt. Pass an installed package entrypoint as the script's +first argument for package coverage. + ## Agent evaluation and conformance records `npm run verify:agent` evaluates investigation scenarios through a real local diff --git a/package.json b/package.json index 1e112fcb81..1acdd1234f 100644 --- a/package.json +++ b/package.json @@ -286,6 +286,7 @@ "verify:browser:source-maps": "npm run build:cached && node scripts/verify-browser-source-maps.mjs", "verify:browser:runtime": "npm run build:cached && node scripts/verify-browser-runtime.mjs", "verify:web:network-captures": "npm run build:cached && node scripts/verify-web-network-captures.mjs", + "verify:agentenv:environment": "npm run build:cached && node scripts/verify-agentenv-environment.mjs", "verify:web:http3": "npm run build:cached && node scripts/verify-http3-network-captures.mjs", "verify:binary:layout": "npm run build:cached && node scripts/verify-binary-layout.mjs", "verify:go:binary": "npm run build:cached && node scripts/verify-go-binary.mjs", diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index d36a181f3f..d80e5ca6e0 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -438,6 +438,25 @@ export const SCOPES = { "src/cli.ts", "src/artifacts/readStableArtifact.ts", ], + agentenv: [ + ".github/workflows/real-agentenv.yml", + "scripts/verify-agentenv-environment.mjs", + "src/application/environment/**", + "src/server/environment/**", + "src/cli/environmentCommands.ts", + "src/cli.ts", + "src/cliCommandNames.ts", + "src/config/environmentName.ts", + "src/config/mcpHttpEndpoint.ts", + "src/server/http/**", + "src/domain/analysisView/environmentManifest*.ts", + "src/application/analysisView/EnvironmentManifest*.ts", + "src/process/OwnedCommand.ts", + "src/process/ProviderProcess.ts", + "src/process/ProcessOwnership*.ts", + "src/process/PrivateRuntimeRoot.ts", + "tests/fixtures/historicalHar.ts", + ], nativeaot: [ "bridge/nativeaot/**", "scripts/generate-nativeaot-fixtures.mjs", diff --git a/scripts/verify-agentenv-environment.mjs b/scripts/verify-agentenv-environment.mjs new file mode 100644 index 0000000000..701662dbb6 --- /dev/null +++ b/scripts/verify-agentenv-environment.mjs @@ -0,0 +1,249 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; +import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +const AGENTENV_PIN = "agentenv-framework 0.9.1334"; +const TEMPLATE_RULES = [ + "after-uid-literal", + "opaque-token", + "sibling-variance", +]; +const execFileAsync = promisify(execFile); + +// Both tools are caller-supplied absolute paths, as every real-* lane names its upstream. +for (const key of ["REA_AGENT_ENV_COMMAND", "REA_DOCKER_COMMAND"]) + if (!isAbsolute(process.env[key] ?? "")) + throw new Error( + `verify:agentenv:environment requires absolute ${key}; see docs/testing.md. No dependencies are installed by this verifier.`, + ); +const agentEnv = process.env.REA_AGENT_ENV_COMMAND; +const docker = process.env.REA_DOCKER_COMMAND; +await Promise.all([access(agentEnv), access(docker)]); +const entrypoint = + process.argv[2] ?? fileURLToPath(new URL("./rea.mjs", import.meta.url)); +const checkout = resolve(dirname(entrypoint), ".."); +const run = createVerifierRun(); +const runtime = await PrivateRuntimeRoot.create({ + prefix: "rea-agentenv-verifier-", +}); +const environment = Object.fromEntries( + Object.entries(process.env).filter(([, value]) => typeof value === "string"), +); +const failures = []; +let cases = 0; +const receipt = {}; +try { + const containersBefore = await containerIds(); + receipt.docker_before = { + containers: containersBefore.length, + images: (await imageIds()).length, + }; + // Source-owned capture: the Grafana-shaped HAR builder the view tests use. + const { environmentManifestHar } = await import( + new URL("../tests/fixtures/historicalHar.ts", import.meta.url).href + ); + const harPath = join(runtime.path, "capture.har"); + await writeFile(harPath, JSON.stringify(environmentManifestHar())); + const capture = await cli( + "inspect-web-network-capture", + harPath, + "har", + "--format", + "json", + ); + assert.equal(capture.operation, "inspect_web_network_capture"); + assert.equal(capture.normalized_result.total_records, 28); + cases++; + const viewPath = join(runtime.path, "view.json"); + await writeFile( + viewPath, + JSON.stringify({ + source: { kind: "inline", evidence: capture }, + view: { kind: "environment-manifest", template_rules: TEMPLATE_RULES }, + }), + ); + const manifest = await cli( + "inspect-analysis-view", + viewPath, + "--format", + "json", + ); + assert.equal(manifest.normalized_result.kind, "environment-manifest"); + assert.equal( + manifest.normalized_result.parent_evidence_id, + capture.evidence_id, + ); + assert.equal(manifest.normalized_result.name, "grafana-example"); + const toolCount = manifest.normalized_result.tools.length; + assert.ok(toolCount > 0); + cases++; + const capturePath = join(runtime.path, "capture.json"); + const manifestPath = join(runtime.path, "manifest.json"); + await writeFile(capturePath, JSON.stringify(capture)); + await writeFile(manifestPath, JSON.stringify(manifest)); + // The lane proves this checkout, not the registry package: pack it and install the tarball in the image. + const { stdout: packed } = await execFileAsync( + "npm", + ["pack", "--ignore-scripts", "--json", "--pack-destination", runtime.path], + { cwd: checkout, env: environment, maxBuffer: 16 * 1024 * 1024 }, + ); + const tarball = join(runtime.path, packedFilename(JSON.parse(packed))); + await access(tarball); + const bundle = join(runtime.path, "grafana-example"); + const exported = await cli( + "environment-export", + manifestPath, + "--evidence", + capturePath, + bundle, + "--package-tarball", + tarball, + ); + assert.equal(exported.status, "exported"); + assert.equal(exported.name, "grafana-example"); + assert.equal(exported.tool_count, toolCount); + assert.equal(exported.package_source.kind, "tarball"); + assert.deepEqual(exported.files.map((file) => file.path).sort(), [ + "README.md", + "artifacts/check_grafana-example/verify.py", + "envs/grafana-example/Dockerfile", + "envs/grafana-example/env.toml", + "envs/grafana-example/evidence.json", + "envs/grafana-example/manifest.json", + "envs/grafana-example/rea-agents.tgz", + "tasks/smoke.json", + ]); + assert.equal( + await readFile(join(bundle, "envs/grafana-example/manifest.json"), "utf8"), + await readFile(manifestPath, "utf8"), + ); + const dockerfile = await readFile( + join(bundle, "envs/grafana-example/Dockerfile"), + "utf8", + ); + assert.ok(dockerfile.includes("COPY rea-agents.tgz /tmp/")); + assert.ok(dockerfile.includes("/tmp/rea-agents.tgz")); + cases++; + // The agentenv runner builds the image, deploys it on the local sandbox and runs the verifier. + // Bounded well under the 30-minute job so a hung run ends with this receipt, not the runner's kill. + const started = Date.now(); + const snapshot = await runOwnedCommand( + { + command: agentEnv, + arguments: ["run", bundle, "--sandbox", "local"], + cwd: runtime.path, + runId: `rea-agentenv-run-${randomUUID()}`, + hostEnvironment: environment, + }, + { timeoutMs: 15 * 60_000, diagnosticBytes: 4 * 1024 * 1024 }, + ); + receipt.agent_env_run = { + exit_code: snapshot.exitCode, + seconds: Math.round((Date.now() - started) / 1000), + stdout_tail: snapshot.stdout.text.trim().split("\n").slice(-6), + }; + assert.equal(snapshot.exitCode, 0); + assert.match(snapshot.stdout.text, /tasks\/smoke\.json v\d+: passed/u); + assert.match( + snapshot.stdout.text, + new RegExp( + `replay: listed ${String(toolCount)} tools, expected ${String(toolCount)};`, + "u", + ), + ); + cases++; + const containersAfter = await containerIds(); + const leaked = containersAfter.filter((id) => !containersBefore.includes(id)); + receipt.docker_after = { + containers: containersAfter.length, + images: (await imageIds()).length, + new_containers: leaked, + }; + assert.deepEqual(leaked, []); + cases++; +} catch (cause) { + failures.push(cause); +} finally { + try { + await runtime.close(); + } catch (cause) { + failures.push(cause); + } +} +const verifier = await completeVerifierRun(run); +try { + assert.equal(verifier.process_lineage.status, "verified"); + assert.deepEqual(verifier.process_lineage.descendants, []); +} catch (cause) { + failures.push(cause); +} +if (failures.length > 0) { + console.error( + JSON.stringify( + { status: "failed", public_cases: cases, ...receipt, verifier }, + null, + 2, + ), + ); + throw new AggregateError( + failures, + "Environment bundle verification failed; export, run and cleanup failures are retained.", + ); +} +console.log( + JSON.stringify( + { + status: "passed", + public_cases: cases, + upstream: AGENTENV_PIN, + runner: `${process.platform} ${process.arch}; host Docker`, + offline: false, + ...receipt, + verifier, + }, + null, + 2, + ), +); + +async function cli(...args) { + const { stdout } = await execFileAsync( + process.execPath, + [entrypoint, ...args, "--json"], + { env: environment, cwd: runtime.path, maxBuffer: 64 * 1024 * 1024 }, + ); + return JSON.parse(stdout); +} + +// npm 11 prints `npm pack --json` as a one-element array; npm 12 keys the same record by package name. +function packedFilename(report) { + const record = Array.isArray(report) ? report[0] : Object.values(report)[0]; + assert.equal(typeof record?.filename, "string"); + return record.filename; +} + +// Containers are not verifier descendants, so leaks are checked by id against the pre-run set. +async function containerIds() { + const { stdout } = await execFileAsync(docker, ["ps", "-aq"], { + env: environment, + }); + return stdout + .split("\n") + .filter((line) => line.length > 0) + .sort(); +} + +async function imageIds() { + const { stdout } = await execFileAsync(docker, ["images", "-q"], { + env: environment, + }); + return [...new Set(stdout.split("\n").filter((line) => line.length > 0))]; +} diff --git a/src/application/environment/environmentBundle.test.ts b/src/application/environment/environmentBundle.test.ts new file mode 100644 index 0000000000..5482416164 --- /dev/null +++ b/src/application/environment/environmentBundle.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it } from "vitest"; + +import { environmentServeSource } from "../../../tests/fixtures/environmentServe.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js"; +import { + BUNDLE_BASE_IMAGE, + bundleDockerfile, + planEnvironmentBundle, + registryPackageSource, +} from "./environmentBundle.js"; + +const plan = (options: { name?: string; tarball?: string } = {}) => { + const planned = planEnvironmentBundle(environmentServeSource(), { + name: options.name, + packageSource: + options.tarball === undefined + ? registryPackageSource() + : { kind: "tarball", path: options.tarball }, + }); + if (!planned.ok) throw planned.error; + return planned.value; +}; + +const file = (planned: ReturnType, path: string): string => { + const found = planned.files.find((entry) => entry.path === path); + if (found === undefined) throw new Error(`Planned file missing: ${path}`); + return found.content; +}; + +describe("planEnvironmentBundle", () => { + it("plans the echo-shaped file set under the manifest name with byte-exact copies beside the Dockerfile", () => { + const planned = plan(); + expect(planned.name).toBe("grafana-example"); + expect(planned.toolCount).toBe(19); + expect(planned.files.map((entry) => entry.path)).toEqual([ + "README.md", + "envs/grafana-example/env.toml", + "envs/grafana-example/Dockerfile", + "artifacts/check_grafana-example/verify.py", + "tasks/smoke.json", + ]); + expect(planned.copies).toEqual({ + manifest: "envs/grafana-example/manifest.json", + evidence: "envs/grafana-example/evidence.json", + tarball: null, + }); + expect(file(planned, "envs/grafana-example/env.toml")).toBe( + 'environment_name = "grafana-example"\n', + ); + }); + + it("pins the registry package to this build's exact version in the Dockerfile", () => { + const dockerfile = file(plan(), "envs/grafana-example/Dockerfile"); + const lines = dockerfile.trimEnd().split("\n"); + expect(lines[0]).toBe(`FROM ${BUNDLE_BASE_IMAGE}`); + expect(lines).toContain("ENV MCP_HOST=0.0.0.0"); + expect(lines).toContain("ENV MCP_PORT=18765"); + expect(lines).toContain( + "RUN apt-get update && apt-get install -y --no-install-recommends python3 && rm -rf /var/lib/apt/lists/*", + ); + expect(lines).toContain( + `RUN npm install -g --ignore-scripts --no-fund --no-audit rea-agents@${PACKAGE_METADATA.version}`, + ); + expect(lines).toContain("COPY manifest.json evidence.json /env/"); + expect(lines).toContain("USER node"); + expect(lines.at(-1)).toBe( + 'CMD ["rea","environment-serve","/env/manifest.json","--evidence","/env/evidence.json"]', + ); + expect(dockerfile).not.toContain("rea-agents.tgz"); + expect(dockerfile.indexOf("RUN npm install")).toBeLessThan( + dockerfile.indexOf("USER node"), + ); + }); + + it("copies and installs the tarball instead of the registry package when one is given", () => { + const planned = plan({ tarball: "/packed/rea-agents-1.2.3.tgz" }); + expect(planned.copies.tarball).toBe("envs/grafana-example/rea-agents.tgz"); + const dockerfile = file(planned, "envs/grafana-example/Dockerfile"); + expect(dockerfile).toContain("COPY rea-agents.tgz /tmp/\n"); + expect(dockerfile).toContain( + "RUN npm install -g --ignore-scripts --no-fund --no-audit /tmp/rea-agents.tgz && rm /tmp/rea-agents.tgz\n", + ); + expect(dockerfile).not.toContain(`rea-agents@${PACKAGE_METADATA.version}`); + expect(bundleDockerfile(registryPackageSource())).not.toBe(dockerfile); + }); + + it("writes a verifier that checks the tool count and the first parameterless GET tool's recorded status", () => { + const planned = plan(); + // Manifest tools are name-ordered, so the first parameterless GET is the annotations list. + expect(planned.check).toEqual({ + tool: "get_annotations", + recordOrdinal: 25, + status: 200, + }); + const verifier = file(planned, "artifacts/check_grafana-example/verify.py"); + expect(verifier).toContain("from mcp import ClientSession"); + expect(verifier).toContain( + "from mcp.client.streamable_http import streamable_http_client", + ); + expect(verifier).toContain("async def verify(mcp_url):"); + expect(verifier).toContain("TOOL_COUNT = 19"); + expect(verifier).toContain('TOOL = "get_annotations"'); + expect(verifier).toContain("RECORDED_STATUS = 200"); + expect(verifier).toContain("await session.list_tools()"); + expect(verifier).toContain("await session.call_tool(TOOL, {})"); + expect(verifier).toContain( + 'return [{"id": "replay", "result": passed, "description": description}]', + ); + }); + + it("writes the smoke task deploying the env locally before the replay verifier", () => { + expect(JSON.parse(file(plan(), "tasks/smoke.json"))).toEqual([ + { + id: "env", + type: "deploy_env", + env_id: "grafana-example", + sandbox_type: "local", + }, + { + id: "replay", + type: "env_outcome_verifier", + env_id: "grafana-example", + file_artifact_id: "check_grafana-example", + verifier_id: "replay", + }, + ]); + }); + + it("records provenance, replay limits, the run command and the rights note in the README", () => { + const source = environmentServeSource(); + const readme = file(plan(), "README.md"); + expect(readme).toContain( + `- Capture Evidence (the manifest's parent): \`${source.evidenceId}\``, + ); + expect(readme).toContain( + `- Manifest Evidence: \`${source.manifestEvidenceId}\``, + ); + expect(readme).toContain( + `- Manifest view digest: \`${source.manifest.view_digest}\``, + ); + expect(readme).toContain( + `- Capture artifact SHA-256: \`${source.captureSha256 ?? ""}\``, + ); + expect(readme).toContain("- Tools: 19"); + expect(readme).toContain(`rea-agents@${PACKAGE_METADATA.version}`); + expect(readme).toContain("```sh\nagent-env run .\n```"); + expect(readme).not.toContain("agent-env run grafana-example"); + expect(readme).toContain("Replay is not emulation"); + expect(readme).toContain("belongs to the captured application"); + expect(readme).toContain("Do not publish this bundle without the rights"); + expect(readme).toContain("calls `get_annotations`"); + }); + + it("accepts a kebab-case name override and refuses any other name as invalid input", () => { + const renamed = plan({ name: "replay-2" }); + expect(renamed.files.map((entry) => entry.path)).toContain( + "envs/replay-2/Dockerfile", + ); + expect(file(renamed, "envs/replay-2/env.toml")).toBe( + 'environment_name = "replay-2"\n', + ); + expect(JSON.parse(file(renamed, "tasks/smoke.json"))[1]).toMatchObject({ + env_id: "replay-2", + file_artifact_id: "check_replay-2", + }); + for (const name of ["Grafana", "a_b", "-a", "a--b", "", "a".repeat(65)]) { + const refused = planEnvironmentBundle(environmentServeSource(), { + name, + packageSource: registryPackageSource(), + }); + if (refused.ok) throw new Error(`Expected ${name} to be refused`); + expect(projectAnalysisError(refused.error)).toMatchObject({ + code: "invalid_request", + details: { + operation: "environment-export", + issues: [{ path: ["name"], reason: "invalid_format" }], + }, + }); + } + }); +}); diff --git a/src/application/environment/environmentBundle.ts b/src/application/environment/environmentBundle.ts new file mode 100644 index 0000000000..5947f14ca3 --- /dev/null +++ b/src/application/environment/environmentBundle.ts @@ -0,0 +1,306 @@ +import { PRODUCT_IDENTITY } from "../../identity.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import type { + EnvironmentManifestTool, + EnvironmentSource, +} from "./environmentSource.js"; + +/** Operation name every refusal of the export command reports. */ +export const ENVIRONMENT_EXPORT_OPERATION = "environment-export"; + +/** The image the exported Dockerfile builds from; the agentenv runner builds it with Docker. */ +export const BUNDLE_BASE_IMAGE = "node:24.18.0-bookworm-slim"; + +/** The port the agentenv framework maps for an MCP server env; it sets the host but not the port. */ +export const BUNDLE_MCP_PORT = 18765; + +/** Name of the packed CLI inside the env folder when a tarball is exported. */ +export const BUNDLE_TARBALL_NAME = "rea-agents.tgz"; + +/** Folder and env id rule: a kebab-case label the agentenv framework accepts as a local id. */ +const BUNDLE_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; +const BUNDLE_NAME_MAX_LENGTH = 64; + +/** Where the Dockerfile installs the CLI from. */ +export type EnvironmentPackageSource = + | { readonly kind: "registry"; readonly specifier: string } + | { readonly kind: "tarball"; readonly path: string }; + +/** One text file of the bundle, relative to the bundle root. */ +export interface EnvironmentBundleFile { + readonly path: string; + readonly content: string; +} + +/** The replay check the verifier performs against the served environment. */ +export interface EnvironmentReplayCheck { + readonly tool: string; + readonly recordOrdinal: number; + readonly status: number | null; +} + +/** Everything the export writes, derived from the source alone. */ +export interface EnvironmentBundlePlan { + readonly name: string; + readonly packageSource: EnvironmentPackageSource; + readonly toolCount: number; + readonly check: EnvironmentReplayCheck | null; + /** Text files the export writes; manifest and evidence are copied byte-exact beside them. */ + readonly files: readonly EnvironmentBundleFile[]; + /** Relative paths of the byte-exact copies in write order. */ + readonly copies: { + readonly manifest: string; + readonly evidence: string; + readonly tarball: string | null; + }; +} + +export interface EnvironmentBundleOptions { + /** Folder and env id; defaults to the manifest's kebab-case name. */ + readonly name?: string | undefined; + readonly packageSource: EnvironmentPackageSource; +} + +/** The registry source pinning the exact version of this build. */ +export const registryPackageSource = (): EnvironmentPackageSource => ({ + kind: "registry", + specifier: PRODUCT_IDENTITY.registrationPackageSpecifier, +}); + +const nameError = (name: string): AnalysisInputError => + new AnalysisInputError( + ENVIRONMENT_EXPORT_OPERATION, + { + userMessage: `The bundle name ${JSON.stringify(name)} is not a kebab-case label of at most ${String(BUNDLE_NAME_MAX_LENGTH)} characters.`, + }, + [ + { + path: ["name"], + reason: "invalid_format", + expected: BUNDLE_NAME.source, + message: + "The bundle name is the env folder and env id; use lowercase letters, digits and single hyphens.", + }, + ], + ); + +const installLine = (source: EnvironmentPackageSource): string => + source.kind === "registry" + ? `RUN npm install -g --ignore-scripts --no-fund --no-audit ${source.specifier}` + : `COPY ${BUNDLE_TARBALL_NAME} /tmp/\nRUN npm install -g --ignore-scripts --no-fund --no-audit /tmp/${BUNDLE_TARBALL_NAME} && rm /tmp/${BUNDLE_TARBALL_NAME}`; + +/** The agentenv local sandbox probes container readiness with python3 inside the container; the slim image has none. */ +const READINESS_PROBE_LAYER = + "RUN apt-get update && apt-get install -y --no-install-recommends python3 && rm -rf /var/lib/apt/lists/*"; + +/** The Dockerfile text: pinned base image, readiness probe interpreter, pinned CLI, listener defaults, non-root user. */ +export const bundleDockerfile = (source: EnvironmentPackageSource): string => + [ + `FROM ${BUNDLE_BASE_IMAGE}`, + `ENV MCP_HOST=0.0.0.0`, + `ENV MCP_PORT=${String(BUNDLE_MCP_PORT)}`, + READINESS_PROBE_LAYER, + installLine(source), + "COPY manifest.json evidence.json /env/", + "USER node", + 'CMD ["rea","environment-serve","/env/manifest.json","--evidence","/env/evidence.json"]', + "", + ].join("\n"); + +/** The first GET tool without path parameters, in manifest order, and its first record's status. */ +const replayCheck = ( + source: EnvironmentSource, +): EnvironmentReplayCheck | null => { + const tool: EnvironmentManifestTool | undefined = source.manifest.tools.find( + (candidate) => + candidate.method === "GET" && + candidate.params.length === 0 && + candidate.record_ordinals.length > 0, + ); + if (tool === undefined) return null; + const ordinal = tool.record_ordinals[0] ?? -1; + const exchange = source.exchanges.get(ordinal); + return { + tool: tool.name, + recordOrdinal: ordinal, + status: exchange?.status ?? tool.statuses[0] ?? null, + }; +}; + +const pythonLiteral = (value: string | number | null): string => + value === null ? "None" : JSON.stringify(value); + +/** The verifier script in the shape of the agentenv echo example's `verify(mcp_url)`. */ +export const bundleVerifier = ( + toolCount: number, + check: EnvironmentReplayCheck | null, +): string => { + const header = [ + '"""Checks the replay environment through its MCP endpoint: the tool count and one replayed status."""', + "import json", + "", + "from mcp import ClientSession", + "from mcp.client.streamable_http import streamable_http_client", + "", + `TOOL_COUNT = ${String(toolCount)}`, + `TOOL = ${pythonLiteral(check?.tool ?? null)}`, + `RECORDED_STATUS = ${pythonLiteral(check?.status ?? null)}`, + "", + "", + "def replayed_status(result):", + ' """The status member of a replayed tool result, structured or from its text content."""', + " data = result.structuredContent", + " if data is None:", + ' text = "".join(getattr(part, "text", "") for part in result.content)', + " data = json.loads(text) if text else {}", + ' return data.get("status") if isinstance(data, dict) else None', + "", + "", + "async def verify(mcp_url):", + " async with streamable_http_client(mcp_url) as (read, write, _):", + " async with ClientSession(read, write) as session:", + " await session.initialize()", + " listed = await session.list_tools()", + " result = await session.call_tool(TOOL, {}) if TOOL is not None else None", + " count = len(listed.tools)", + " passed = count == TOOL_COUNT", + ' description = f"listed {count} tools, expected {TOOL_COUNT}"', + " if result is not None:", + " status = None if result.isError else replayed_status(result)", + " passed = passed and status == RECORDED_STATUS", + ' description += f"; {TOOL} replayed status {status!r}, recorded {RECORDED_STATUS!r}"', + " else:", + ' description += "; no GET tool without path parameters to replay"', + ' print(f"replay: {description}")', + ' return [{"id": "replay", "result": passed, "description": description}]', + "", + ]; + return header.join("\n"); +}; + +/** The smoke task: deploy the env locally, then run the replay verifier against it. */ +export const bundleSmokeTask = (name: string): string => + `${JSON.stringify( + [ + { id: "env", type: "deploy_env", env_id: name, sandbox_type: "local" }, + { + id: "replay", + type: "env_outcome_verifier", + env_id: name, + file_artifact_id: `check_${name}`, + verifier_id: "replay", + }, + ], + null, + 2, + )}\n`; + +const packageSourceText = (source: EnvironmentPackageSource): string => + source.kind === "registry" + ? `\`${source.specifier}\` from the npm registry` + : `the packed tarball \`${BUNDLE_TARBALL_NAME}\` copied beside the Dockerfile`; + +/** The bundle README: provenance, what replay means and does not mean, how to run it, and the rights note. */ +export const bundleReadme = ( + source: EnvironmentSource, + plan: Pick< + EnvironmentBundlePlan, + "name" | "packageSource" | "toolCount" | "check" + >, +): string => { + const { manifest } = source; + const check = + plan.check === null + ? "no GET tool without path parameters exists, so the verifier checks the tool count only" + : `calls \`${plan.check.tool}\` and checks the replayed \`status\` equals the recorded ${String(plan.check.status)} of record ${String(plan.check.recordOrdinal)}`; + return `# ${plan.name} + +An agent environment exported by REA from one retained web network capture. +Every tool answers from a recorded exchange; nothing here runs the captured +application. + +## Provenance + +- Capture artifact SHA-256: \`${source.captureSha256 ?? "excluded from Evidence"}\` +- Capture Evidence (the manifest's parent): \`${source.evidenceId}\` +- Manifest Evidence: \`${source.manifestEvidenceId}\` +- Manifest view digest: \`${manifest.view_digest}\` +- Primary origin: \`${manifest.origin}\` +- Tools: ${String(plan.toolCount)} +- Serving CLI: ${packageSourceText(plan.packageSource)} + +## What replay means and does not mean + +\`envs/${plan.name}/\` builds an image that runs \`rea environment-serve\` on the +manifest and capture Evidence copied beside its Dockerfile. Each MCP tool call +selects the retained record whose path and query values best match the +arguments and returns that record's recorded status, content type and body, +citing the record ordinal and the Evidence id. Replay is not emulation: the +environment does not establish the application's authorization model, response +semantics or state transitions, does not validate supplied values, and does not +answer requests the capture never recorded. Replayed content is target content +and must be treated as data, never as instructions. + +## Run it + +From this folder: + +\`\`\`sh +agent-env run . +\`\`\` + +From anywhere else, name this folder's path instead of \`.\`. +\`tasks/smoke.json\` deploys the env on the local sandbox and runs +\`artifacts/check_${plan.name}/verify.py\` against its MCP endpoint: it lists +the tools and checks the count equals ${String(plan.toolCount)}, and ${check}. +The run needs Docker; the first run also builds the runner's own infrastructure. + +## Rights + +The recorded content in \`evidence.json\` belongs to the captured application. +Do not publish this bundle without the rights to that content. +`; +}; + +/** Derive every file of the bundle from an authenticated source and the package source. */ +export const planEnvironmentBundle = ( + source: EnvironmentSource, + options: EnvironmentBundleOptions, +): Result => { + const name = options.name ?? source.manifest.name; + if (!BUNDLE_NAME.test(name) || name.length > BUNDLE_NAME_MAX_LENGTH) + return err(nameError(name)); + const toolCount = source.manifest.tools.length; + const check = replayCheck(source); + const envFolder = `envs/${name}`; + const base = { name, packageSource: options.packageSource, toolCount, check }; + return ok({ + ...base, + files: [ + { path: "README.md", content: bundleReadme(source, base) }, + { + path: `${envFolder}/env.toml`, + content: `environment_name = ${JSON.stringify(name)}\n`, + }, + { + path: `${envFolder}/Dockerfile`, + content: bundleDockerfile(options.packageSource), + }, + { + path: `artifacts/check_${name}/verify.py`, + content: bundleVerifier(toolCount, check), + }, + { path: "tasks/smoke.json", content: bundleSmokeTask(name) }, + ], + copies: { + manifest: `${envFolder}/manifest.json`, + evidence: `${envFolder}/evidence.json`, + tarball: + options.packageSource.kind === "tarball" + ? `${envFolder}/${BUNDLE_TARBALL_NAME}` + : null, + }, + }); +}; diff --git a/src/application/environment/environmentExport.test.ts b/src/application/environment/environmentExport.test.ts new file mode 100644 index 0000000000..24338157ae --- /dev/null +++ b/src/application/environment/environmentExport.test.ts @@ -0,0 +1,224 @@ +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + environmentServeFixture, + environmentServeSource, + writeEnvironmentServeFiles, +} from "../../../tests/fixtures/environmentServe.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js"; +import { + exportEnvironmentBundle, + readEnvironmentExportInputs, +} from "./environmentExport.js"; + +const listFiles = async (root: string): Promise => + (await readdir(root, { recursive: true, withFileTypes: true })) + .filter((entry) => entry.isFile()) + .map((entry) => join(entry.parentPath, entry.name).slice(root.length + 1)) + .sort(); + +const prepare = async (prefix: string) => { + const root = await createTestTempDirectory(prefix); + const fixture = environmentServeFixture(); + const files = await writeEnvironmentServeFiles(root, fixture); + const inputs = await readEnvironmentExportInputs( + files.manifestPath, + files.capturePath, + ); + if (!inputs.ok) throw inputs.error; + return { + root, + files, + input: { ...inputs.value, source: environmentServeSource(fixture) }, + directory: join(root, "bundle"), + }; +}; + +describe("readEnvironmentExportInputs", () => { + it("reads each file once as the bytes on disk and the value they parse to", async () => { + const root = await createTestTempDirectory("rea-environment-inputs-"); + const fixture = environmentServeFixture(); + const files = await writeEnvironmentServeFiles(root, fixture); + const inputs = await readEnvironmentExportInputs( + files.manifestPath, + files.capturePath, + ); + if (!inputs.ok) throw inputs.error; + expect(inputs.value.manifest.path).toBe(files.manifestPath); + expect(inputs.value.manifest.bytes).toEqual( + await readFile(files.manifestPath), + ); + expect(inputs.value.manifest.value).toEqual(fixture.manifest); + expect(inputs.value.evidence.bytes).toEqual( + await readFile(files.capturePath), + ); + expect(inputs.value.evidence.value).toEqual(fixture.capture); + }); + + it("refuses inline JSON, a directory and a file that is not JSON by role", async () => { + const root = await createTestTempDirectory("rea-environment-inputs-bad-"); + const files = await writeEnvironmentServeFiles(root); + const inline = await readEnvironmentExportInputs( + await readFile(files.manifestPath, "utf8"), + files.capturePath, + ); + if (inline.ok) throw new Error("Expected the inline manifest refusal"); + expect(projectAnalysisError(inline.error)).toMatchObject({ + code: "invalid_request", + details: { + operation: "environment-export", + issues: [{ path: ["manifest"], reason: "invalid_value" }], + }, + }); + const directory = await readEnvironmentExportInputs( + files.manifestPath, + root, + ); + if (directory.ok) throw new Error("Expected the directory refusal"); + expect(projectAnalysisError(directory.error).details?.issues).toEqual([ + expect.objectContaining({ path: ["evidence"], reason: "invalid_value" }), + ]); + const broken = join(root, "broken.json"); + await writeFile(broken, '{"evidence_id": '); + const notJson = await readEnvironmentExportInputs( + files.manifestPath, + broken, + ); + if (notJson.ok) throw new Error("Expected the malformed JSON refusal"); + expect(projectAnalysisError(notJson.error)).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { path: ["evidence"], reason: "invalid_format", expected: "JSON" }, + ], + }, + }); + expect(projectAnalysisError(notJson.error).message).toContain(broken); + }); +}); + +describe("exportEnvironmentBundle", () => { + it("writes the bundle into an absent directory with the read bytes copied unchanged", async () => { + const { root, files, input, directory } = await prepare( + "rea-environment-export-", + ); + const exported = await exportEnvironmentBundle({ ...input, directory }); + if (!exported.ok) throw exported.error; + expect(exported.value).toMatchObject({ + status: "exported", + directory, + name: "grafana-example", + tool_count: 19, + evidence_id: input.source.evidenceId, + manifest_evidence_id: input.source.manifestEvidenceId, + package_source: { + kind: "registry", + specifier: `rea-agents@${PACKAGE_METADATA.version}`, + }, + }); + const written = exported.value.files.map((file) => file.path).sort(); + expect(written).toEqual(await listFiles(directory)); + expect(written).toEqual([ + "README.md", + "artifacts/check_grafana-example/verify.py", + "envs/grafana-example/Dockerfile", + "envs/grafana-example/env.toml", + "envs/grafana-example/evidence.json", + "envs/grafana-example/manifest.json", + "tasks/smoke.json", + ]); + for (const file of exported.value.files) + expect(file.bytes).toBe( + (await readFile(join(directory, file.path))).length, + ); + expect( + await readFile(join(directory, "envs/grafana-example/manifest.json")), + ).toEqual(await readFile(files.manifestPath)); + expect( + await readFile(join(directory, "envs/grafana-example/evidence.json")), + ).toEqual(input.evidence.bytes); + expect(await listFiles(root)).toEqual( + expect.arrayContaining(["capture.json", "manifest.json"]), + ); + }); + + it("copies the package tarball beside the Dockerfile and reports it as the package source", async () => { + const { root, input, directory } = await prepare( + "rea-environment-export-tarball-", + ); + const tarball = join(root, "rea-agents-0.0.0.tgz"); + await writeFile(tarball, Buffer.from([0x1f, 0x8b, 8, 0, 1, 2, 3])); + const exported = await exportEnvironmentBundle({ + ...input, + directory, + packageTarball: tarball, + name: "replay-copy", + }); + if (!exported.ok) throw exported.error; + expect(exported.value.package_source).toEqual({ + kind: "tarball", + path: tarball, + }); + expect(exported.value.files).toContainEqual({ + path: "envs/replay-copy/rea-agents.tgz", + bytes: 7, + }); + expect( + await readFile(join(directory, "envs/replay-copy/rea-agents.tgz")), + ).toEqual(await readFile(tarball)); + expect( + await readFile(join(directory, "envs/replay-copy/Dockerfile"), "utf8"), + ).toContain("COPY rea-agents.tgz /tmp/"); + }); + + it("refuses a non-empty directory, a file path and a missing tarball before writing anything", async () => { + const { root, input, directory } = await prepare( + "rea-environment-export-refusals-", + ); + await mkdir(directory); + await writeFile(join(directory, "keep.txt"), "existing"); + const nonEmpty = await exportEnvironmentBundle({ ...input, directory }); + if (nonEmpty.ok) + throw new Error("Expected the non-empty directory refusal"); + expect(projectAnalysisError(nonEmpty.error)).toMatchObject({ + code: "invalid_request", + details: { + operation: "environment-export", + issues: [{ path: ["directory"], reason: "invalid_value" }], + }, + }); + expect(projectAnalysisError(nonEmpty.error).message).toContain("not empty"); + expect(await listFiles(directory)).toEqual(["keep.txt"]); + const asFile = await exportEnvironmentBundle({ + ...input, + directory: join(directory, "keep.txt"), + }); + if (asFile.ok) throw new Error("Expected the file path refusal"); + expect(projectAnalysisError(asFile.error).details?.issues).toEqual([ + expect.objectContaining({ path: ["directory"] }), + ]); + const missingTarball = await exportEnvironmentBundle({ + ...input, + directory: join(root, "fresh"), + packageTarball: join(root, "absent.tgz"), + }); + if (missingTarball.ok) throw new Error("Expected the tarball refusal"); + expect(projectAnalysisError(missingTarball.error).details?.issues).toEqual([ + expect.objectContaining({ path: ["package_tarball"] }), + ]); + expect(await listFiles(root)).not.toContain("fresh/README.md"); + }); + + it("accepts an existing empty directory", async () => { + const { input, directory } = await prepare("rea-environment-export-empty-"); + await mkdir(directory); + const exported = await exportEnvironmentBundle({ ...input, directory }); + expect(exported.ok).toBe(true); + expect(await listFiles(directory)).toHaveLength(7); + }); +}); diff --git a/src/application/environment/environmentExport.ts b/src/application/environment/environmentExport.ts new file mode 100644 index 0000000000..c5fcd10151 --- /dev/null +++ b/src/application/environment/environmentExport.ts @@ -0,0 +1,318 @@ +import { constants } from "node:fs"; +import { copyFile, mkdir, readdir, stat, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisInputError, + AnalysisResourceConstraintError, + type AnalysisInputIssue, +} from "../../domain/analysisErrorCore.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { readRegularFile } from "../RegularFileRead.js"; +import { parseUtf8Json } from "../Utf8JsonInput.js"; +import { + ENVIRONMENT_EXPORT_OPERATION, + planEnvironmentBundle, + registryPackageSource, + type EnvironmentBundlePlan, + type EnvironmentPackageSource, +} from "./environmentBundle.js"; +import type { EnvironmentSource } from "./environmentSource.js"; + +/** One input file read once: the bytes the bundle copies and the value they parse to. */ +export interface EnvironmentExportFile { + readonly path: string; + readonly bytes: Buffer; + readonly value: unknown; +} + +/** The manifest and capture files as read for authentication and copying. */ +export interface EnvironmentExportInputs { + readonly manifest: EnvironmentExportFile; + readonly evidence: EnvironmentExportFile; +} + +export interface EnvironmentExportInput extends EnvironmentExportInputs { + /** The authenticated pair the bundle replays, loaded from the parsed values of the two files. */ + readonly source: EnvironmentSource; + /** Absolute bundle root; absent or empty. */ + readonly directory: string; + readonly name?: string | undefined; + /** Absolute path of a packed CLI tarball the Dockerfile installs instead of the registry package. */ + readonly packageTarball?: string | undefined; +} + +/** One written file with its size, as the command reports it. */ +export interface WrittenBundleFile { + readonly path: string; + readonly bytes: number; +} + +export interface EnvironmentExportResult { + readonly status: "exported"; + readonly directory: string; + readonly name: string; + readonly tool_count: number; + readonly evidence_id: string; + readonly manifest_evidence_id: string; + readonly package_source: JsonValue; + readonly files: readonly WrittenBundleFile[]; +} + +const inputError = ( + issue: AnalysisInputIssue, + userMessage: string, +): AnalysisInputError => + new AnalysisInputError(ENVIRONMENT_EXPORT_OPERATION, { userMessage }, [ + issue, + ]); + +const systemCode = (cause: unknown): string | undefined => + cause instanceof Error && "code" in cause && typeof cause.code === "string" + ? cause.code + : undefined; + +/** Host permission failures are typed; any other filesystem failure is unexpected and propagates. */ +const rethrowUnlessDenied = (cause: unknown, path: string): never => { + const code = systemCode(cause); + if (code === "EACCES" || code === "EPERM") + throw new AnalysisAccessDeniedError( + ENVIRONMENT_EXPORT_OPERATION, + path, + code, + { + cause, + }, + ); + throw cause; +}; + +/** The directory may be absent or an empty directory; anything else is refused before any write. */ +const checkDirectory = async ( + directory: string, +): Promise> => { + let entries: readonly string[]; + try { + entries = await readdir(directory); + } catch (cause: unknown) { + const code = systemCode(cause); + if (code === "ENOENT") return ok(undefined); + if (code === "ENOTDIR") + return err( + inputError( + { + path: ["directory"], + reason: "invalid_value", + message: "The bundle directory path names a file.", + }, + `${directory} exists and is not a directory.`, + ), + ); + return rethrowUnlessDenied(cause, directory); + } + if (entries.length > 0) + return err( + inputError( + { + path: ["directory"], + reason: "invalid_value", + message: + "The bundle directory must be absent or empty; nothing is overwritten.", + }, + `${directory} is not empty (${String(entries.length)} entries); choose an absent or empty directory.`, + ), + ); + return ok(undefined); +}; + +/** The tarball must be a regular file; its bytes are copied, never inspected. */ +const checkTarball = async ( + path: string, +): Promise> => { + try { + const info = await stat(path); + if (!info.isFile()) + return err( + inputError( + { + path: ["package_tarball"], + reason: "invalid_value", + message: "The package tarball must be a regular file.", + }, + `${path} is not a regular file.`, + ), + ); + } catch (cause: unknown) { + const code = systemCode(cause); + if (code === "ENOENT" || code === "ENOTDIR" || code === "ENAMETOOLONG") + return err( + inputError( + { + path: ["package_tarball"], + reason: "invalid_value", + message: "The package tarball path must name an existing file.", + }, + `${path} does not exist.`, + ), + ); + return rethrowUnlessDenied(cause, path); + } + return ok({ kind: "tarball", path }); +}; + +/** A path that is not a readable regular file, including inline JSON text offered as one. */ +const NOT_A_FILE_CODES = new Set([ + "ENOENT", + "EISDIR", + "ENOTFILE", + "ENOTDIR", + "ENAMETOOLONG", +]); + +/** Read one input once: the same bytes are parsed for authentication and copied into the bundle. */ +const readInput = async ( + role: "manifest" | "evidence", + path: string, +): Promise> => { + let bytes: Buffer; + try { + bytes = await readRegularFile(path); + } catch (cause: unknown) { + if (NOT_A_FILE_CODES.has(systemCode(cause) ?? "")) + return err( + inputError( + { + path: [role], + reason: "invalid_value", + message: `The ${role} argument must be a readable file path; inline JSON cannot be exported.`, + }, + `${path} could not be read as the ${role} file.`, + ), + ); + return rethrowUnlessDenied(cause, path); + } + let parsed: ReturnType; + try { + parsed = parseUtf8Json(bytes, ENVIRONMENT_EXPORT_OPERATION, path); + } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) return err(cause); + throw cause; + } + if (!parsed.ok) + return err( + inputError( + { + path: [role], + reason: "invalid_format", + expected: "JSON", + message: parsed.error, + }, + `${path} is not valid JSON: ${parsed.error}`, + ), + ); + return ok({ path, bytes, value: parsed.value }); +}; + +/** Read both files once; the caller authenticates the parsed values and the export copies the same bytes. */ +export const readEnvironmentExportInputs = async ( + manifestPath: string, + evidencePath: string, +): Promise> => { + const manifest = await readInput("manifest", manifestPath); + if (!manifest.ok) return manifest; + const evidence = await readInput("evidence", evidencePath); + if (!evidence.ok) return evidence; + return ok({ manifest: manifest.value, evidence: evidence.value }); +}; + +const writeNew = async ( + root: string, + relative: string, + content: string | Uint8Array, +): Promise => { + const path = join(root, relative); + try { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, content, { flag: "wx" }); + } catch (cause: unknown) { + rethrowUnlessDenied(cause, path); + } + return { path: relative, bytes: Buffer.byteLength(content) }; +}; + +const copyNew = async ( + root: string, + relative: string, + from: string, +): Promise => { + const path = join(root, relative); + try { + await mkdir(dirname(path), { recursive: true }); + await copyFile(from, path, constants.COPYFILE_EXCL); + return { path: relative, bytes: (await stat(path)).size }; + } catch (cause: unknown) { + return rethrowUnlessDenied(cause, path); + } +}; + +const packageSourceValue = (source: EnvironmentPackageSource): JsonValue => + source.kind === "registry" + ? { kind: "registry", specifier: source.specifier } + : { kind: "tarball", path: source.path }; + +/** Write every planned file in plan order; manifest and evidence are the authenticated bytes unchanged. */ +const writeBundle = async ( + input: EnvironmentExportInput, + plan: EnvironmentBundlePlan, +): Promise => { + const written: WrittenBundleFile[] = []; + for (const file of plan.files) + written.push(await writeNew(input.directory, file.path, file.content)); + written.push( + await writeNew(input.directory, plan.copies.manifest, input.manifest.bytes), + ); + written.push( + await writeNew(input.directory, plan.copies.evidence, input.evidence.bytes), + ); + if (plan.copies.tarball !== null && plan.packageSource.kind === "tarball") + written.push( + await copyNew( + input.directory, + plan.copies.tarball, + plan.packageSource.path, + ), + ); + return written; +}; + +/** Export the authenticated pair as an agent environment bundle folder. */ +export const exportEnvironmentBundle = async ( + input: EnvironmentExportInput, +): Promise> => { + const directory = await checkDirectory(input.directory); + if (!directory.ok) return directory; + const packageSource = + input.packageTarball === undefined + ? ok(registryPackageSource()) + : await checkTarball(input.packageTarball); + if (!packageSource.ok) return packageSource; + const plan = planEnvironmentBundle(input.source, { + name: input.name, + packageSource: packageSource.value, + }); + if (!plan.ok) return plan; + const files = await writeBundle(input, plan.value); + return ok({ + status: "exported", + directory: input.directory, + name: plan.value.name, + tool_count: plan.value.toolCount, + evidence_id: input.source.evidenceId, + manifest_evidence_id: input.source.manifestEvidenceId, + package_source: packageSourceValue(plan.value.packageSource), + files, + }); +}; diff --git a/src/application/environment/environmentSource.ts b/src/application/environment/environmentSource.ts index 635e9fd08c..1dece609cf 100644 --- a/src/application/environment/environmentSource.ts +++ b/src/application/environment/environmentSource.ts @@ -44,22 +44,26 @@ export interface EnvironmentSource { readonly manifest: EnvironmentManifest; readonly manifestEvidenceId: string; readonly evidenceId: string; + /** SHA-256 of the captured artifact, or null when the capture excluded it from Evidence. */ + readonly captureSha256: string | null; /** Parent records cited by at least one tool, keyed by producer ordinal. */ readonly exchanges: ReadonlyMap; } const inputError = ( + operation: string, issues: readonly AnalysisInputIssue[], userMessage?: string, ): AnalysisInputError => new AnalysisInputError( - ENVIRONMENT_SERVE_OPERATION, + operation, userMessage === undefined ? undefined : { userMessage }, issues, ); /** Authenticate one Evidence file; schema failures name the file role in their paths. */ const parseEvidenceFile = ( + operation: string, role: "manifest" | "evidence", value: unknown, ): Result => { @@ -69,6 +73,7 @@ const parseEvidenceFile = ( if (!(cause instanceof z.ZodError)) throw cause; return err( inputError( + operation, projectInputIssues(cause.issues, value).map((issue) => ({ ...issue, path: [role, ...issue.path], @@ -121,12 +126,13 @@ const recordedExchange = ( /** Authenticate the manifest file and require the environment-manifest view kind. */ const parseManifestEvidence = ( + operation: string, value: unknown, ): Result< { readonly manifest: EnvironmentManifest; readonly evidenceId: string }, AnalysisError > => { - const evidence = parseEvidenceFile("manifest", value); + const evidence = parseEvidenceFile(operation, "manifest", value); if (!evidence.ok) return evidence; const view = analysisViewResultSchema.safeParse( evidence.value.normalized_result, @@ -138,6 +144,7 @@ const parseManifestEvidence = ( ) return err( inputError( + operation, [ { path: ["manifest", "normalized_result", "kind"], @@ -155,12 +162,17 @@ const parseManifestEvidence = ( /** Authenticate the evidence file and require complete capture Evidence. */ const parseCaptureEvidence = ( + operation: string, value: unknown, ): Result< - { readonly records: readonly WebNetworkCaptureRecord[]; evidenceId: string }, + { + readonly records: readonly WebNetworkCaptureRecord[]; + readonly evidenceId: string; + readonly captureSha256: string | null; + }, AnalysisError > => { - const evidence = parseEvidenceFile("evidence", value); + const evidence = parseEvidenceFile(operation, "evidence", value); if (!evidence.ok) return evidence; const capture = webNetworkCaptureSchema.safeParse( evidence.value.normalized_result, @@ -171,6 +183,7 @@ const parseCaptureEvidence = ( ) return err( inputError( + operation, [ { path: ["evidence", "operation"], @@ -186,6 +199,7 @@ const parseCaptureEvidence = ( return ok({ records: capture.data.records, evidenceId: evidence.value.evidence_id, + captureSha256: capture.data.artifact.sha256, }); }; @@ -212,19 +226,24 @@ const indexExchanges = ( return exchanges; }; -/** Load the manifest Evidence and its parent capture Evidence, refusing any pair that does not belong together. */ +/** + * Load the manifest Evidence and its parent capture Evidence, refusing any pair + * that does not belong together; refusals name the command that loaded them. + */ export const loadEnvironmentSource = ( manifestValue: unknown, evidenceValue: unknown, + operation: string = ENVIRONMENT_SERVE_OPERATION, ): Result => { - const manifest = parseManifestEvidence(manifestValue); + const manifest = parseManifestEvidence(operation, manifestValue); if (!manifest.ok) return manifest; - const capture = parseCaptureEvidence(evidenceValue); + const capture = parseCaptureEvidence(operation, evidenceValue); if (!capture.ok) return capture; const parentId = manifest.value.manifest.parent_evidence_id; if (parentId !== capture.value.evidenceId) return err( inputError( + operation, [ { path: ["evidence", "evidence_id"], @@ -240,6 +259,7 @@ export const loadEnvironmentSource = ( manifest: manifest.value.manifest, manifestEvidenceId: manifest.value.evidenceId, evidenceId: capture.value.evidenceId, + captureSha256: capture.value.captureSha256, exchanges: indexExchanges(manifest.value.manifest, capture.value.records), }); }; diff --git a/src/cli/environmentCommands.ts b/src/cli/environmentCommands.ts index 034d1cfbd1..afd6fa491b 100644 --- a/src/cli/environmentCommands.ts +++ b/src/cli/environmentCommands.ts @@ -1,6 +1,14 @@ +import { resolve } from "node:path"; + import { z } from "incur"; import type { Logger } from "pino"; +import { ENVIRONMENT_EXPORT_OPERATION } from "../application/environment/environmentBundle.js"; +import { + exportEnvironmentBundle, + readEnvironmentExportInputs, + type EnvironmentExportResult, +} from "../application/environment/environmentExport.js"; import { ENVIRONMENT_SERVE_OPERATION, loadEnvironmentSource, @@ -140,28 +148,121 @@ const serveEnvironmentCommand = async ( }; }; -/** Register the long-running agent environment server command. */ +interface ExportArguments { + readonly manifest: string; + readonly evidence: string; + readonly directory: string; + readonly packageTarball?: string | undefined; + readonly name?: string | undefined; +} + +/** The written files with sizes and the pinned package source, as the command prints them. */ +const exportDocument = (result: EnvironmentExportResult): JsonValue => ({ + status: result.status, + directory: result.directory, + name: result.name, + tool_count: result.tool_count, + evidence_id: result.evidence_id, + manifest_evidence_id: result.manifest_evidence_id, + package_source: result.package_source, + files: result.files.map(({ path, bytes }) => ({ path, bytes })), +}); + +/** Read each file once, authenticate the pair as the serve command does, then write the bundle from the same bytes. */ +const exportEnvironmentCommand = async ( + input: ExportArguments, +): Promise => { + const inputs = await readEnvironmentExportInputs( + input.manifest, + input.evidence, + ); + if (!inputs.ok) return cliError(inputs.error); + const source = loadEnvironmentSource( + inputs.value.manifest.value, + inputs.value.evidence.value, + ENVIRONMENT_EXPORT_OPERATION, + ); + if (!source.ok) return cliError(source.error); + const exported = await exportEnvironmentBundle({ + ...inputs.value, + source: source.value, + directory: resolve(input.directory), + name: input.name, + packageTarball: + input.packageTarball === undefined + ? undefined + : resolve(input.packageTarball), + }); + return exported.ok + ? exportDocument(exported.value) + : cliError(exported.error); +}; + +const manifestArgument = z + .string() + .describe( + "Environment manifest Evidence JSON path (inspect-analysis-view environment-manifest output)", + ); +const evidenceOption = z + .string() + .describe( + "Parent capture Evidence JSON path (inspect-web-network-capture output) the manifest cites", + ); + +/** Register the one-shot bundle export command. */ +const registerExportCommand = (cli: CliInstance, logger: Logger): void => { + cli.command(CLI_COMMANDS.environmentExport, { + description: + "Export an environment manifest and its capture Evidence as an agent environment bundle folder the agentenv framework runs with agent-env run", + args: z.object({ + manifest: manifestArgument, + directory: z + .string() + .describe( + "Bundle directory to create; must be absent or empty, nothing is overwritten", + ), + }), + options: z.object({ + evidence: evidenceOption, + packageTarball: z + .string() + .optional() + .describe( + "Packed rea-agents tarball (npm pack) copied into the env folder and installed by the Dockerfile instead of the registry package pinned to this version", + ), + name: z + .string() + .optional() + .describe( + "Env folder and env id (kebab-case); default the manifest's name", + ), + }), + run: ({ args, options }) => + logCliCommand(logger, CLI_COMMANDS.environmentExport, () => + exportEnvironmentCommand({ + manifest: args.manifest, + evidence: options.evidence, + directory: args.directory, + packageTarball: options.packageTarball, + name: options.name, + }), + ), + }); +}; + +/** Register the long-running agent environment server command and the bundle export. */ export const registerEnvironmentCommands = ( cli: CliInstance, logger: Logger, environment: Readonly, ): void => { + registerExportCommand(cli, logger); cli.command(CLI_COMMANDS.environmentServe, { description: "Serve an environment manifest as an agent environment that replays retained capture records over streamable HTTP until SIGINT or SIGTERM", - args: z.object({ - manifest: z - .string() - .describe( - "Environment manifest Evidence JSON path (inspect-analysis-view environment-manifest output)", - ), - }), + args: z.object({ manifest: manifestArgument }), options: z.object({ - evidence: z - .string() - .describe( - "Parent capture Evidence JSON path (inspect-web-network-capture output) the manifest cites", - ), + evidence: evidenceOption, host: z .string() .min(1) diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 84abeb9dbe..2341bb527a 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -123,6 +123,7 @@ export const CLI_COMMANDS = Object.freeze({ evidenceImport: "evidence-import", evidenceExport: "evidence-export", environmentServe: "environment-serve", + environmentExport: "environment-export", compare: "compare", captureProcess: "capture-process", compareProcessCaptures: "compare-process-captures", diff --git a/tests/boundary/cli/environmentExportCli.test.ts b/tests/boundary/cli/environmentExportCli.test.ts new file mode 100644 index 0000000000..a9029349da --- /dev/null +++ b/tests/boundary/cli/environmentExportCli.test.ts @@ -0,0 +1,162 @@ +import { mkdir, readdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect } from "vitest"; + +import { + analysisViewCaptureEvidence, + analysisViewCaptureFixture, +} from "../../fixtures/analysisView.js"; +import { + environmentServeFixture, + writeEnvironmentServeFiles, +} from "../../fixtures/environmentServe.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const BUNDLE_FILES = [ + "README.md", + "artifacts/check_grafana-example/verify.py", + "envs/grafana-example/Dockerfile", + "envs/grafana-example/env.toml", + "envs/grafana-example/evidence.json", + "envs/grafana-example/manifest.json", + "tasks/smoke.json", +]; + +describe("rea environment-export", () => { + cliTest( + "writes the bundle folder from the fixture pair and prints the written files with sizes", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-environment-export-cli-"); + const files = await writeEnvironmentServeFiles(root); + const directory = join(root, "bundle"); + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + files.capturePath, + directory, + "--json", + ], + timeoutMs: 30_000, + }); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + status: "exported", + directory, + name: "grafana-example", + tool_count: 19, + package_source: { kind: "registry" }, + }); + const document = result.json as { + files: { path: string; bytes: number }[]; + }; + expect(document.files.map((file) => file.path).sort()).toEqual( + BUNDLE_FILES, + ); + const written = ( + await readdir(directory, { recursive: true, withFileTypes: true }) + ) + .filter((entry) => entry.isFile()) + .map((entry) => + join(entry.parentPath, entry.name).slice(directory.length + 1), + ) + .sort(); + expect(written).toEqual(BUNDLE_FILES); + expect( + await readFile(join(directory, "envs/grafana-example/manifest.json")), + ).toEqual(await readFile(files.manifestPath)); + expect( + await readFile(join(directory, "envs/grafana-example/evidence.json")), + ).toEqual(await readFile(files.capturePath)); + const dockerfile = await readFile( + join(directory, "envs/grafana-example/Dockerfile"), + "utf8", + ); + expect(dockerfile).toMatch(/^FROM node:24\.18\.0-bookworm-slim\n/u); + expect(dockerfile).toContain('CMD ["rea","environment-serve"'); + }, + ); + + cliTest( + "refuses a non-empty directory with a typed invalid request and leaves it unchanged", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-environment-export-busy-", + ); + const files = await writeEnvironmentServeFiles(root); + const directory = join(root, "occupied"); + await mkdir(directory); + await writeFile(join(directory, "notes.txt"), "keep"); + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + files.capturePath, + directory, + "--json", + ], + timeoutMs: 30_000, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + error: "Application workflow failed", + code: "invalid_request", + category: "invalid_input", + details: { + operation: "environment-export", + issues: [{ path: ["directory"], reason: "invalid_value" }], + }, + }); + expect(result.stdout).toContain("not empty"); + expect(await readdir(directory)).toEqual(["notes.txt"]); + }, + ); + + cliTest( + "refuses a manifest whose parent is not the supplied evidence before creating the directory", + async ({ cli }) => { + const root = await createTestTempDirectory( + "rea-environment-export-mismatch-", + ); + const files = await writeEnvironmentServeFiles( + root, + environmentServeFixture(), + ); + const other = join(root, "other.json"); + await writeFile( + other, + JSON.stringify( + analysisViewCaptureEvidence(analysisViewCaptureFixture()), + ), + ); + const directory = join(root, "bundle"); + const result = await cli.run({ + arguments: [ + "environment-export", + files.manifestPath, + "--evidence", + other, + directory, + "--json", + ], + timeoutMs: 30_000, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + error: "Application workflow failed", + code: "invalid_request", + details: { + operation: "environment-export", + issues: [{ path: ["evidence", "evidence_id"] }], + }, + }); + expect(result.stdout).toContain("different capture"); + expect(await readdir(root)).not.toContain("bundle"); + }, + ); +}); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index cb8519f705..d930642b95 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -792,9 +792,27 @@ it.each([ "src/domain/analysisErrorProjection.ts", { code: true, sqlite: true, runtime: true }, ], + [ + "src/application/environment/environmentExport.ts", + { + code: true, + runtime: true, + agentenv: true, + browser: false, + package: false, + }, + ], + [ + "scripts/verify-agentenv-environment.mjs", + { code: true, agentenv: true, runtime: false, package: false }, + ], + [ + ".github/workflows/real-agentenv.yml", + { code: false, agentenv: true, workflows: true, package: false }, + ], [ "src/domain/javascript/example.ts", - { code: true, sqlite: false, ghidra_bridge: false }, + { code: true, sqlite: false, ghidra_bridge: false, agentenv: false }, ], ])("selects the appropriate lanes for %s", async (path, expected) => { const { directory, base } = await fixture(); @@ -893,6 +911,7 @@ it.each([ ["web_network", "real-web-network-captures"], ["web_runtime", "real-web-runtime"], ["web_source_map", "real-web-source-map"], + ["agentenv", "real-agentenv"], ["nativeaot", "nativeaot-fixtures"], ["ghidra_bridge", "ghidra-bridge-sources"], ["website", "website-check"], diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 457d532560..81fb54bc54 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -219,6 +219,7 @@ it("binds provider build reuse to the current run while keeping standalone dispa "web_network", "web_runtime", "web_source_map", + "agentenv", ]; const buildTerms = z .string() @@ -443,6 +444,7 @@ it.each([ ["sqlite", "sqlite-runtime"], ["workflows", "static"], ["ghidra_bridge", "ghidra_bridge"], + ["agentenv", "agentenv"], ] as const) { // GitHub exposes only direct dependencies in the required job's needs context. const needs: Record< diff --git a/tests/conformance/package/environmentBundlePackagePin.test.ts b/tests/conformance/package/environmentBundlePackagePin.test.ts new file mode 100644 index 0000000000..35f9a79963 --- /dev/null +++ b/tests/conformance/package/environmentBundlePackagePin.test.ts @@ -0,0 +1,40 @@ +import { readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { + bundleDockerfile, + registryPackageSource, +} from "../../../src/application/environment/environmentBundle.js"; +import { PRODUCT_IDENTITY } from "../../../src/identity.js"; + +/** + * The exported Dockerfile installs the CLI that produced the bundle, so the + * pin must be the exact package version of this build, read from the package + * manifest rather than restated anywhere in source. + */ +describe("environment bundle package pin", () => { + it("pins the registry install to the package.json version", async () => { + const manifest = z + .object({ name: z.literal("rea-agents"), version: z.string().min(1) }) + .parse( + JSON.parse( + await readFile( + new URL("../../../package.json", import.meta.url), + "utf8", + ), + ), + ); + const dockerfile = bundleDockerfile(registryPackageSource()); + const install = dockerfile + .split("\n") + .find((line) => line.startsWith("RUN npm install -g")); + expect(install).toBeDefined(); + expect(install?.endsWith(` ${manifest.name}@${manifest.version}`)).toBe( + true, + ); + expect(PRODUCT_IDENTITY.packageVersion).toBe(manifest.version); + expect(dockerfile).not.toMatch(/rea-agents@(?:latest|\^|~)/u); + }); +});