From a2a952dad970f61ef4ff915080eaf8199e7ec38f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 18 Oct 2026 09:12:26 -0700 Subject: [PATCH] fix(environment): retry the bundle run once on public ECR's anonymous pull quota Hosted runners share that registry's quota, and the framework builds its infra images from it on every fresh runner; a 429 from it earns one retry after 90 seconds, recorded in the receipt, while any other failure ends the lane at once. Co-Authored-By: Claude Fable 5.1 --- .../environment-export/verification.md | 8 ++- docs/testing.md | 7 ++- scripts/verify-agentenv-environment.mjs | 50 +++++++++++++++---- 3 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/specs/agent-environments/environment-export/verification.md b/docs/specs/agent-environments/environment-export/verification.md index b45be682ea..9b13a6d832 100644 --- a/docs/specs/agent-environments/environment-export/verification.md +++ b/docs/specs/agent-environments/environment-export/verification.md @@ -183,7 +183,13 @@ sandbox.") but the first form of the leak check counted the helper and failed. The check now lists containers with their names, excludes `agentenv-registry[-]` and reports it under `image_store_helpers`; the local receipt gained `image_store_helpers: []` because the helper already -existed before the run here. Removing that helper (`docker rm -f +existed before the run here. Public ECR's anonymous pull quota refused the +framework's `postgres:16-alpine` pull with `429 Too Many Requests, Data limit +exceeded` on two hosted runs out of six (the second and, after the stack was +rebuilt, the sixth); the verifier now retries the run once, after 90 seconds, +when the failure carries that registry's signature, and records +`attempts: 2` with `retried_for` in the receipt. The local receipt reports +`attempts: 1`. Removing that helper (`docker rm -f agentenv-registry`) and running the verifier again reproduced the fresh-host path locally: the run recreated it and the receipt passed with `image_store_helpers: ["agentenv-registry"]`, `new_containers: []` diff --git a/docs/testing.md b/docs/testing.md index 3c841dd18e..9b4e80bf41 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1567,7 +1567,12 @@ nothing. Its disposable CI runner creates the virtual environment in runner storage, exports both paths and uses the runner's own Docker; no developer host configuration changes. The `agent-env run` itself is bounded to 15 minutes inside the 30-minute job, so a hung run ends with the verifier's failed receipt -rather than the runner's kill. +rather than the runner's kill. The framework builds its gateway and service-db +images from `public.ecr.aws` on every fresh runner, and hosted runners share +that registry's anonymous pull quota; when the run fails with that registry's +`429 Too Many Requests`, the verifier waits 90 seconds and runs once more, +recording `attempts: 2` and `retried_for` in the receipt. Any other failure +ends the lane at once. The verifier writes the Grafana-shaped HAR the view tests own, inspects it and projects the `environment-manifest` view with all three template rules through diff --git a/scripts/verify-agentenv-environment.mjs b/scripts/verify-agentenv-environment.mjs index 882170bc51..15f3213c83 100644 --- a/scripts/verify-agentenv-environment.mjs +++ b/scripts/verify-agentenv-environment.mjs @@ -4,13 +4,22 @@ import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import { access, readFile, writeFile } from "node:fs/promises"; import { dirname, isAbsolute, join, resolve } from "node:path"; +import { setTimeout as pause } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js"; -import { runOwnedCommand } from "../dist/process/OwnedCommand.js"; +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../dist/process/OwnedCommand.js"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const AGENTENV_PIN = "agentenv-framework 0.9.1334"; +// The framework builds its infra images from public ECR, whose anonymous pull quota hosted +// runners share; that one external refusal earns one retry, anything else fails at once. +const PUBLIC_ECR_QUOTA = + /public\.ecr\.aws[\s\S]{0,400}?(?:429 Too Many Requests|toomanyrequests)/u; +const QUOTA_RETRY_PAUSE_MS = 90_000; const TEMPLATE_RULES = [ "after-uid-literal", "opaque-token", @@ -139,19 +148,38 @@ try { // The agentenv runner builds the image, deploys it on the local sandbox and runs the verifier. // Bounded well under the 30-minute job so a hung run ends with this receipt, not the runner's kill. const started = Date.now(); - const snapshot = await runOwnedCommand( - { - command: agentEnv, - arguments: ["run", bundle, "--sandbox", "local"], - cwd: runtime.path, - runId: `rea-agentenv-run-${randomUUID()}`, - hostEnvironment: environment, - }, - { timeoutMs: 15 * 60_000, diagnosticBytes: 4 * 1024 * 1024 }, - ); + const runBundle = () => + runOwnedCommand( + { + command: agentEnv, + arguments: ["run", bundle, "--sandbox", "local"], + cwd: runtime.path, + runId: `rea-agentenv-run-${randomUUID()}`, + hostEnvironment: environment, + }, + { timeoutMs: 15 * 60_000, diagnosticBytes: 4 * 1024 * 1024 }, + ); + let attempts = 1; + let snapshot; + try { + snapshot = await runBundle(); + } catch (cause) { + if ( + !(cause instanceof OwnedCommandFailure) || + !PUBLIC_ECR_QUOTA.test(cause.snapshot?.stderr.text ?? "") + ) + throw cause; + attempts = 2; + await pause(QUOTA_RETRY_PAUSE_MS); + snapshot = await runBundle(); + } receipt.agent_env_run = { exit_code: snapshot.exitCode, seconds: Math.round((Date.now() - started) / 1000), + attempts, + ...(attempts > 1 + ? { retried_for: "public ECR anonymous pull quota (429)" } + : {}), stdout_tail: snapshot.stdout.text.trim().split("\n").slice(-6), }; assert.equal(snapshot.exitCode, 0);