From 171da5591ea159dc5aa66c749ec092d1bdddb943 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 19 Oct 2026 10:53:01 -0700 Subject: [PATCH] fix(ci): record every agentenv run and retry only public ECR rate refusals The receipt's agent_env_run is recorded in a finally block, so a failed receipt says how the last run ended, how many runs there were and which public ECR refusal ended it: public_ecr_quota is data_limit for the per-IP data-transfer cap and rate for a request-rate refusal. Only a rate refusal earns the one retry; the data cap fails the lane at once, since a pause on the same runner cannot lift it. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- docs/testing.md | 10 ++- scripts/verify-agentenv-environment.mjs | 83 +++++++++++++++++-------- 2 files changed, 65 insertions(+), 28 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f50f3e83df..449c762766 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1579,9 +1579,13 @@ inside the 30-minute job (the first attempt at most 15 minutes), so a hung or retried run ends with the verifier's failed receipt rather than the runner's kill. The framework builds its gateway and service-db images from `public.ecr.aws` on every fresh runner, and hosted runners share -that registry's anonymous pull quota; when the run fails with that registry's -`429 Too Many Requests` and at least five minutes of the budget would remain -after the pause, the verifier waits 90 seconds and runs once more, +that registry's anonymous pull quota. The receipt's `agent_env_run` records the +last run whether it passed or failed, with `public_ecr_quota` set to +`data_limit` or `rate` when that registry's `429 Too Many Requests` ended it. +The data-transfer cap (`toomanyrequests: Data limit exceeded`) is counted per +source IP, so it fails the lane at once: a pause on the same runner cannot lift +it. When a rate refusal ends the run and at least five minutes of the budget +would remain after the pause, the verifier waits 90 seconds and runs once more, recording `attempts: 2` and `retried_for` in the receipt. Any other failure ends the lane at once. diff --git a/scripts/verify-agentenv-environment.mjs b/scripts/verify-agentenv-environment.mjs index a5b12cdfd8..97e6778486 100644 --- a/scripts/verify-agentenv-environment.mjs +++ b/scripts/verify-agentenv-environment.mjs @@ -16,9 +16,12 @@ import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; const AGENTENV_PIN = "agentenv-framework 0.9.1334"; // The framework builds its infra images from public ECR, whose anonymous pull quota hosted -// runners share; that one external refusal earns one retry, anything else fails at once. +// runners share; a rate refusal earns one retry, anything else fails at once. const PUBLIC_ECR_QUOTA = /public\.ecr\.aws[\s\S]{0,400}?(?:429 Too Many Requests|toomanyrequests)/u; +// The anonymous data-transfer cap is counted per source IP, so a pause on the same runner +// cannot lift it; the receipt names it instead of retrying. +const PUBLIC_ECR_DATA_LIMIT = /toomanyrequests: Data limit exceeded/u; const QUOTA_RETRY_PAUSE_MS = 90_000; // Both bundle attempts share this budget inside the 30-minute job; a retry needs real time left. const RUN_BUDGET_MS = 22 * 60_000; @@ -155,18 +158,36 @@ try { // with this receipt rather than the runner's kill. const started = Date.now(); const deadline = started + RUN_BUDGET_MS; - const runBundle = (timeoutMs) => - runOwnedCommand( - { - command: agentEnv, - arguments: ["run", bundle, "--sandbox", "local"], - cwd: runtime.path, - runId: `rea-agentenv-run-${randomUUID()}`, - hostEnvironment: environment, - }, - { timeoutMs, diagnosticBytes: 4 * 1024 * 1024 }, - ); - let attempts = 1; + let attempts = 0; + const runBundle = async (timeoutMs) => { + attempts += 1; + let outcome; + try { + outcome = await runOwnedCommand( + { + command: agentEnv, + arguments: ["run", bundle, "--sandbox", "local"], + cwd: runtime.path, + runId: `rea-agentenv-run-${randomUUID()}`, + hostEnvironment: environment, + }, + { timeoutMs, diagnosticBytes: 4 * 1024 * 1024 }, + ); + return outcome; + } catch (cause) { + outcome = + cause instanceof OwnedCommandFailure ? cause.snapshot : undefined; + throw cause; + } finally { + // Recorded before the outcome is judged, so a failed receipt still says how the last + // run ended, how many runs there were and whether the registry quota refused it. + receipt.agent_env_run = runRecord( + outcome, + Date.now() - started, + attempts, + ); + } + }; let snapshot; try { snapshot = await runBundle( @@ -174,24 +195,13 @@ try { ); } catch (cause) { if ( - !(cause instanceof OwnedCommandFailure) || - !PUBLIC_ECR_QUOTA.test(cause.snapshot?.stderr.text ?? "") || + receipt.agent_env_run?.public_ecr_quota !== "rate" || deadline - Date.now() - QUOTA_RETRY_PAUSE_MS < MIN_RETRY_MS ) throw cause; - attempts = 2; await pause(QUOTA_RETRY_PAUSE_MS); snapshot = await runBundle(deadline - Date.now()); } - receipt.agent_env_run = { - exit_code: snapshot.exitCode, - seconds: Math.round((Date.now() - started) / 1000), - attempts, - ...(attempts > 1 - ? { retried_for: "public ECR anonymous pull quota (429)" } - : {}), - stdout_tail: snapshot.stdout.text.trim().split("\n").slice(-6), - }; assert.equal(snapshot.exitCode, 0); assert.match(snapshot.stdout.text, /tasks\/smoke\.json v\d+: passed/u); assert.match( @@ -281,6 +291,29 @@ async function cli(...args) { return JSON.parse(stdout); } +// The receipt's view of one `agent-env run`, passed or failed; `outcome` is absent when the +// command failed before producing a snapshot. +function runRecord(outcome, elapsedMs, attempts) { + const quota = publicEcrQuota(outcome?.stderr.text ?? ""); + return { + exit_code: outcome?.exitCode ?? null, + ...(outcome?.signal ? { signal: outcome.signal } : {}), + seconds: Math.round(elapsedMs / 1000), + attempts, + ...(attempts > 1 + ? { retried_for: "public ECR anonymous pull rate limit (429)" } + : {}), + ...(quota === undefined ? {} : { public_ecr_quota: quota }), + stdout_tail: (outcome?.stdout.text ?? "").trim().split("\n").slice(-6), + }; +} + +// Which public ECR refusal ended a run: the per-IP data cap or a request-rate refusal. +function publicEcrQuota(stderr) { + if (!PUBLIC_ECR_QUOTA.test(stderr)) return undefined; + return PUBLIC_ECR_DATA_LIMIT.test(stderr) ? "data_limit" : "rate"; +} + // npm 11 prints `npm pack --json` as a one-element array; npm 12 keys the same record by package name. function packedFilename(report) { const record = Array.isArray(report) ? report[0] : Object.values(report)[0];