From ddcbb13b8473a3fd0d87cc642d68cabbc2e36e85 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sun, 18 Oct 2026 20:52:36 -0700 Subject: [PATCH] fix(ci): one deadline for both bundle attempts and a container inventory on every outcome Each attempt had its own 15-minute timeout, so a quota retry could outlast the 30-minute job before the failed receipt was written. Both attempts now share a 22-minute budget and the retry runs only with five minutes to spare. The container inventory ran only after a successful run; it now runs in finally, so a failed or timed-out receipt still lists what the run left behind. Co-Authored-By: Claude Opus 5.5 --- docs/testing.md | 13 +++-- scripts/verify-agentenv-environment.mjs | 63 ++++++++++++++++--------- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 9b4e80bf41..f2687635aa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1565,12 +1565,14 @@ network for `pip` on the runner and for `npm` inside the image build. Both variables are required; the verifier resolves nothing from `PATH` and installs nothing. Its disposable CI runner creates the virtual environment in runner storage, exports both paths and uses the runner's own Docker; no developer host -configuration changes. The `agent-env run` itself is bounded to 15 minutes -inside the 30-minute job, so a hung run ends with the verifier's failed receipt -rather than the runner's kill. The framework builds its gateway and service-db +configuration changes. `agent-env run` attempts share one 22-minute budget +inside the 30-minute job (the first attempt at most 15 minutes), so a hung or +retried run ends with the verifier's failed receipt rather than the runner's +kill. The framework builds its gateway and service-db images from `public.ecr.aws` on every fresh runner, and hosted runners share that registry's anonymous pull quota; when the run fails with that registry's -`429 Too Many Requests`, the verifier waits 90 seconds and runs once more, +`429 Too Many Requests` and at least five minutes of the budget would remain +after the pause, the verifier waits 90 seconds and runs once more, recording `attempts: 2` and `retried_for` in the receipt. Any other failure ends the lane at once. @@ -1582,7 +1584,8 @@ registry package, and runs `agent-env run` on the bundle. It checks that the export reports the planned file set with the inputs copied byte-exact, that the runner exits `0` with `tasks/smoke.json` reported as passed and the verifier's description naming the tool count, and that no container exists -after the run that did not exist before. The one exception is the framework's +after the run that did not exist before; the container inventory runs even when +the run fails or times out, so a failed receipt still lists what it left. The one exception is the framework's own image-store helper, `agentenv-registry` (or `agentenv-registry-`), a `registry:2` container the local provider starts on first use and keeps running across runs; a first run on a fresh host creates it, so the receipt diff --git a/scripts/verify-agentenv-environment.mjs b/scripts/verify-agentenv-environment.mjs index 15f3213c83..a5b12cdfd8 100644 --- a/scripts/verify-agentenv-environment.mjs +++ b/scripts/verify-agentenv-environment.mjs @@ -20,6 +20,10 @@ const AGENTENV_PIN = "agentenv-framework 0.9.1334"; const PUBLIC_ECR_QUOTA = /public\.ecr\.aws[\s\S]{0,400}?(?:429 Too Many Requests|toomanyrequests)/u; const QUOTA_RETRY_PAUSE_MS = 90_000; +// Both bundle attempts share this budget inside the 30-minute job; a retry needs real time left. +const RUN_BUDGET_MS = 22 * 60_000; +const FIRST_ATTEMPT_MS = 15 * 60_000; +const MIN_RETRY_MS = 5 * 60_000; const TEMPLATE_RULES = [ "after-uid-literal", "opaque-token", @@ -52,9 +56,10 @@ const environment = Object.fromEntries( const failures = []; let cases = 0; const receipt = {}; +let knownContainers; try { const containersBefore = await containers(); - const knownContainers = new Set(containersBefore.map((entry) => entry.id)); + knownContainers = new Set(containersBefore.map((entry) => entry.id)); receipt.docker_before = { containers: containersBefore.length, images: (await imageIds()).length, @@ -146,9 +151,11 @@ try { assert.ok(dockerfile.includes("/tmp/rea-agents.tgz")); cases++; // The agentenv runner builds the image, deploys it on the local sandbox and runs the verifier. - // Bounded well under the 30-minute job so a hung run ends with this receipt, not the runner's kill. + // One deadline for both attempts, well under the 30-minute job, so a hung or retried run ends + // with this receipt rather than the runner's kill. const started = Date.now(); - const runBundle = () => + const deadline = started + RUN_BUDGET_MS; + const runBundle = (timeoutMs) => runOwnedCommand( { command: agentEnv, @@ -157,21 +164,24 @@ try { runId: `rea-agentenv-run-${randomUUID()}`, hostEnvironment: environment, }, - { timeoutMs: 15 * 60_000, diagnosticBytes: 4 * 1024 * 1024 }, + { timeoutMs, diagnosticBytes: 4 * 1024 * 1024 }, ); let attempts = 1; let snapshot; try { - snapshot = await runBundle(); + snapshot = await runBundle( + Math.min(FIRST_ATTEMPT_MS, deadline - Date.now()), + ); } catch (cause) { if ( !(cause instanceof OwnedCommandFailure) || - !PUBLIC_ECR_QUOTA.test(cause.snapshot?.stderr.text ?? "") + !PUBLIC_ECR_QUOTA.test(cause.snapshot?.stderr.text ?? "") || + deadline - Date.now() - QUOTA_RETRY_PAUSE_MS < MIN_RETRY_MS ) throw cause; attempts = 2; await pause(QUOTA_RETRY_PAUSE_MS); - snapshot = await runBundle(); + snapshot = await runBundle(deadline - Date.now()); } receipt.agent_env_run = { exit_code: snapshot.exitCode, @@ -192,23 +202,34 @@ try { ), ); cases++; - const containersAfter = await containers(); - const added = containersAfter.filter( - (entry) => !knownContainers.has(entry.id), - ); - const helpers = added.filter((entry) => IMAGE_STORE_HELPER.test(entry.name)); - const leaked = added.filter((entry) => !IMAGE_STORE_HELPER.test(entry.name)); - receipt.docker_after = { - containers: containersAfter.length, - images: (await imageIds()).length, - image_store_helpers: helpers.map((entry) => entry.name), - new_containers: leaked.map((entry) => `${entry.id} ${entry.name}`), - }; - assert.deepEqual(leaked, []); - cases++; } catch (cause) { failures.push(cause); } finally { + // The container inventory runs whether the bundle passed, failed or timed out, so a failed + // receipt still says what the run left behind. + if (knownContainers !== undefined) + try { + const containersAfter = await containers(); + const added = containersAfter.filter( + (entry) => !knownContainers.has(entry.id), + ); + const helpers = added.filter((entry) => + IMAGE_STORE_HELPER.test(entry.name), + ); + const leaked = added.filter( + (entry) => !IMAGE_STORE_HELPER.test(entry.name), + ); + receipt.docker_after = { + containers: containersAfter.length, + images: (await imageIds()).length, + image_store_helpers: helpers.map((entry) => entry.name), + new_containers: leaked.map((entry) => `${entry.id} ${entry.name}`), + }; + assert.deepEqual(leaked, []); + if (failures.length === 0) cases++; + } catch (cause) { + failures.push(cause); + } try { await runtime.close(); } catch (cause) {