From d137ffa76232c45c740b5fdf5e47f6341b1f13ca Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 20 Nov 2026 10:05:34 -0500 Subject: [PATCH] test(agentenv): consolidate verification, capture and cleanup checks (#12855) * test(agentenv): consolidate solve verification and cleanup checks * test(agentenv): verify capture argument delivery and retention together * test: update lifecycle registry after campaign test renames --- bridge/agentenv/test_capture.py | 26 ++++---- bridge/agentenv/test_solve.py | 18 +++--- bridge/agentenv/test_solve_cleanup.py | 70 +++++++++------------- tests/support/providerLifecycleCoverage.ts | 4 +- 4 files changed, 53 insertions(+), 65 deletions(-) diff --git a/bridge/agentenv/test_capture.py b/bridge/agentenv/test_capture.py index f9753d729b..ef43014c26 100644 --- a/bridge/agentenv/test_capture.py +++ b/bridge/agentenv/test_capture.py @@ -1,5 +1,6 @@ """Exercise the capture controller's filesystem inventory boundary.""" +import hashlib import json import io from contextlib import redirect_stdout @@ -82,7 +83,7 @@ class LongRunTest(unittest.TestCase): rea = self.bin / "rea" rea.write_text( f"#!{sys.executable}\n" - "import os, signal, subprocess, sys, time\n" + "import hashlib, json, os, signal, subprocess, sys, time\n" "signal.signal(signal.SIGINT, lambda *_: sys.exit(130))\n" "if len(sys.argv) > 1 and sys.argv[1] == 'wait':\n" " open(sys.argv[2], 'w').close()\n" @@ -93,6 +94,7 @@ class LongRunTest(unittest.TestCase): " while not os.path.exists(sys.argv[3]): time.sleep(0.02)\n" "if len(sys.argv) > 1 and sys.argv[1] == 'stderr':\n" " sys.stderr.write('x' * 100000 + 'TAIL')\n" + " print(hashlib.sha256(json.dumps(sys.argv[2:]).encode()).hexdigest())\n" "if len(sys.argv) > 1 and sys.argv[1] == 'hold':\n" " held = subprocess.Popen([sys.executable, '-c', 'import os,sys,time\\nwhile not os.path.exists(sys.argv[1]): time.sleep(0.02)', sys.argv[2]], stdout=subprocess.DEVNULL, stderr=sys.stderr, start_new_session=True)\n" " open(sys.argv[2] + '.pid.tmp', 'w').write(str(held.pid))\n" @@ -199,17 +201,21 @@ class LongRunTest(unittest.TestCase): self.assertEqual(result["exit_code"], 0) self.assertEqual(self.output.read_text().strip(), "no token") - def test_passes_large_command_without_persisting_its_arguments(self): - self.assertEqual(self.begin(["done", *("x" * 4000 for _ in range(20))]), {"started": True}) - result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) - self.assertEqual(result["exit_code"], 0) - self.assertFalse((self.job / "request.json").exists()) + def test_delivers_large_arguments_without_persisting_them_and_bounds_stderr(self): + canary = "large-command-last-argument" + arguments = [f"{index}-" + "x" * 4000 for index in range(20)] + [canary] + digest = hashlib.sha256(json.dumps(arguments).encode()).hexdigest() + self.assertEqual(self.begin(["stderr", *arguments]), {"started": True}) - def test_keeps_only_a_bounded_stderr_tail(self): - self.assertEqual(self.begin(["stderr"]), {"started": True}) result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5) - self.assertEqual(len(result["stderr"]), 4096) - self.assertTrue(result["stderr"].endswith("TAIL")) + self.assertTrue(result["finished"]) + self.assertEqual(result["exit_code"], 0) + self.assertEqual(self.output.read_text().splitlines(), [digest, "no token"]) + self.assertEqual(result["stderr"], "x" * 4092 + "TAIL") + self.assertFalse((self.job / "request.json").exists()) + for path in self.job.rglob("*"): + if path.is_file(): + self.assertNotIn(canary.encode(), path.read_bytes(), str(path)) def test_cancels_a_running_child_and_confirms_exit(self): ready = self.root / "ready" diff --git a/bridge/agentenv/test_solve.py b/bridge/agentenv/test_solve.py index fdc3c79eae..e33ebacec4 100644 --- a/bridge/agentenv/test_solve.py +++ b/bridge/agentenv/test_solve.py @@ -12,14 +12,6 @@ def run_with(verifications, outcome="passed"): class SolveFactsTest(unittest.TestCase): - def test_matching_verifier_success_and_failure(self): - for passed, score, outcome in ((True, 1.0, "passed"), (False, 0.0, "scored below 1")): - with self.subTest(passed=passed): - run = run_with({"wanted": {"results": [{"id": "selected", "result": passed}], - "score": score}}) - self.assertEqual(_solve_facts(run, "wanted"), - (outcome, [{"id": "selected", "passed": passed, "description": ""}])) - def test_missing_empty_and_unscored_selected_verifier_do_not_borrow_success(self): other = {"results": [{"id": "unrelated", "result": True}], "score": 1.0} for selected in (None, {}, {"results": [], "score": 1.0}, @@ -44,12 +36,16 @@ class SolveFactsTest(unittest.TestCase): checks = [{"id": "0", "passed": False, "description": ""}, {"id": "zero", "passed": False, "description": ""}, {"id": "yes", "passed": True, "description": ""}] - for score, expected in ((0, "scored below 1"), (0.5, "scored below 1"), - (1, "passed"), (1.5, "passed")): + for score, prior, expected in ( + (0, "passed", "scored below 1"), + (0.5, "passed", "scored below 1"), + (1, "scored below 1", "passed"), + (1.5, "scored below 1", "passed"), + ): with self.subTest(score=score): verifications = {"other": {"score": 0}, "wanted": {"results": rows, "score": score}} - self.assertEqual(_solve_facts(run_with(verifications, "scored below 1"), "wanted"), + self.assertEqual(_solve_facts(run_with(verifications, prior), "wanted"), (expected, checks)) def test_failure_and_cancellation_preserve_partial_named_checks(self): diff --git a/bridge/agentenv/test_solve_cleanup.py b/bridge/agentenv/test_solve_cleanup.py index d0ebdd453b..02e551728b 100644 --- a/bridge/agentenv/test_solve_cleanup.py +++ b/bridge/agentenv/test_solve_cleanup.py @@ -14,15 +14,15 @@ class SolveCleanupTest(unittest.TestCase): self.temp = tempfile.TemporaryDirectory() self.addCleanup(self.temp.cleanup) self.root = Path(self.temp.name) - self.folder = self.root / "agent-env-local-a1b2c3d4-abcdef12" - self.folder.mkdir() - self.terminated = [] + self.folders = {} + self.folder = self._add_sandbox("local-a1b2c3d4") + self.events = [] self.module = types.ModuleType("agent_env.providers.sandbox_providers.local_docker.provider") test = self class Provider: async def terminate_sandbox(self, sandbox_id): - test.terminated.append(sandbox_id) + test.events.append(("terminate", sandbox_id)) self.module.LocalSandboxProvider = Provider self.module.local_sandbox_ids = self._listed @@ -32,52 +32,47 @@ class SolveCleanupTest(unittest.TestCase): self.patch_module.start() self.addCleanup(self.patch_module.stop) + def _add_sandbox(self, sandbox_id): + folder = self.root / f"agent-env-{sandbox_id}-abcdef12" + folder.mkdir() + self.folders[sandbox_id] = folder + return folder + def _listed(self): - # As agentenv lists them: `agent-env--` folders, whatever the id's shape. - found = set() - for entry in self.root.iterdir(): - sandbox_id, _, suffix = entry.name.removeprefix("agent-env-").rpartition("-") - if entry.name.startswith("agent-env-") and sandbox_id and suffix and entry.is_dir(): - found.add(sandbox_id) - return sorted(found) + return [sandbox_id for sandbox_id, folder in self.folders.items() if folder.is_dir()] def _remove(self, sandbox_id): - folder = next(self.root.glob(f"agent-env-{sandbox_id}-*")) + self.events.append(("remove", sandbox_id)) + folder = self.folders[sandbox_id] folder.rmdir() return folder def run_cleanup(self): asyncio.run(solve_cleanup.cleanup(self.root)) - def test_terminates_owned_sandbox_before_removing_its_folder(self): - with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): - self.run_cleanup() - self.assertEqual(self.terminated, ["local-a1b2c3d4"]) - self.assertFalse(self.folder.exists()) + def test_terminates_before_removing_and_retries_only_the_retained_sandbox(self): + blocked_id = "local-f1223344e5" + blocked = self._add_sandbox(blocked_id) - def test_retains_folder_when_docker_reports_container_then_retries(self): - with patch.object(solve_cleanup, "_docker", return_value="still-running"), patch.object(solve_cleanup.sys, "platform", "linux"): - with self.assertRaisesRegex(RuntimeError, "still has its agent container"): - self.run_cleanup() - self.assertTrue(self.folder.exists()) - with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): - self.run_cleanup() - self.assertFalse(self.folder.exists()) - self.assertEqual(len(self.terminated), 2) - - def test_partial_cleanup_and_late_folder_are_retried(self): - later = self.root / "agent-env-local-f1223344-abcdef12" - later.mkdir() def docker(*args): - return "still-running" if "local-f1223344" in " ".join(args) else "" + return "still-running" if args[-1] == f"name=^agent-{blocked_id}$" else "" + with patch.object(solve_cleanup, "_docker", side_effect=docker), patch.object(solve_cleanup.sys, "platform", "linux"): with self.assertRaisesRegex(RuntimeError, "still has its agent container"): self.run_cleanup() - self.assertTrue(later.exists()) self.assertFalse(self.folder.exists()) + self.assertTrue(blocked.exists()) + self.assertEqual(self.events, [ + ("terminate", "local-a1b2c3d4"), + ("remove", "local-a1b2c3d4"), + ("terminate", blocked_id), + ]) + with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): self.run_cleanup() self.assertEqual(list(self.root.iterdir()), []) + self.assertEqual(self.events[-2:], [("terminate", blocked_id), ("remove", blocked_id)]) + self.assertEqual(len(self.events), 5) def test_macos_keeps_folder_when_host_process_cleanup_cannot_be_verified(self): with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "darwin"): @@ -85,22 +80,13 @@ class SolveCleanupTest(unittest.TestCase): self.run_cleanup() self.assertTrue(self.folder.exists()) - def test_cleans_up_each_sandbox_agentenv_lists_whatever_its_id_looks_like(self): - self.folder.rmdir() - listed = self.root / "agent-env-local-a1b2c3d4e5-abcdef12" - listed.mkdir() - with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): - self.run_cleanup() - self.assertEqual(self.terminated, ["local-a1b2c3d4e5"]) - self.assertFalse(listed.exists()) - def test_unexpected_entry_remains_owned(self): self.folder.rmdir() (self.root / "unexpected").mkdir() with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"): with self.assertRaisesRegex(RuntimeError, "Unexpected sandbox root entries: unexpected"): self.run_cleanup() - self.assertEqual(self.terminated, []) + self.assertEqual(self.events, []) self.assertTrue((self.root / "unexpected").exists()) diff --git a/tests/support/providerLifecycleCoverage.ts b/tests/support/providerLifecycleCoverage.ts index 1eab914ce9..41ac09252a 100644 --- a/tests/support/providerLifecycleCoverage.ts +++ b/tests/support/providerLifecycleCoverage.ts @@ -752,7 +752,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< handshake: covered( test( "tests/boundary/providers/jebMcpCatalog.test.ts", - "reads every JEB catalog page and refreshes the connected engine catalog", + "uses the selected discovery budget across handshake and catalog pages", ), test( "tests/boundary/providers/jebMcpConnection.test.ts", @@ -1343,7 +1343,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly< covered( test( "tests/acceptance/erlangBeamCli.test.ts", - "reports BEAM tables and code through the public CLI", + "reports BEAM tables and code for a path relative to the CLI working directory", ), ), typedFailures("tests/boundary/filesystem/erlangBeam.test.ts"),