test(agentenv): consolidate verification, capture and cleanup checks (#12855)

* test(agentenv): consolidate solve verification and cleanup checks

* test(agentenv): verify capture argument delivery and retention together

* test: update lifecycle registry after campaign test renames
This commit is contained in:
morluto authored and GitHub committed 2026-11-20 15:05:34 +00:00
1 parent a1dec2d2d5
commit d137ffa762
4 files changed
+53 -65

No files matched your search

+16 -10
View File
@@ -1,5 +1,6 @@
"""Exercise the capture controller's filesystem inventory boundary."""
import hashlib
import json
import io
from contextlib import redirect_stdout
@@ -82,7 +83,7 @@ class LongRunTest(unittest.TestCase):
rea = self.bin / "rea"
rea.write_text(
f"#!{sys.executable}\n"
"import os, signal, subprocess, sys, time\n"
"import hashlib, json, os, signal, subprocess, sys, time\n"
"signal.signal(signal.SIGINT, lambda *_: sys.exit(130))\n"
"if len(sys.argv) > 1 and sys.argv[1] == 'wait':\n"
" open(sys.argv[2], 'w').close()\n"
@@ -93,6 +94,7 @@ class LongRunTest(unittest.TestCase):
" while not os.path.exists(sys.argv[3]): time.sleep(0.02)\n"
"if len(sys.argv) > 1 and sys.argv[1] == 'stderr':\n"
" sys.stderr.write('x' * 100000 + 'TAIL')\n"
" print(hashlib.sha256(json.dumps(sys.argv[2:]).encode()).hexdigest())\n"
"if len(sys.argv) > 1 and sys.argv[1] == 'hold':\n"
" held = subprocess.Popen([sys.executable, '-c', 'import os,sys,time\\nwhile not os.path.exists(sys.argv[1]): time.sleep(0.02)', sys.argv[2]], stdout=subprocess.DEVNULL, stderr=sys.stderr, start_new_session=True)\n"
" open(sys.argv[2] + '.pid.tmp', 'w').write(str(held.pid))\n"
@@ -199,17 +201,21 @@ class LongRunTest(unittest.TestCase):
self.assertEqual(result["exit_code"], 0)
self.assertEqual(self.output.read_text().strip(), "no token")
def test_passes_large_command_without_persisting_its_arguments(self):
self.assertEqual(self.begin(["done", *("x" * 4000 for _ in range(20))]), {"started": True})
result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5)
self.assertEqual(result["exit_code"], 0)
self.assertFalse((self.job / "request.json").exists())
def test_delivers_large_arguments_without_persisting_them_and_bounds_stderr(self):
canary = "large-command-last-argument"
arguments = [f"{index}-" + "x" * 4000 for index in range(20)] + [canary]
digest = hashlib.sha256(json.dumps(arguments).encode()).hexdigest()
self.assertEqual(self.begin(["stderr", *arguments]), {"started": True})
def test_keeps_only_a_bounded_stderr_tail(self):
self.assertEqual(self.begin(["stderr"]), {"started": True})
result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5)
self.assertEqual(len(result["stderr"]), 4096)
self.assertTrue(result["stderr"].endswith("TAIL"))
self.assertTrue(result["finished"])
self.assertEqual(result["exit_code"], 0)
self.assertEqual(self.output.read_text().splitlines(), [digest, "no token"])
self.assertEqual(result["stderr"], "x" * 4092 + "TAIL")
self.assertFalse((self.job / "request.json").exists())
for path in self.job.rglob("*"):
if path.is_file():
self.assertNotIn(canary.encode(), path.read_bytes(), str(path))
def test_cancels_a_running_child_and_confirms_exit(self):
ready = self.root / "ready"
+7 -11
View File
@@ -12,14 +12,6 @@ def run_with(verifications, outcome="passed"):
class SolveFactsTest(unittest.TestCase):
def test_matching_verifier_success_and_failure(self):
for passed, score, outcome in ((True, 1.0, "passed"), (False, 0.0, "scored below 1")):
with self.subTest(passed=passed):
run = run_with({"wanted": {"results": [{"id": "selected", "result": passed}],
"score": score}})
self.assertEqual(_solve_facts(run, "wanted"),
(outcome, [{"id": "selected", "passed": passed, "description": ""}]))
def test_missing_empty_and_unscored_selected_verifier_do_not_borrow_success(self):
other = {"results": [{"id": "unrelated", "result": True}], "score": 1.0}
for selected in (None, {}, {"results": [], "score": 1.0},
@@ -44,12 +36,16 @@ class SolveFactsTest(unittest.TestCase):
checks = [{"id": "0", "passed": False, "description": ""},
{"id": "zero", "passed": False, "description": ""},
{"id": "yes", "passed": True, "description": ""}]
for score, expected in ((0, "scored below 1"), (0.5, "scored below 1"),
(1, "passed"), (1.5, "passed")):
for score, prior, expected in (
(0, "passed", "scored below 1"),
(0.5, "passed", "scored below 1"),
(1, "scored below 1", "passed"),
(1.5, "scored below 1", "passed"),
):
with self.subTest(score=score):
verifications = {"other": {"score": 0},
"wanted": {"results": rows, "score": score}}
self.assertEqual(_solve_facts(run_with(verifications, "scored below 1"), "wanted"),
self.assertEqual(_solve_facts(run_with(verifications, prior), "wanted"),
(expected, checks))
def test_failure_and_cancellation_preserve_partial_named_checks(self):
+28 -42
View File
@@ -14,15 +14,15 @@ class SolveCleanupTest(unittest.TestCase):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
self.folder = self.root / "agent-env-local-a1b2c3d4-abcdef12"
self.folder.mkdir()
self.terminated = []
self.folders = {}
self.folder = self._add_sandbox("local-a1b2c3d4")
self.events = []
self.module = types.ModuleType("agent_env.providers.sandbox_providers.local_docker.provider")
test = self
class Provider:
async def terminate_sandbox(self, sandbox_id):
test.terminated.append(sandbox_id)
test.events.append(("terminate", sandbox_id))
self.module.LocalSandboxProvider = Provider
self.module.local_sandbox_ids = self._listed
@@ -32,52 +32,47 @@ class SolveCleanupTest(unittest.TestCase):
self.patch_module.start()
self.addCleanup(self.patch_module.stop)
def _add_sandbox(self, sandbox_id):
folder = self.root / f"agent-env-{sandbox_id}-abcdef12"
folder.mkdir()
self.folders[sandbox_id] = folder
return folder
def _listed(self):
# As agentenv lists them: `agent-env-<id>-<suffix>` folders, whatever the id's shape.
found = set()
for entry in self.root.iterdir():
sandbox_id, _, suffix = entry.name.removeprefix("agent-env-").rpartition("-")
if entry.name.startswith("agent-env-") and sandbox_id and suffix and entry.is_dir():
found.add(sandbox_id)
return sorted(found)
return [sandbox_id for sandbox_id, folder in self.folders.items() if folder.is_dir()]
def _remove(self, sandbox_id):
folder = next(self.root.glob(f"agent-env-{sandbox_id}-*"))
self.events.append(("remove", sandbox_id))
folder = self.folders[sandbox_id]
folder.rmdir()
return folder
def run_cleanup(self):
asyncio.run(solve_cleanup.cleanup(self.root))
def test_terminates_owned_sandbox_before_removing_its_folder(self):
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
self.run_cleanup()
self.assertEqual(self.terminated, ["local-a1b2c3d4"])
self.assertFalse(self.folder.exists())
def test_terminates_before_removing_and_retries_only_the_retained_sandbox(self):
blocked_id = "local-f1223344e5"
blocked = self._add_sandbox(blocked_id)
def test_retains_folder_when_docker_reports_container_then_retries(self):
with patch.object(solve_cleanup, "_docker", return_value="still-running"), patch.object(solve_cleanup.sys, "platform", "linux"):
with self.assertRaisesRegex(RuntimeError, "still has its agent container"):
self.run_cleanup()
self.assertTrue(self.folder.exists())
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
self.run_cleanup()
self.assertFalse(self.folder.exists())
self.assertEqual(len(self.terminated), 2)
def test_partial_cleanup_and_late_folder_are_retried(self):
later = self.root / "agent-env-local-f1223344-abcdef12"
later.mkdir()
def docker(*args):
return "still-running" if "local-f1223344" in " ".join(args) else ""
return "still-running" if args[-1] == f"name=^agent-{blocked_id}$" else ""
with patch.object(solve_cleanup, "_docker", side_effect=docker), patch.object(solve_cleanup.sys, "platform", "linux"):
with self.assertRaisesRegex(RuntimeError, "still has its agent container"):
self.run_cleanup()
self.assertTrue(later.exists())
self.assertFalse(self.folder.exists())
self.assertTrue(blocked.exists())
self.assertEqual(self.events, [
("terminate", "local-a1b2c3d4"),
("remove", "local-a1b2c3d4"),
("terminate", blocked_id),
])
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
self.run_cleanup()
self.assertEqual(list(self.root.iterdir()), [])
self.assertEqual(self.events[-2:], [("terminate", blocked_id), ("remove", blocked_id)])
self.assertEqual(len(self.events), 5)
def test_macos_keeps_folder_when_host_process_cleanup_cannot_be_verified(self):
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "darwin"):
@@ -85,22 +80,13 @@ class SolveCleanupTest(unittest.TestCase):
self.run_cleanup()
self.assertTrue(self.folder.exists())
def test_cleans_up_each_sandbox_agentenv_lists_whatever_its_id_looks_like(self):
self.folder.rmdir()
listed = self.root / "agent-env-local-a1b2c3d4e5-abcdef12"
listed.mkdir()
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
self.run_cleanup()
self.assertEqual(self.terminated, ["local-a1b2c3d4e5"])
self.assertFalse(listed.exists())
def test_unexpected_entry_remains_owned(self):
self.folder.rmdir()
(self.root / "unexpected").mkdir()
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
with self.assertRaisesRegex(RuntimeError, "Unexpected sandbox root entries: unexpected"):
self.run_cleanup()
self.assertEqual(self.terminated, [])
self.assertEqual(self.events, [])
self.assertTrue((self.root / "unexpected").exists())
+2 -2
View File
@@ -752,7 +752,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly<
handshake: covered(
test(
"tests/boundary/providers/jebMcpCatalog.test.ts",
"reads every JEB catalog page and refreshes the connected engine catalog",
"uses the selected discovery budget across handshake and catalog pages",
),
test(
"tests/boundary/providers/jebMcpConnection.test.ts",
@@ -1343,7 +1343,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly<
covered(
test(
"tests/acceptance/erlangBeamCli.test.ts",
"reports BEAM tables and code through the public CLI",
"reports BEAM tables and code for a path relative to the CLI working directory",
),
),
typedFailures("tests/boundary/filesystem/erlangBeam.test.ts"),