mirror of
https://github.com/morluto/rea.git
synced 2026-11-21 14:05:49 +00:00
test(agentenv): consolidate verification, capture and cleanup checks (#12855)
* test(agentenv): consolidate solve verification and cleanup checks * test(agentenv): verify capture argument delivery and retention together * test: update lifecycle registry after campaign test renames
This commit is contained in:
1 parent
a1dec2d2d5
commit
d137ffa762
4 files changed
+53
-65
No files matched your search
@@ -1,5 +1,6 @@
|
||||
"""Exercise the capture controller's filesystem inventory boundary."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import io
|
||||
from contextlib import redirect_stdout
|
||||
@@ -82,7 +83,7 @@ class LongRunTest(unittest.TestCase):
|
||||
rea = self.bin / "rea"
|
||||
rea.write_text(
|
||||
f"#!{sys.executable}\n"
|
||||
"import os, signal, subprocess, sys, time\n"
|
||||
"import hashlib, json, os, signal, subprocess, sys, time\n"
|
||||
"signal.signal(signal.SIGINT, lambda *_: sys.exit(130))\n"
|
||||
"if len(sys.argv) > 1 and sys.argv[1] == 'wait':\n"
|
||||
" open(sys.argv[2], 'w').close()\n"
|
||||
@@ -93,6 +94,7 @@ class LongRunTest(unittest.TestCase):
|
||||
" while not os.path.exists(sys.argv[3]): time.sleep(0.02)\n"
|
||||
"if len(sys.argv) > 1 and sys.argv[1] == 'stderr':\n"
|
||||
" sys.stderr.write('x' * 100000 + 'TAIL')\n"
|
||||
" print(hashlib.sha256(json.dumps(sys.argv[2:]).encode()).hexdigest())\n"
|
||||
"if len(sys.argv) > 1 and sys.argv[1] == 'hold':\n"
|
||||
" held = subprocess.Popen([sys.executable, '-c', 'import os,sys,time\\nwhile not os.path.exists(sys.argv[1]): time.sleep(0.02)', sys.argv[2]], stdout=subprocess.DEVNULL, stderr=sys.stderr, start_new_session=True)\n"
|
||||
" open(sys.argv[2] + '.pid.tmp', 'w').write(str(held.pid))\n"
|
||||
@@ -199,17 +201,21 @@ class LongRunTest(unittest.TestCase):
|
||||
self.assertEqual(result["exit_code"], 0)
|
||||
self.assertEqual(self.output.read_text().strip(), "no token")
|
||||
|
||||
def test_passes_large_command_without_persisting_its_arguments(self):
|
||||
self.assertEqual(self.begin(["done", *("x" * 4000 for _ in range(20))]), {"started": True})
|
||||
result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5)
|
||||
self.assertEqual(result["exit_code"], 0)
|
||||
self.assertFalse((self.job / "request.json").exists())
|
||||
def test_delivers_large_arguments_without_persisting_them_and_bounds_stderr(self):
|
||||
canary = "large-command-last-argument"
|
||||
arguments = [f"{index}-" + "x" * 4000 for index in range(20)] + [canary]
|
||||
digest = hashlib.sha256(json.dumps(arguments).encode()).hexdigest()
|
||||
self.assertEqual(self.begin(["stderr", *arguments]), {"started": True})
|
||||
|
||||
def test_keeps_only_a_bounded_stderr_tail(self):
|
||||
self.assertEqual(self.begin(["stderr"]), {"started": True})
|
||||
result = self.command("run-poll", job=str(self.job), token="owner-token", timeout_s=5)
|
||||
self.assertEqual(len(result["stderr"]), 4096)
|
||||
self.assertTrue(result["stderr"].endswith("TAIL"))
|
||||
self.assertTrue(result["finished"])
|
||||
self.assertEqual(result["exit_code"], 0)
|
||||
self.assertEqual(self.output.read_text().splitlines(), [digest, "no token"])
|
||||
self.assertEqual(result["stderr"], "x" * 4092 + "TAIL")
|
||||
self.assertFalse((self.job / "request.json").exists())
|
||||
for path in self.job.rglob("*"):
|
||||
if path.is_file():
|
||||
self.assertNotIn(canary.encode(), path.read_bytes(), str(path))
|
||||
|
||||
def test_cancels_a_running_child_and_confirms_exit(self):
|
||||
ready = self.root / "ready"
|
||||
|
||||
@@ -12,14 +12,6 @@ def run_with(verifications, outcome="passed"):
|
||||
|
||||
|
||||
class SolveFactsTest(unittest.TestCase):
|
||||
def test_matching_verifier_success_and_failure(self):
|
||||
for passed, score, outcome in ((True, 1.0, "passed"), (False, 0.0, "scored below 1")):
|
||||
with self.subTest(passed=passed):
|
||||
run = run_with({"wanted": {"results": [{"id": "selected", "result": passed}],
|
||||
"score": score}})
|
||||
self.assertEqual(_solve_facts(run, "wanted"),
|
||||
(outcome, [{"id": "selected", "passed": passed, "description": ""}]))
|
||||
|
||||
def test_missing_empty_and_unscored_selected_verifier_do_not_borrow_success(self):
|
||||
other = {"results": [{"id": "unrelated", "result": True}], "score": 1.0}
|
||||
for selected in (None, {}, {"results": [], "score": 1.0},
|
||||
@@ -44,12 +36,16 @@ class SolveFactsTest(unittest.TestCase):
|
||||
checks = [{"id": "0", "passed": False, "description": ""},
|
||||
{"id": "zero", "passed": False, "description": ""},
|
||||
{"id": "yes", "passed": True, "description": ""}]
|
||||
for score, expected in ((0, "scored below 1"), (0.5, "scored below 1"),
|
||||
(1, "passed"), (1.5, "passed")):
|
||||
for score, prior, expected in (
|
||||
(0, "passed", "scored below 1"),
|
||||
(0.5, "passed", "scored below 1"),
|
||||
(1, "scored below 1", "passed"),
|
||||
(1.5, "scored below 1", "passed"),
|
||||
):
|
||||
with self.subTest(score=score):
|
||||
verifications = {"other": {"score": 0},
|
||||
"wanted": {"results": rows, "score": score}}
|
||||
self.assertEqual(_solve_facts(run_with(verifications, "scored below 1"), "wanted"),
|
||||
self.assertEqual(_solve_facts(run_with(verifications, prior), "wanted"),
|
||||
(expected, checks))
|
||||
|
||||
def test_failure_and_cancellation_preserve_partial_named_checks(self):
|
||||
|
||||
@@ -14,15 +14,15 @@ class SolveCleanupTest(unittest.TestCase):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.root = Path(self.temp.name)
|
||||
self.folder = self.root / "agent-env-local-a1b2c3d4-abcdef12"
|
||||
self.folder.mkdir()
|
||||
self.terminated = []
|
||||
self.folders = {}
|
||||
self.folder = self._add_sandbox("local-a1b2c3d4")
|
||||
self.events = []
|
||||
self.module = types.ModuleType("agent_env.providers.sandbox_providers.local_docker.provider")
|
||||
test = self
|
||||
|
||||
class Provider:
|
||||
async def terminate_sandbox(self, sandbox_id):
|
||||
test.terminated.append(sandbox_id)
|
||||
test.events.append(("terminate", sandbox_id))
|
||||
|
||||
self.module.LocalSandboxProvider = Provider
|
||||
self.module.local_sandbox_ids = self._listed
|
||||
@@ -32,52 +32,47 @@ class SolveCleanupTest(unittest.TestCase):
|
||||
self.patch_module.start()
|
||||
self.addCleanup(self.patch_module.stop)
|
||||
|
||||
def _add_sandbox(self, sandbox_id):
|
||||
folder = self.root / f"agent-env-{sandbox_id}-abcdef12"
|
||||
folder.mkdir()
|
||||
self.folders[sandbox_id] = folder
|
||||
return folder
|
||||
|
||||
def _listed(self):
|
||||
# As agentenv lists them: `agent-env-<id>-<suffix>` folders, whatever the id's shape.
|
||||
found = set()
|
||||
for entry in self.root.iterdir():
|
||||
sandbox_id, _, suffix = entry.name.removeprefix("agent-env-").rpartition("-")
|
||||
if entry.name.startswith("agent-env-") and sandbox_id and suffix and entry.is_dir():
|
||||
found.add(sandbox_id)
|
||||
return sorted(found)
|
||||
return [sandbox_id for sandbox_id, folder in self.folders.items() if folder.is_dir()]
|
||||
|
||||
def _remove(self, sandbox_id):
|
||||
folder = next(self.root.glob(f"agent-env-{sandbox_id}-*"))
|
||||
self.events.append(("remove", sandbox_id))
|
||||
folder = self.folders[sandbox_id]
|
||||
folder.rmdir()
|
||||
return folder
|
||||
|
||||
def run_cleanup(self):
|
||||
asyncio.run(solve_cleanup.cleanup(self.root))
|
||||
|
||||
def test_terminates_owned_sandbox_before_removing_its_folder(self):
|
||||
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
self.run_cleanup()
|
||||
self.assertEqual(self.terminated, ["local-a1b2c3d4"])
|
||||
self.assertFalse(self.folder.exists())
|
||||
def test_terminates_before_removing_and_retries_only_the_retained_sandbox(self):
|
||||
blocked_id = "local-f1223344e5"
|
||||
blocked = self._add_sandbox(blocked_id)
|
||||
|
||||
def test_retains_folder_when_docker_reports_container_then_retries(self):
|
||||
with patch.object(solve_cleanup, "_docker", return_value="still-running"), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
with self.assertRaisesRegex(RuntimeError, "still has its agent container"):
|
||||
self.run_cleanup()
|
||||
self.assertTrue(self.folder.exists())
|
||||
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
self.run_cleanup()
|
||||
self.assertFalse(self.folder.exists())
|
||||
self.assertEqual(len(self.terminated), 2)
|
||||
|
||||
def test_partial_cleanup_and_late_folder_are_retried(self):
|
||||
later = self.root / "agent-env-local-f1223344-abcdef12"
|
||||
later.mkdir()
|
||||
def docker(*args):
|
||||
return "still-running" if "local-f1223344" in " ".join(args) else ""
|
||||
return "still-running" if args[-1] == f"name=^agent-{blocked_id}$" else ""
|
||||
|
||||
with patch.object(solve_cleanup, "_docker", side_effect=docker), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
with self.assertRaisesRegex(RuntimeError, "still has its agent container"):
|
||||
self.run_cleanup()
|
||||
self.assertTrue(later.exists())
|
||||
self.assertFalse(self.folder.exists())
|
||||
self.assertTrue(blocked.exists())
|
||||
self.assertEqual(self.events, [
|
||||
("terminate", "local-a1b2c3d4"),
|
||||
("remove", "local-a1b2c3d4"),
|
||||
("terminate", blocked_id),
|
||||
])
|
||||
|
||||
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
self.run_cleanup()
|
||||
self.assertEqual(list(self.root.iterdir()), [])
|
||||
self.assertEqual(self.events[-2:], [("terminate", blocked_id), ("remove", blocked_id)])
|
||||
self.assertEqual(len(self.events), 5)
|
||||
|
||||
def test_macos_keeps_folder_when_host_process_cleanup_cannot_be_verified(self):
|
||||
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "darwin"):
|
||||
@@ -85,22 +80,13 @@ class SolveCleanupTest(unittest.TestCase):
|
||||
self.run_cleanup()
|
||||
self.assertTrue(self.folder.exists())
|
||||
|
||||
def test_cleans_up_each_sandbox_agentenv_lists_whatever_its_id_looks_like(self):
|
||||
self.folder.rmdir()
|
||||
listed = self.root / "agent-env-local-a1b2c3d4e5-abcdef12"
|
||||
listed.mkdir()
|
||||
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
self.run_cleanup()
|
||||
self.assertEqual(self.terminated, ["local-a1b2c3d4e5"])
|
||||
self.assertFalse(listed.exists())
|
||||
|
||||
def test_unexpected_entry_remains_owned(self):
|
||||
self.folder.rmdir()
|
||||
(self.root / "unexpected").mkdir()
|
||||
with patch.object(solve_cleanup, "_docker", return_value=""), patch.object(solve_cleanup.sys, "platform", "linux"):
|
||||
with self.assertRaisesRegex(RuntimeError, "Unexpected sandbox root entries: unexpected"):
|
||||
self.run_cleanup()
|
||||
self.assertEqual(self.terminated, [])
|
||||
self.assertEqual(self.events, [])
|
||||
self.assertTrue((self.root / "unexpected").exists())
|
||||
|
||||
|
||||
|
||||
@@ -752,7 +752,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly<
|
||||
handshake: covered(
|
||||
test(
|
||||
"tests/boundary/providers/jebMcpCatalog.test.ts",
|
||||
"reads every JEB catalog page and refreshes the connected engine catalog",
|
||||
"uses the selected discovery budget across handshake and catalog pages",
|
||||
),
|
||||
test(
|
||||
"tests/boundary/providers/jebMcpConnection.test.ts",
|
||||
@@ -1343,7 +1343,7 @@ export const PROVIDER_LIFECYCLE_COVERAGE: Readonly<
|
||||
covered(
|
||||
test(
|
||||
"tests/acceptance/erlangBeamCli.test.ts",
|
||||
"reports BEAM tables and code through the public CLI",
|
||||
"reports BEAM tables and code for a path relative to the CLI working directory",
|
||||
),
|
||||
),
|
||||
typedFailures("tests/boundary/filesystem/erlangBeam.test.ts"),
|
||||
|
||||
Reference in new issue
Block a user