diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdec07f83d..4e00fa0779 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,9 @@ jobs: qualification_run_id: ${{ steps.qualification.outputs.run_id }} code: ${{ steps.plan.outputs.code }} reverse_tools: ${{ steps.plan.outputs.reverse_tools }} + python: ${{ steps.plan.outputs.python }} + packet_capture: ${{ steps.plan.outputs.packet_capture }} + signal_recording: ${{ steps.plan.outputs.signal_recording }} sqlite: ${{ steps.plan.outputs.sqlite }} runtime: ${{ steps.plan.outputs.runtime }} package: ${{ steps.plan.outputs.package }} @@ -267,6 +270,27 @@ jobs: if-no-files-found: ignore retention-days: 7 + python: + name: Python producer tests + needs: changes + if: needs.changes.outputs.python == 'true' && needs.changes.outputs.tier == 'fast' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache + - run: npm ci + # Analysis-efficiency checks import compiled process modules. + - name: Reuse portable build outputs across runs + uses: ./.github/actions/portable-build-cache + - run: npm run test:prepare + - run: npm run test:python + test-source: name: Source tests needs: changes @@ -394,7 +418,9 @@ jobs: needs.changes.outputs.web_network == 'true' || needs.changes.outputs.web_runtime == 'true' || needs.changes.outputs.web_source_map == 'true' || - needs.changes.outputs.agentenv == 'true' + needs.changes.outputs.agentenv == 'true' || + needs.changes.outputs.packet_capture == 'true' || + needs.changes.outputs.signal_recording == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -1095,6 +1121,26 @@ jobs: build_cache_artifact: rea-build-cache remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} + packet_capture: + needs: [changes, static, build] + if: needs.changes.outputs.packet_capture == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + uses: ./.github/workflows/real-packet-capture.yml + with: + build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} + + signal_recording: + needs: [changes, static, build] + if: needs.changes.outputs.signal_recording == 'true' + secrets: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + uses: ./.github/workflows/real-signal-recording.yml + with: + build_cache_artifact: rea-build-cache + remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }} + nativeaot: needs: [changes, static] if: needs.changes.outputs.nativeaot == 'true' @@ -1121,6 +1167,9 @@ jobs: static-deep, build, reverse-tools-quality, + python, + packet_capture, + signal_recording, test-shard, test-source, sqlite-runtime, diff --git a/.github/workflows/real-packet-capture.yml b/.github/workflows/real-packet-capture.yml index ff4defd254..a4bf3eead8 100644 --- a/.github/workflows/real-packet-capture.yml +++ b/.github/workflows/real-packet-capture.yml @@ -1,39 +1,35 @@ name: Real packet capture on: + workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false workflow_dispatch: - pull_request: - paths: - - "tests/boundary/captures/**" - - "tests/boundary/cli/captureCleanup.test.ts" - - "tests/boundary/mcp/packetCaptureMcp.test.ts" - - "src/captures/**" - - "src/pcap/**" - - "src/domain/pcap/**" - - "src/domain/analysisView/pcapCaptureView.ts" - - "src/application/pcap/**" - - "src/contracts/pcap/**" - - "src/server/registerPcapTools.ts" - - "tests/boundary/pcap/**" - - "src/domain/captures/**" - - "src/application/captures/**" - - "src/contracts/captures/**" - - "src/composition/captures.ts" - - "src/cli/captureCommands.ts" - - "src/server/registerCaptureTools.ts" - - "scripts/verify-packet-capture.mjs" - - ".github/workflows/real-packet-capture.yml" -permissions: - contents: read + env: HUSKY: "0" - # Writes only on the default branch for push, schedule, manual dispatch, - # and workflow_call. Pull requests get an empty token and remote read-only mode. - TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" -concurrency: - group: real-packet-capture-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" + +permissions: + contents: read jobs: tshark: runs-on: ubuntu-24.04 @@ -48,8 +44,18 @@ jobs: uses: ./.github/actions/npm-cache - run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tshark util-linux python3 - run: npm ci + - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run build:cached - - run: npm run test:focused -- tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts + - name: Run focused tests and require Linux resource and cleanup cases + run: | + report=".cache/rea-packet-required-tests.json" + mkdir -p .cache + rm -f "$report" + node scripts/run-exact-tests.mjs run --reporter=default --reporter=json --outputFile.json="$report" \ + tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts + node scripts/ci/require-provider-tests.mjs packet "$report" - run: node scripts/verify-packet-capture.mjs env: REA_TSHARK_COMMAND: /usr/bin/tshark diff --git a/.github/workflows/real-signal-recording.yml b/.github/workflows/real-signal-recording.yml index aeb4217b84..187ccf5d96 100644 --- a/.github/workflows/real-signal-recording.yml +++ b/.github/workflows/real-signal-recording.yml @@ -1,32 +1,35 @@ name: Real signal recording on: + workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" + remote_cache_write: + description: Trusted release qualification may publish the remote cache. + required: false + type: boolean + default: false + secrets: + TURBO_TOKEN: + description: Remote cache token. Only trusted callers pass it. + required: false workflow_dispatch: - pull_request: - paths: - - "tests/boundary/signals/**" - - "tests/boundary/cli/signalCleanup.test.ts" - - "tests/boundary/mcp/signalRecordingMcp.test.ts" - - "src/signals/**" - - "src/domain/signals/**" - - "src/application/signals/**" - - "src/contracts/signals/**" - - "src/composition/signals.ts" - - "src/cli/signalCommands.ts" - - "src/server/registerSignalTools.ts" - - "scripts/verify-signal-recording.mjs" - - ".github/workflows/real-signal-recording.yml" -permissions: - contents: read + env: HUSKY: "0" - # Writes only on the default branch for push, schedule, manual dispatch, - # and workflow_call. Pull requests get an empty token and remote read-only mode. - TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}" + # A called workflow sees the caller event, so workflow_call is not an event + # name here. Writes when remote_cache_write is set, or on the default branch + # for push, schedule, and manual dispatch. Open pull requests stay empty-token + # and use the local cache only. + TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}" TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}" -concurrency: - group: real-signal-recording-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}" + +permissions: + contents: read jobs: sigrok: runs-on: ubuntu-24.04 @@ -41,8 +44,18 @@ jobs: uses: ./.github/actions/npm-cache - run: sudo apt-get update && sudo apt-get install -y sigrok-cli util-linux python3 - run: npm ci + - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run build:cached - - run: npm run test:focused -- tests/boundary/signals/decoderDiscovery.test.ts src/process/LinuxResourceLimits.test.ts src/domain/signals/annotationTextWindow.test.ts tests/boundary/signals/recordingProvider.test.ts tests/boundary/signals/recordingDescriptorCleanup.test.ts tests/boundary/signals/resourceLimits.test.ts tests/boundary/mcp/signalRecordingMcp.test.ts tests/boundary/cli/signalCleanup.test.ts + - name: Run focused tests and require Linux resource and cleanup cases + run: | + report=".cache/rea-signal-required-tests.json" + mkdir -p .cache + rm -f "$report" + node scripts/run-exact-tests.mjs run --reporter=default --reporter=json --outputFile.json="$report" \ + tests/boundary/signals/decoderDiscovery.test.ts src/process/LinuxResourceLimits.test.ts src/domain/signals/annotationTextWindow.test.ts tests/boundary/signals/recordingProvider.test.ts tests/boundary/signals/recordingDescriptorCleanup.test.ts tests/boundary/signals/resourceLimits.test.ts tests/boundary/mcp/signalRecordingMcp.test.ts tests/boundary/cli/signalCleanup.test.ts + node scripts/ci/require-provider-tests.mjs signal "$report" - run: node scripts/verify-signal-recording.mjs env: REA_SIGROK_COMMAND: /usr/bin/sigrok-cli diff --git a/docs/testing.md b/docs/testing.md index 4609628192..a40bbcbcb4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -301,8 +301,9 @@ Turbo caches deterministic preparation and static checks, but `test:run` always executes. Repository files do not fully describe installed tool contents or host state. A previous test pass therefore cannot replace current validation, even when the source tree is unchanged. `check:pr` also runs the fast Python producer -suite; CI runs it in the static job for code changes. These producer tests do not -establish real LLDB engine coverage. +suite. Fast CI runs it from the selected Python job after installing dependencies +and preparing the compiled runtime; full qualification runs it from the deep +static job. These producer tests do not establish real LLDB engine coverage. Vitest's persistent compile cache remains disabled. The `cliTest` fixture shares a temporary Node compile cache among its fresh CLI subprocesses. The worker @@ -851,7 +852,13 @@ Native, provider, fixture, SQLite compatibility, and website lanes wait for the fast static checks (typecheck, built lint, and format) to succeed before allocating their runners. Knip, duplication, architecture guards, test discovery, and the Python producer run in the separate full-tier deep static -job and do not gate provider admission. Portable test shards and documentation +job and do not gate provider admission. On the fast tier, changes to Python +bridges, analysis-efficiency inputs, or the Python test runner select a separate +required Python job. That job installs locked dependencies and prepares the +compiled runtime before the suite, because analysis-efficiency checks import +built process modules. Bridge tests are discovered from the source tree. The +mitmproxy, native, and pwndbg families must each keep a test module, and a +newly added failing test fails the job. Portable test shards and documentation checks remain independent of static results, so they can still provide diagnostic feedback. The required gate rejects skipped selected lanes when static checks fail; on the full tier it also requires the deep static job. @@ -1579,7 +1586,14 @@ Inspector, setup, configuration and packaging entrypoints. Runtime sources under and the Linux x64 installed-package/Inspector baseline only. Test-only changes run the Linux suite without native package verification, except test files that the Apple or Windows curated lanes invoke explicitly, which also select that -lane. Edits to verifier entrypoints run only by manual, licensed or self-hosted +lane. Packet-capture and signal-recording tests and their producers select the +corresponding real-tool lane; both also run in full qualification. The shared +Linux resource-limit source and its tests select both. Each lane requires +its Linux kernel resource-limit cases and all seven descriptor-cleanup cases to +pass in the focused Vitest report. A missing report, missing case, skipped case, +or duplicate identity fails the lane; unrelated host-specific skips remain +allowed. These tests use synthetic producers. The separate real TShark or +sigrok verifier establishes the actual-tool observation. Edits to verifier entrypoints run only by manual, licensed or self-hosted workflows run static checks and the Linux suite. Website changes run website checks. Provider workflow changes validate the workflow and run that provider's verifier. Changes under `bridge/ghidra/` also select the `ghidra_bridge` lane, @@ -1847,8 +1861,8 @@ Portable owner-based test selection remains observational. Its reports compare proposed omissions with the full run; incomplete or skipped observations do not establish that fewer tests would preserve regression detection. -The nine automatic browser, binary-layout, EVM, JavaScript-recovery, -recorded-crash, web and environment-bundle provider lanes download +The automatic browser, binary-layout, EVM, JavaScript-recovery, packet-capture, +signal-recording, recorded-crash, web and environment-bundle provider lanes download `rea-build-cache` from their own run after the build and static jobs succeed. Their existing build commands still validate task hashes and rebuild mismatched inputs; provider observations always run. Provider-workflow-only edits also admit the build without enabling source diff --git a/package.json b/package.json index 177a2a62f8..4f14889f3c 100644 --- a/package.json +++ b/package.json @@ -152,7 +152,7 @@ "test": "npm run test:with-build", "test:with-build": "npm run test:prepare && vitest run", "test:run": "vitest run", - "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/pwndbg/test_core_context.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s tests/support/analysis-efficiency -p 'test_*.py' -v", + "test:python": "python3 scripts/run-python-tests.py", "test:local": "node scripts/run-development-tests.mjs local", "test:fast": "node scripts/ci/test-group.mjs source", "test:built": "node scripts/ci/test-group.mjs built", diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 4804d057df..4480d17d65 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -10,7 +10,13 @@ import { import { planTestSelection } from "./test-selection.mjs"; // Test files select the host/runtime lanes that execute their focused suites. -const HOST_TEST_OWNERS = ["apple", "package", "sqlite"]; +const HOST_TEST_OWNERS = [ + "apple", + "package", + "sqlite", + "packet_capture", + "signal_recording", +]; // Actionlint validates workflow syntax only. The semantic suite asserts these // workflows' release, publication and self-hosted runner safeguards. const GUARDED_WORKFLOWS = new Set([ @@ -111,6 +117,10 @@ function classifyPackage(base, head) { selected.reverse_tools = true; continue; } + if (script === "test:python") { + selected.python = true; + continue; + } if (script === "test:sqlite" || script === "test:sqlite:run") { selected.sqlite = true; continue; @@ -216,6 +226,15 @@ if (tier === "reused") { WINDOWS_EXECUTABLE_SELECTION_INPUTS, ); if (matchesAny(file, SCOPES.reverse_tools)) selected.reverse_tools = true; + if (matchesAny(file, SCOPES.python)) selected.python = true; + // Both live capture lanes exercise Linux resource admission. + if ( + matchesAny(file, ["src/process/LinuxResourceLimits*"]) || + file === "scripts/ci/require-provider-tests.mjs" + ) { + selected.packet_capture = true; + selected.signal_recording = true; + } if ( /^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test( file, diff --git a/scripts/ci/require-provider-tests.mjs b/scripts/ci/require-provider-tests.mjs new file mode 100644 index 0000000000..d4f1571a24 --- /dev/null +++ b/scripts/ci/require-provider-tests.mjs @@ -0,0 +1,67 @@ +import { readFile } from "node:fs/promises"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +const resources = [ + "enforces both address-space limits before any producer allocation", + "preserves lower inherited file/CPU limits in the real launch", +]; +const descriptors = [ + "capture", + "tool", + "limiter", + "kernel", + "field-tool", + "field-limiter", + "field-kernel", +].map( + (member) => + `retains failed ${member} descriptors and retries them before further acquisition and shutdown`, +); +const requirements = { + packet: { + "tests/boundary/captures/resourceLimits.test.ts": [ + ...resources, + "reports a real kernel file-limit termination with its effective ceiling", + ], + "tests/boundary/captures/captureDescriptorCleanup.test.ts": descriptors, + }, + signal: { + "tests/boundary/signals/resourceLimits.test.ts": resources, + "tests/boundary/signals/recordingDescriptorCleanup.test.ts": descriptors, + }, +}; + +const [lane, reportPath] = process.argv.slice(2); +if (!Object.hasOwn(requirements, lane ?? "") || !reportPath) + throw new Error( + "Expected provider test lane (packet or signal) and report path", + ); +const root = resolve(process.cwd()); +const rootPrefix = `${root}${sep}`; +const reportFile = resolve(root, reportPath); +if (!reportFile.startsWith(rootPrefix)) { + const failure = { code: "report_path_outside_checkout" }; + throw Object.assign(new Error(JSON.stringify(failure)), failure); +} +const report = JSON.parse(await readFile(reportFile, "utf8")); +if (report?.success !== true || !Array.isArray(report.testResults)) + throw new Error( + "Provider test report is missing a successful test inventory", + ); +for (const [path, cases] of Object.entries(requirements[lane])) { + const matches = report.testResults.filter((file) => { + if (typeof file?.name !== "string") return false; + const filePath = relative(process.cwd(), resolve(file.name)); + return !isAbsolute(filePath) && filePath.split(sep).join("/") === path; + }); + if (matches.length !== 1 || !Array.isArray(matches[0].assertionResults)) + throw new Error(`Expected exactly one provider test result for ${path}`); + for (const name of cases) { + const results = matches[0].assertionResults.filter( + (test) => test?.fullName === name, + ); + if (results.length !== 1 || results[0].status !== "passed") + throw new Error(`Required provider test did not pass: ${path}: ${name}`); + } +} +console.log(`Required ${lane} resource and cleanup cases passed`); diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index a4da755d77..f43491186a 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -45,6 +45,7 @@ const lanes = { readiness: ["reconstruction-readiness"], docs: ["docs"], workflows: ["static"], + python: [outputs.tier === "full" ? "static-deep" : "python"], }; for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope])); const failed = [...new Set(required)].filter( diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 6991b2c78a..fb148959ac 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -12,6 +12,50 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [ // Shared build inputs are classified once in plan.mjs rather than repeated in // every workflow. export const SCOPES = { + python: [ + "bridge/**/*.py", + "tests/support/analysis-efficiency/**", + "scripts/run-python-tests.py", + ], + packet_capture: [ + "tests/boundary/captures/**", + "tests/boundary/cli/captureCleanup.test.ts", + "tests/boundary/mcp/packetCaptureMcp.test.ts", + "tests/boundary/pcap/**", + "src/captures/**", + "src/pcap/**", + "src/domain/pcap/**", + "src/domain/analysisView/pcapCaptureView.ts", + "src/application/pcap/**", + "src/contracts/pcap/**", + "src/server/registerPcapTools.ts", + "src/domain/captures/**", + "src/application/captures/**", + "src/contracts/captures/**", + "src/composition/captures.ts", + "src/cli/captureCommands.ts", + "src/server/registerCaptureTools.ts", + "src/process/LinuxResourceLimits*", + "scripts/verify-packet-capture.mjs", + "scripts/ci/require-provider-tests.mjs", + ".github/workflows/real-packet-capture.yml", + ], + signal_recording: [ + "tests/boundary/signals/**", + "tests/boundary/cli/signalCleanup.test.ts", + "tests/boundary/mcp/signalRecordingMcp.test.ts", + "src/signals/**", + "src/domain/signals/**", + "src/application/signals/**", + "src/contracts/signals/**", + "src/composition/signals.ts", + "src/cli/signalCommands.ts", + "src/server/registerSignalTools.ts", + "src/process/LinuxResourceLimits*", + "scripts/verify-signal-recording.mjs", + "scripts/ci/require-provider-tests.mjs", + ".github/workflows/real-signal-recording.yml", + ], readiness: [ "tests/conformance/readiness/**", "scripts/verify-reconstruction-readiness.mjs", diff --git a/scripts/run-python-tests.py b/scripts/run-python-tests.py new file mode 100644 index 0000000000..c80ddd9ae5 --- /dev/null +++ b/scripts/run-python-tests.py @@ -0,0 +1,50 @@ +"""Run every source-owned Python bridge and analysis-efficiency test.""" + +from pathlib import Path +import json +import subprocess +import sys + +root = Path(__file__).resolve().parent.parent +# The previous explicit command named these families. Discovery still runs +# every bridge test, and each named family must keep a test module. +REQUIRED_BRIDGE_FAMILIES = ("mitmproxy", "native", "pwndbg") + + +def fail(code, **details): + print(json.dumps({"code": code, **details}, sort_keys=True), file=sys.stderr) + raise SystemExit(1) + + +def family_tests(family): + return [ + path + for path in root.joinpath("bridge", family).rglob("test_*.py") + if path.is_file() + ] + + +missing_families = [ + family for family in REQUIRED_BRIDGE_FAMILIES if not family_tests(family) +] +if missing_families: + fail("missing_bridge_family", families=missing_families) +bridge_tests = sorted(root.joinpath("bridge").rglob("test_*.py")) +if not bridge_tests: + fail("missing_bridge_tests") +if not list(root.joinpath("tests/support/analysis-efficiency").glob("test_*.py")): + fail("missing_analysis_efficiency_tests") +modules = [ + ".".join(path.relative_to(root).with_suffix("").parts) for path in bridge_tests +] +commands = [ + [sys.executable, "-m", "unittest", "-v", *modules], + [ + sys.executable, "-m", "unittest", "discover", + "-s", "tests/support/analysis-efficiency", "-p", "test_*.py", "-v", + ], +] +for command in commands: + result = subprocess.run(command, cwd=root, check=False) + if result.returncode: + sys.exit(result.returncode) diff --git a/tests/acceptance/gpuCodeObjectsCli.test.ts b/tests/acceptance/gpuCodeObjectsCli.test.ts index f83e2f9af6..5edcf3b942 100644 --- a/tests/acceptance/gpuCodeObjectsCli.test.ts +++ b/tests/acceptance/gpuCodeObjectsCli.test.ts @@ -13,7 +13,7 @@ import { parseEvidence, createEvidence } from "../../src/domain/evidence.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js"; import { parseMcpToolError } from "../fixtures/mcpToolError.js"; -import { cliTest } from "../support/cli/cliFixture.js"; +import { cliTest, runFileCli } from "../support/cli/cliFixture.js"; import { cudaFixture, peGpuFixture, @@ -88,7 +88,7 @@ cliTest( const bytes = Buffer.concat([header, compressed]), path = join(root, `v${version}`); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -147,7 +147,7 @@ cliTest( const bytes = hipFixture(payload, bundleId), path = join(root, name); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -197,7 +197,7 @@ cliTest( } const path = join(root, "raw-bitcode"); await writeFile(path, bitcode); - const raw = await cli.run({ + const raw = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -235,7 +235,7 @@ cliTest( ]), path = join(root, name); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -319,7 +319,7 @@ cliTest( ]), path = join(root, name); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -360,7 +360,7 @@ cliTest( ]), path = join(root, "large-parameter.cubin"); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -397,7 +397,7 @@ cliTest( const bytes = hipFixture(payload, "hip-amdgcn-amd-amdhsa--gfx906+xnack"), path = join(root, "feature.bundle"); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -442,7 +442,7 @@ cliTest( root = await createTestTempDirectory("rea-ptx-callprototype-"), path = join(root, "prototype.ptx"); await writeFile(path, source); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -492,7 +492,7 @@ cliTest( const bytes = cubinFixture({ extendedSectionIndex: true }); const path = join(root, "extended.cubin"); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -540,7 +540,7 @@ cliTest( malformedBytes.writeBigUInt64LE(4n, sectionTable + indexTable! * 64 + 32); const malformedPath = join(root, "short-shndx.cubin"); await writeFile(malformedPath, malformedBytes); - const malformed = await cli.run({ + const malformed = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", malformedPath, @@ -571,7 +571,7 @@ cliTest( const text = `.version 7.0\n.target sm_80\n.address_size 64\n.visible .entry kernel() {\n.reg .b32 ${register};\nmov.u32 ${register}, 0;\nret;\n}\n`; const path = join(root, `${register.slice(1)}.ptx`); await writeFile(path, text); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -608,7 +608,7 @@ cliTest( const text = `.version 8.8\n.target sm_80\n.address_size 64\n.entry %kernel(.param .u32 input) {\nret;\n}\n`; const path = join(root, "identifiers.ptx"); await writeFile(path, text); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -671,7 +671,7 @@ cliTest( const text = `${preamble}${declaration}\n`, path = join(root, `${name}.ptx`); await writeFile(path, text); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -721,7 +721,7 @@ cliTest( const text = `.version 8.8\n.target compute_80\n.address_size 64\n.entry kernel() {\nret;\n}\n`; const path = join(root, "target.ptx"); await writeFile(path, text); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -751,7 +751,7 @@ cliTest( const text = `.version 1.3\n.target sm_80\n.address_size 64\n.entry kernel {\n.param .u32 input;\nret;\n}\n`; const path = join(root, "legacy.ptx"); await writeFile(path, text); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, @@ -790,7 +790,7 @@ cliTest( malformedPath, `.version 1.3\n.target sm_80\n.entry kernel {\n.param .u32 input\n`, ); - const malformed = await cli.run({ + const malformed = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", malformedPath, @@ -1273,7 +1273,7 @@ cliTest( ] as const) { const path = join(root, name); await writeFile(path, bytes); - const response = await cli.run({ + const response = await runFileCli(cli, { arguments: [ "inspect-gpu-code-objects", path, diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index eb065c106f..206c781da2 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -53,7 +53,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( `${base}[0,1].value,${base}.1.value`, `${base}.1.value,${base}[0,1].value`, ]) { - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -72,7 +72,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, }); } - const refused = await cli.run({ + const refused = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -105,7 +105,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( } finally { database.close(); } - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -383,7 +383,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, }, }); - const cliResult = await cli.run({ + const cliResult = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -529,7 +529,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( } const { call } = await connectLocalToolsMcp(); for (const limit of [1001, Number.MAX_SAFE_INTEGER]) { - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -752,7 +752,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( const wal = before[1]; expect(db).toBeDefined(); expect(wal).toBeDefined(); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -818,7 +818,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( ], }, }; - const corrupt = await cli.run({ + const corrupt = await runFileCli(cli, { arguments: ["inspect-sqlite-database", selected, "--json"], environment: { REA_LOG_LEVEL: "silent" }, }); @@ -854,7 +854,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, }, }); - const ambiguous = await cli.run({ + const ambiguous = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", valid, @@ -874,7 +874,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }) ).isError, ).toBe(true); - const missing = await cli.run({ + const missing = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", valid, @@ -924,7 +924,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( producer.close(); } const before = await sourceState(path); - const control = await cli.run({ + const control = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1011,7 +1011,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( const path = createSqliteApplicationCollationFixture(root); const before = await sourceState(path); const { call } = await connectLocalToolsMcp(); - const schema = await cli.run({ + const schema = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1050,7 +1050,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( expect(schemaResponse.isError).not.toBe(true); expect(schemaResponse.structuredContent).toMatchObject(expectedSchema); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1125,7 +1125,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }, }; for (const columns of [["b", "a"], undefined]) { - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1146,7 +1146,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( }); expect(parseMcpToolError(response).error).toMatchObject(expectedError); } - const base = await cli.run({ + const base = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1203,7 +1203,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( "selected_table", "database", ] as const) { - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1348,7 +1348,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( native.close(); } - const cliResult = await cli.run({ + const cliResult = await runFileCli(cli, { arguments: [ "inspect-sqlite-database", path, @@ -1427,7 +1427,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( producer.close(); } const original = await readFile(path); - const result = await cli.run({ + const result = await runFileCli(cli, { arguments: ["inspect-sqlite-database", path, "--json"], environment: { REA_LOG_LEVEL: "silent" }, }); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 270979a696..40c16b158f 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -103,6 +103,27 @@ async function classify( return z.record(z.string(), z.unknown()).parse(JSON.parse(result.stdout)); } +it.each([ + ["bridge/native/rea_lldb_tracer.py", "python"], + ["bridge/pwndbg/test_core_context.py", "python"], + ["tests/support/analysis-efficiency/test_signal_oracle.py", "python"], + ["scripts/run-python-tests.py", "python"], + ["src/captures/CaptureProvider.ts", "packet_capture"], + ["tests/boundary/mcp/packetCaptureMcp.test.ts", "packet_capture"], + ["src/signals/RecordingProvider.ts", "signal_recording"], + ["tests/boundary/mcp/signalRecordingMcp.test.ts", "signal_recording"], + ["src/process/LinuxResourceLimits.ts", "packet_capture"], + ["src/process/LinuxResourceLimits.ts", "signal_recording"], + ["scripts/ci/require-provider-tests.mjs", "packet_capture"], + ["scripts/ci/require-provider-tests.mjs", "signal_recording"], +])("selects %s for %s", async (path, scope) => { + const { directory, base } = await fixture(); + const head = await change(directory, [path]); + expect(await classify(directory, base, head)).toMatchObject({ + [scope]: true, + }); +}); + it("reuses only scheduled qualification and preserves every explicit full request", async () => { const { directory, base } = await fixture(); const reused = await classify(directory, base, base, "schedule", false, true); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 8e994e4385..83e73a24e8 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -213,6 +213,7 @@ it("holds native and provider fan-out behind static validation while portable fe "changes", "static", "static-deep", + "python", "build", "test-shard", "test-source", @@ -382,6 +383,8 @@ it("binds provider build reuse to the current run while keeping standalone dispa "web_runtime", "web_source_map", "agentenv", + "packet_capture", + "signal_recording", ]; const buildTerms = z .string() @@ -574,6 +577,9 @@ it.each([ ["workflows", "static"], ["ghidra_bridge", "ghidra_bridge"], ["agentenv", "agentenv"], + ["python", "python"], + ["packet_capture", "packet_capture"], + ["signal_recording", "signal_recording"], ] as const) { // GitHub exposes only direct dependencies in the required job's needs context. const needs: Record< diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index e9839770c9..fc49c99a8c 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -60,6 +60,7 @@ const workflow = async () => test: z.object({ steps: stepsSchema }), build: z.object({ steps: stepsSchema }), docs: z.object({ steps: stepsSchema }), + python: z.object({ if: z.string(), steps: stepsSchema }), static: z.object({ steps: stepsSchema }), "static-deep": z.object({ if: z.string(), @@ -415,9 +416,27 @@ it("runs the existing onboarding and Python producer guards from their selected expect(scripts["check:ci:fast:tasks"]).toBe( "node scripts/run-turbo.mjs run typecheck lint format:check", ); - expect(scripts["test:python"]).toContain( - "bridge/native/test_rea_lldb_tracer.py", + expect(scripts["test:python"]).toBe("python3 scripts/run-python-tests.py"); + expect(jobs.python.if).toContain("outputs.python == 'true'"); + expect(jobs.python.if).toContain("tier == 'fast'"); + const downloadCache = jobs.python.steps.findIndex( + (step) => step.uses === "./.github/actions/npm-cache", ); + const install = jobs.python.steps.findIndex((step) => step.run === "npm ci"); + const prepare = jobs.python.steps.findIndex( + (step) => step.run === "npm run test:prepare", + ); + const suite = jobs.python.steps.findIndex( + (step) => step.run === "npm run test:python", + ); + const cache = jobs.python.steps.findIndex( + (step) => step.uses === "./.github/actions/portable-build-cache", + ); + expect(downloadCache).toBeGreaterThanOrEqual(0); + expect(install).toBeGreaterThan(downloadCache); + expect(cache).toBeGreaterThan(install); + expect(prepare).toBeGreaterThan(cache); + expect(suite).toBeGreaterThan(prepare); }); it("exports portable timing history and removes it on a failed, partial, or invalid rerun", async () => { @@ -713,3 +732,56 @@ it("records failed omissions from full reports without treating partial or unhan readFile(join(root, ".cache/ci-test-history/test-results.json")), ).rejects.toMatchObject({ code: "ENOENT" }); }); + +it("discovers Python bridge tests and fails on a newly added failing producer test", async () => { + const root = await createTestTempDirectory("rea-python-inventory-"); + await mkdir(join(root, "scripts"), { recursive: true }); + await mkdir(join(root, "bridge", "example"), { recursive: true }); + await mkdir(join(root, "tests", "support", "analysis-efficiency"), { + recursive: true, + }); + await writeFile( + join(root, "scripts", "run-python-tests.py"), + await readFile( + new URL("../../../scripts/run-python-tests.py", import.meta.url), + ), + ); + const passing = + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self): pass\n"; + for (const family of ["mitmproxy", "native", "pwndbg"]) { + await mkdir(join(root, "bridge", family), { recursive: true }); + await writeFile(join(root, "bridge", family, "capture.py"), "VALUE = 1\n"); + await writeFile(join(root, "bridge", family, "test_case.py"), passing); + } + await writeFile(join(root, "bridge", "example", "test_first.py"), passing); + await writeFile( + join(root, "tests", "support", "analysis-efficiency", "test_example.py"), + passing, + ); + const command = ["scripts/run-python-tests.py"]; + const discovered = await execute("python3", command, { cwd: root }); + expect(discovered.stderr).toContain("bridge.mitmproxy.test_case"); + expect(discovered.stderr).toContain("bridge.native.test_case"); + expect(discovered.stderr).toContain("bridge.pwndbg.test_case"); + expect(discovered.stderr).toContain("OK"); + await writeFile( + join(root, "bridge", "example", "test_second.py"), + "import unittest\nclass Example(unittest.TestCase):\n def test_case(self): self.fail('new test executed')\n", + ); + await expect( + execute("python3", command, { cwd: root }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("new test executed"), + }); + await rm(join(root, "bridge", "mitmproxy", "test_case.py")); + const missingFamily = await execute("python3", command, { cwd: root }).then( + () => undefined, + (error: { code?: unknown; stderr?: unknown }) => error, + ); + expect(missingFamily).toMatchObject({ code: 1 }); + expect(JSON.parse(String(missingFamily?.stderr))).toEqual({ + code: "missing_bridge_family", + families: ["mitmproxy"], + }); +}); diff --git a/tests/evaluation/providerRequiredCases.test.ts b/tests/evaluation/providerRequiredCases.test.ts new file mode 100644 index 0000000000..6f86f6e837 --- /dev/null +++ b/tests/evaluation/providerRequiredCases.test.ts @@ -0,0 +1,215 @@ +import { execFileSync } from "node:child_process"; +import { + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; + +const script = fileURLToPath( + new URL("../../scripts/ci/require-provider-tests.mjs", import.meta.url), +); +const cleanupCases = [ + "capture", + "tool", + "limiter", + "kernel", + "field-tool", + "field-limiter", + "field-kernel", +].map( + (item) => + `retains failed ${item} descriptors and retries them before further acquisition and shutdown`, +); +const commonLimits = [ + "enforces both address-space limits before any producer allocation", + "preserves lower inherited file/CPU limits in the real launch", +]; +const testFile = (path: string, names: string[]) => ({ + name: resolve(path), + assertionResults: names.map((fullName) => ({ fullName, status: "passed" })), +}); +const report = (lane: "packet" | "signal") => ({ + success: true, + testResults: + lane === "packet" + ? [ + testFile("tests/boundary/captures/resourceLimits.test.ts", [ + ...commonLimits, + "reports a real kernel file-limit termination with its effective ceiling", + ]), + testFile( + "tests/boundary/captures/captureDescriptorCleanup.test.ts", + cleanupCases, + ), + ] + : [ + testFile( + "tests/boundary/signals/resourceLimits.test.ts", + commonLimits, + ), + testFile( + "tests/boundary/signals/recordingDescriptorCleanup.test.ts", + cleanupCases, + ), + ], +}); +const run = (lane: string, value: unknown) => { + mkdirSync(join(process.cwd(), ".cache"), { recursive: true }); + const directory = mkdtempSync( + join(process.cwd(), ".cache", "rea-required-cases-"), + ); + try { + const path = join(directory, "report.json"); + writeFileSync(path, JSON.stringify(value)); + return execFileSync(process.execPath, [script, lane, path], { + cwd: process.cwd(), + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}; +const outsideReportFailure = (reportPath: string) => { + try { + execFileSync(process.execPath, [script, "packet", reportPath], { + cwd: process.cwd(), + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + return undefined; + } catch (error) { + return error; + } +}; + +it.each(["packet", "signal"] as const)( + "requires the %s resource and descriptor cases without banning unrelated skips", + (lane) => { + const result = report(lane); + result.testResults.push({ + name: resolve("tests/unrelated.test.ts"), + assertionResults: [{ fullName: "host-specific", status: "pending" }], + }); + expect(run(lane, result)).toContain("cases passed"); + for (const resultIndex of [0, 1]) { + const skipped = report(lane); + skipped.testResults[resultIndex]!.assertionResults[0]!.status = "pending"; + expect(() => run(lane, skipped)).toThrow( + "Required provider test did not pass", + ); + const failed = report(lane); + failed.testResults[resultIndex]!.assertionResults[0]!.status = "failed"; + expect(() => run(lane, failed)).toThrow( + "Required provider test did not pass", + ); + const removed = report(lane); + removed.testResults[resultIndex]!.assertionResults.shift(); + expect(() => run(lane, removed)).toThrow( + "Required provider test did not pass", + ); + const duplicate = report(lane); + duplicate.testResults[resultIndex]!.assertionResults.push( + duplicate.testResults[resultIndex]!.assertionResults[0]!, + ); + expect(() => run(lane, duplicate)).toThrow( + "Required provider test did not pass", + ); + } + const missingFile = report(lane); + missingFile.testResults.pop(); + expect(() => run(lane, missingFile)).toThrow( + "Expected exactly one provider test result", + ); + expect(() => run(lane, { success: true, testResults: [] })).toThrow(); + expect(() => run(lane, { success: false, testResults: [] })).toThrow(); + }, +); + +it.each([ + ["parent", resolve(process.cwd(), "..", "rea-provider-report.json")], + ["sibling prefix", `${resolve(process.cwd())}-outside/report.json`], + ["absolute outside", resolve(tmpdir(), "rea-provider-report.json")], +] as const)( + "rejects a %s report path outside the checkout", + (_label, reportPath) => { + const failure = z + .object({ status: z.literal(1), stderr: z.string() }) + .parse(outsideReportFailure(reportPath)); + const encoded = failure.stderr.match( + /\{"code":"report_path_outside_checkout"\}/u, + )?.[0]; + expect(JSON.parse(encoded ?? "")).toEqual({ + code: "report_path_outside_checkout", + }); + }, +); + +it("rejects unknown lanes and missing or malformed reports", () => { + expect(() => run("other", report("packet"))).toThrow( + "Expected provider test lane", + ); + expect(() => run("packet", null)).toThrow( + "missing a successful test inventory", + ); + expect(() => + execFileSync(process.execPath, [script, "packet", "/missing/report.json"], { + stdio: "pipe", + }), + ).toThrow(); +}); + +it.each([ + [ + "real-packet-capture.yml", + "tshark", + "packet", + "tests/boundary/captures/resourceLimits.test.ts", + "tests/boundary/captures/captureDescriptorCleanup.test.ts", + ], + [ + "real-signal-recording.yml", + "sigrok", + "signal", + "tests/boundary/signals/resourceLimits.test.ts", + "tests/boundary/signals/recordingDescriptorCleanup.test.ts", + ], +] as const)( + "keeps required case validation in %s", + (workflow, job, lane, limits, cleanup) => { + const file = new URL( + `../../.github/workflows/${workflow}`, + import.meta.url, + ); + const parsed = z + .object({ + jobs: z.record( + z.string(), + z.object({ + steps: z.array(z.object({ run: z.string().optional() })), + }), + ), + }) + .parse(parse(readFileSync(file, "utf8"))); + const step = parsed.jobs[job]?.steps.find((item) => + item.run?.includes("scripts/ci/require-provider-tests.mjs"), + ); + expect(step?.run).toContain('rm -f "$report"'); + expect(step?.run).toContain("node scripts/run-exact-tests.mjs run"); + expect(step?.run).toContain("--reporter=json"); + expect(step?.run).toContain('--outputFile.json="$report"'); + expect(step?.run).toContain(limits); + expect(step?.run).toContain(cleanup); + expect(step?.run).toContain( + `node scripts/ci/require-provider-tests.mjs ${lane} "$report"`, + ); + }, +);