test(ci): require provider checks and reduce test startup (#8145)

* test(ci): require provider coverage and reuse test sessions

* fix(ci): prepare the runtime before Python producer tests

The fast Python job ran the suite on a fresh checkout, so analysis-efficiency
checks failed without compiled process modules. Install locked dependencies
and prepare the runtime before the suite.

* fix(ci): keep provider test reports inside the checkout

The provider-report reader accepted an arbitrary path from argv. Resolve the
path and reject anything outside the checkout before reading it. The packet
and signal workflows write those reports under .cache.

* fix(ci): require a test module in each named Python bridge family

Discovery still runs every bridge test. The mitmproxy, native, and pwndbg
families must each keep a test module, so deleting one cannot pass through
the others.

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
morlutoandCursor Agent authored and GitHub committed 2026-10-30 19:34:19 +08:00
1 parent b1ff7cdacc
commit 77b09314ff
16 files changed
+678 -101

No files matched your search

+50 -1
View File
@@ -61,6 +61,9 @@ jobs:
qualification_run_id: ${{ steps.qualification.outputs.run_id }}
code: ${{ steps.plan.outputs.code }}
reverse_tools: ${{ steps.plan.outputs.reverse_tools }}
python: ${{ steps.plan.outputs.python }}
packet_capture: ${{ steps.plan.outputs.packet_capture }}
signal_recording: ${{ steps.plan.outputs.signal_recording }}
sqlite: ${{ steps.plan.outputs.sqlite }}
runtime: ${{ steps.plan.outputs.runtime }}
package: ${{ steps.plan.outputs.package }}
@@ -267,6 +270,27 @@ jobs:
if-no-files-found: ignore
retention-days: 7
python:
name: Python producer tests
needs: changes
if: needs.changes.outputs.python == 'true' && needs.changes.outputs.tier == 'fast'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version-file: .nvmrc
- name: Cache npm package downloads
id: npm-cache
uses: ./.github/actions/npm-cache
- run: npm ci
# Analysis-efficiency checks import compiled process modules.
- name: Reuse portable build outputs across runs
uses: ./.github/actions/portable-build-cache
- run: npm run test:prepare
- run: npm run test:python
test-source:
name: Source tests
needs: changes
@@ -394,7 +418,9 @@ jobs:
needs.changes.outputs.web_network == 'true' ||
needs.changes.outputs.web_runtime == 'true' ||
needs.changes.outputs.web_source_map == 'true' ||
needs.changes.outputs.agentenv == 'true'
needs.changes.outputs.agentenv == 'true' ||
needs.changes.outputs.packet_capture == 'true' ||
needs.changes.outputs.signal_recording == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -1095,6 +1121,26 @@ jobs:
build_cache_artifact: rea-build-cache
remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }}
packet_capture:
needs: [changes, static, build]
if: needs.changes.outputs.packet_capture == 'true'
secrets:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
uses: ./.github/workflows/real-packet-capture.yml
with:
build_cache_artifact: rea-build-cache
remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }}
signal_recording:
needs: [changes, static, build]
if: needs.changes.outputs.signal_recording == 'true'
secrets:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
uses: ./.github/workflows/real-signal-recording.yml
with:
build_cache_artifact: rea-build-cache
remote_cache_write: ${{ inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')) }}
nativeaot:
needs: [changes, static]
if: needs.changes.outputs.nativeaot == 'true'
@@ -1121,6 +1167,9 @@ jobs:
static-deep,
build,
reverse-tools-quality,
python,
packet_capture,
signal_recording,
test-shard,
test-source,
sqlite-runtime,
+37 -31
View File
@@ -1,39 +1,35 @@
name: Real packet capture
on:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
remote_cache_write:
description: Trusted release qualification may publish the remote cache.
required: false
type: boolean
default: false
secrets:
TURBO_TOKEN:
description: Remote cache token. Only trusted callers pass it.
required: false
workflow_dispatch:
pull_request:
paths:
- "tests/boundary/captures/**"
- "tests/boundary/cli/captureCleanup.test.ts"
- "tests/boundary/mcp/packetCaptureMcp.test.ts"
- "src/captures/**"
- "src/pcap/**"
- "src/domain/pcap/**"
- "src/domain/analysisView/pcapCaptureView.ts"
- "src/application/pcap/**"
- "src/contracts/pcap/**"
- "src/server/registerPcapTools.ts"
- "tests/boundary/pcap/**"
- "src/domain/captures/**"
- "src/application/captures/**"
- "src/contracts/captures/**"
- "src/composition/captures.ts"
- "src/cli/captureCommands.ts"
- "src/server/registerCaptureTools.ts"
- "scripts/verify-packet-capture.mjs"
- ".github/workflows/real-packet-capture.yml"
permissions:
contents: read
env:
HUSKY: "0"
# Writes only on the default branch for push, schedule, manual dispatch,
# and workflow_call. Pull requests get an empty token and remote read-only mode.
TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}"
# A called workflow sees the caller event, so workflow_call is not an event
# name here. Writes when remote_cache_write is set, or on the default branch
# for push, schedule, and manual dispatch. Open pull requests stay empty-token
# and use the local cache only.
TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}"
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}"
concurrency:
group: real-packet-capture-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}"
permissions:
contents: read
jobs:
tshark:
runs-on: ubuntu-24.04
@@ -48,8 +44,18 @@ jobs:
uses: ./.github/actions/npm-cache
- run: sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tshark util-linux python3
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- run: npm run build:cached
- run: npm run test:focused -- tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts
- name: Run focused tests and require Linux resource and cleanup cases
run: |
report=".cache/rea-packet-required-tests.json"
mkdir -p .cache
rm -f "$report"
node scripts/run-exact-tests.mjs run --reporter=default --reporter=json --outputFile.json="$report" \
tests/boundary/captures/fieldDiscovery.test.ts src/process/LinuxResourceLimits.test.ts tests/boundary/captures/captureProvider.test.ts tests/boundary/captures/captureDescriptorCleanup.test.ts tests/boundary/pcap/pcapProvider.test.ts tests/boundary/pcap/pcapProviderCrossPlatform.test.ts tests/boundary/captures/resourceLimits.test.ts tests/boundary/captures/packetOutput.test.ts tests/boundary/mcp/packetCaptureMcp.test.ts tests/boundary/cli/captureCleanup.test.ts
node scripts/ci/require-provider-tests.mjs packet "$report"
- run: node scripts/verify-packet-capture.mjs
env:
REA_TSHARK_COMMAND: /usr/bin/tshark
+37 -24
View File
@@ -1,32 +1,35 @@
name: Real signal recording
on:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
remote_cache_write:
description: Trusted release qualification may publish the remote cache.
required: false
type: boolean
default: false
secrets:
TURBO_TOKEN:
description: Remote cache token. Only trusted callers pass it.
required: false
workflow_dispatch:
pull_request:
paths:
- "tests/boundary/signals/**"
- "tests/boundary/cli/signalCleanup.test.ts"
- "tests/boundary/mcp/signalRecordingMcp.test.ts"
- "src/signals/**"
- "src/domain/signals/**"
- "src/application/signals/**"
- "src/contracts/signals/**"
- "src/composition/signals.ts"
- "src/cli/signalCommands.ts"
- "src/server/registerSignalTools.ts"
- "scripts/verify-signal-recording.mjs"
- ".github/workflows/real-signal-recording.yml"
permissions:
contents: read
env:
HUSKY: "0"
# Writes only on the default branch for push, schedule, manual dispatch,
# and workflow_call. Pull requests get an empty token and remote read-only mode.
TURBO_TOKEN: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && secrets.TURBO_TOKEN || '' }}"
# A called workflow sees the caller event, so workflow_call is not an event
# name here. Writes when remote_cache_write is set, or on the default branch
# for push, schedule, and manual dispatch. Open pull requests stay empty-token
# and use the local cache only.
TURBO_TOKEN: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && secrets.TURBO_TOKEN || '' }}"
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
TURBO_CACHE: "${{ (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')) && 'local:rw,remote:rw' || 'local:rw,remote:r' }}"
concurrency:
group: real-signal-recording-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
TURBO_CACHE: "${{ (inputs.remote_cache_write == true || (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'))) && 'local:rw,remote:rw' || 'local:rw' }}"
permissions:
contents: read
jobs:
sigrok:
runs-on: ubuntu-24.04
@@ -41,8 +44,18 @@ jobs:
uses: ./.github/actions/npm-cache
- run: sudo apt-get update && sudo apt-get install -y sigrok-cli util-linux python3
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- run: npm run build:cached
- run: npm run test:focused -- tests/boundary/signals/decoderDiscovery.test.ts src/process/LinuxResourceLimits.test.ts src/domain/signals/annotationTextWindow.test.ts tests/boundary/signals/recordingProvider.test.ts tests/boundary/signals/recordingDescriptorCleanup.test.ts tests/boundary/signals/resourceLimits.test.ts tests/boundary/mcp/signalRecordingMcp.test.ts tests/boundary/cli/signalCleanup.test.ts
- name: Run focused tests and require Linux resource and cleanup cases
run: |
report=".cache/rea-signal-required-tests.json"
mkdir -p .cache
rm -f "$report"
node scripts/run-exact-tests.mjs run --reporter=default --reporter=json --outputFile.json="$report" \
tests/boundary/signals/decoderDiscovery.test.ts src/process/LinuxResourceLimits.test.ts src/domain/signals/annotationTextWindow.test.ts tests/boundary/signals/recordingProvider.test.ts tests/boundary/signals/recordingDescriptorCleanup.test.ts tests/boundary/signals/resourceLimits.test.ts tests/boundary/mcp/signalRecordingMcp.test.ts tests/boundary/cli/signalCleanup.test.ts
node scripts/ci/require-provider-tests.mjs signal "$report"
- run: node scripts/verify-signal-recording.mjs
env:
REA_SIGROK_COMMAND: /usr/bin/sigrok-cli
+20 -6
View File
@@ -301,8 +301,9 @@ Turbo caches deterministic preparation and static checks, but `test:run` always
executes. Repository files do not fully describe installed tool contents or host
state. A previous test pass therefore cannot replace current validation, even
when the source tree is unchanged. `check:pr` also runs the fast Python producer
suite; CI runs it in the static job for code changes. These producer tests do not
establish real LLDB engine coverage.
suite. Fast CI runs it from the selected Python job after installing dependencies
and preparing the compiled runtime; full qualification runs it from the deep
static job. These producer tests do not establish real LLDB engine coverage.
Vitest's persistent compile cache remains disabled. The `cliTest` fixture shares
a temporary Node compile cache among its fresh CLI subprocesses. The worker
@@ -851,7 +852,13 @@ Native, provider, fixture, SQLite compatibility, and website lanes wait for
the fast static checks (typecheck, built lint, and format) to succeed before
allocating their runners. Knip, duplication, architecture guards, test
discovery, and the Python producer run in the separate full-tier deep static
job and do not gate provider admission. Portable test shards and documentation
job and do not gate provider admission. On the fast tier, changes to Python
bridges, analysis-efficiency inputs, or the Python test runner select a separate
required Python job. That job installs locked dependencies and prepares the
compiled runtime before the suite, because analysis-efficiency checks import
built process modules. Bridge tests are discovered from the source tree. The
mitmproxy, native, and pwndbg families must each keep a test module, and a
newly added failing test fails the job. Portable test shards and documentation
checks remain independent of static results, so they can still provide
diagnostic feedback. The required gate rejects skipped selected lanes when
static checks fail; on the full tier it also requires the deep static job.
@@ -1579,7 +1586,14 @@ Inspector, setup, configuration and packaging entrypoints. Runtime sources under
and the Linux x64 installed-package/Inspector baseline only. Test-only changes
run the Linux suite without native package verification, except test files that
the Apple or Windows curated lanes invoke explicitly, which also select that
lane. Edits to verifier entrypoints run only by manual, licensed or self-hosted
lane. Packet-capture and signal-recording tests and their producers select the
corresponding real-tool lane; both also run in full qualification. The shared
Linux resource-limit source and its tests select both. Each lane requires
its Linux kernel resource-limit cases and all seven descriptor-cleanup cases to
pass in the focused Vitest report. A missing report, missing case, skipped case,
or duplicate identity fails the lane; unrelated host-specific skips remain
allowed. These tests use synthetic producers. The separate real TShark or
sigrok verifier establishes the actual-tool observation. Edits to verifier entrypoints run only by manual, licensed or self-hosted
workflows run static checks and the Linux suite. Website changes run website
checks. Provider workflow changes validate the workflow and run that provider's
verifier. Changes under `bridge/ghidra/` also select the `ghidra_bridge` lane,
@@ -1847,8 +1861,8 @@ Portable owner-based test selection remains observational. Its reports compare
proposed omissions with the full run; incomplete or skipped observations do not
establish that fewer tests would preserve regression detection.
The nine automatic browser, binary-layout, EVM, JavaScript-recovery,
recorded-crash, web and environment-bundle provider lanes download
The automatic browser, binary-layout, EVM, JavaScript-recovery, packet-capture,
signal-recording, recorded-crash, web and environment-bundle provider lanes download
`rea-build-cache` from their own run after the build and static jobs succeed. Their existing build commands still
validate task hashes and rebuild mismatched inputs; provider observations always
run. Provider-workflow-only edits also admit the build without enabling source
+1 -1
View File
@@ -152,7 +152,7 @@
"test": "npm run test:with-build",
"test:with-build": "npm run test:prepare && vitest run",
"test:run": "vitest run",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py bridge/pwndbg/test_core_context.py bridge/mitmproxy/test_capture.py -v && python3 -m unittest discover -s tests/support/analysis-efficiency -p 'test_*.py' -v",
"test:python": "python3 scripts/run-python-tests.py",
"test:local": "node scripts/run-development-tests.mjs local",
"test:fast": "node scripts/ci/test-group.mjs source",
"test:built": "node scripts/ci/test-group.mjs built",
+20 -1
View File
@@ -10,7 +10,13 @@ import {
import { planTestSelection } from "./test-selection.mjs";
// Test files select the host/runtime lanes that execute their focused suites.
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
const HOST_TEST_OWNERS = [
"apple",
"package",
"sqlite",
"packet_capture",
"signal_recording",
];
// Actionlint validates workflow syntax only. The semantic suite asserts these
// workflows' release, publication and self-hosted runner safeguards.
const GUARDED_WORKFLOWS = new Set([
@@ -111,6 +117,10 @@ function classifyPackage(base, head) {
selected.reverse_tools = true;
continue;
}
if (script === "test:python") {
selected.python = true;
continue;
}
if (script === "test:sqlite" || script === "test:sqlite:run") {
selected.sqlite = true;
continue;
@@ -216,6 +226,15 @@ if (tier === "reused") {
WINDOWS_EXECUTABLE_SELECTION_INPUTS,
);
if (matchesAny(file, SCOPES.reverse_tools)) selected.reverse_tools = true;
if (matchesAny(file, SCOPES.python)) selected.python = true;
// Both live capture lanes exercise Linux resource admission.
if (
matchesAny(file, ["src/process/LinuxResourceLimits*"]) ||
file === "scripts/ci/require-provider-tests.mjs"
) {
selected.packet_capture = true;
selected.signal_recording = true;
}
if (
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
file,
+67
View File
@@ -0,0 +1,67 @@
import { readFile } from "node:fs/promises";
import { isAbsolute, relative, resolve, sep } from "node:path";
const resources = [
"enforces both address-space limits before any producer allocation",
"preserves lower inherited file/CPU limits in the real launch",
];
const descriptors = [
"capture",
"tool",
"limiter",
"kernel",
"field-tool",
"field-limiter",
"field-kernel",
].map(
(member) =>
`retains failed ${member} descriptors and retries them before further acquisition and shutdown`,
);
const requirements = {
packet: {
"tests/boundary/captures/resourceLimits.test.ts": [
...resources,
"reports a real kernel file-limit termination with its effective ceiling",
],
"tests/boundary/captures/captureDescriptorCleanup.test.ts": descriptors,
},
signal: {
"tests/boundary/signals/resourceLimits.test.ts": resources,
"tests/boundary/signals/recordingDescriptorCleanup.test.ts": descriptors,
},
};
const [lane, reportPath] = process.argv.slice(2);
if (!Object.hasOwn(requirements, lane ?? "") || !reportPath)
throw new Error(
"Expected provider test lane (packet or signal) and report path",
);
const root = resolve(process.cwd());
const rootPrefix = `${root}${sep}`;
const reportFile = resolve(root, reportPath);
if (!reportFile.startsWith(rootPrefix)) {
const failure = { code: "report_path_outside_checkout" };
throw Object.assign(new Error(JSON.stringify(failure)), failure);
}
const report = JSON.parse(await readFile(reportFile, "utf8"));
if (report?.success !== true || !Array.isArray(report.testResults))
throw new Error(
"Provider test report is missing a successful test inventory",
);
for (const [path, cases] of Object.entries(requirements[lane])) {
const matches = report.testResults.filter((file) => {
if (typeof file?.name !== "string") return false;
const filePath = relative(process.cwd(), resolve(file.name));
return !isAbsolute(filePath) && filePath.split(sep).join("/") === path;
});
if (matches.length !== 1 || !Array.isArray(matches[0].assertionResults))
throw new Error(`Expected exactly one provider test result for ${path}`);
for (const name of cases) {
const results = matches[0].assertionResults.filter(
(test) => test?.fullName === name,
);
if (results.length !== 1 || results[0].status !== "passed")
throw new Error(`Required provider test did not pass: ${path}: ${name}`);
}
}
console.log(`Required ${lane} resource and cleanup cases passed`);
+1
View File
@@ -45,6 +45,7 @@ const lanes = {
readiness: ["reconstruction-readiness"],
docs: ["docs"],
workflows: ["static"],
python: [outputs.tier === "full" ? "static-deep" : "python"],
};
for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope]));
const failed = [...new Set(required)].filter(
+44
View File
@@ -12,6 +12,50 @@ export const WINDOWS_EXECUTABLE_SELECTION_INPUTS = [
// Shared build inputs are classified once in plan.mjs rather than repeated in
// every workflow.
export const SCOPES = {
python: [
"bridge/**/*.py",
"tests/support/analysis-efficiency/**",
"scripts/run-python-tests.py",
],
packet_capture: [
"tests/boundary/captures/**",
"tests/boundary/cli/captureCleanup.test.ts",
"tests/boundary/mcp/packetCaptureMcp.test.ts",
"tests/boundary/pcap/**",
"src/captures/**",
"src/pcap/**",
"src/domain/pcap/**",
"src/domain/analysisView/pcapCaptureView.ts",
"src/application/pcap/**",
"src/contracts/pcap/**",
"src/server/registerPcapTools.ts",
"src/domain/captures/**",
"src/application/captures/**",
"src/contracts/captures/**",
"src/composition/captures.ts",
"src/cli/captureCommands.ts",
"src/server/registerCaptureTools.ts",
"src/process/LinuxResourceLimits*",
"scripts/verify-packet-capture.mjs",
"scripts/ci/require-provider-tests.mjs",
".github/workflows/real-packet-capture.yml",
],
signal_recording: [
"tests/boundary/signals/**",
"tests/boundary/cli/signalCleanup.test.ts",
"tests/boundary/mcp/signalRecordingMcp.test.ts",
"src/signals/**",
"src/domain/signals/**",
"src/application/signals/**",
"src/contracts/signals/**",
"src/composition/signals.ts",
"src/cli/signalCommands.ts",
"src/server/registerSignalTools.ts",
"src/process/LinuxResourceLimits*",
"scripts/verify-signal-recording.mjs",
"scripts/ci/require-provider-tests.mjs",
".github/workflows/real-signal-recording.yml",
],
readiness: [
"tests/conformance/readiness/**",
"scripts/verify-reconstruction-readiness.mjs",
+50
View File
@@ -0,0 +1,50 @@
"""Run every source-owned Python bridge and analysis-efficiency test."""
from pathlib import Path
import json
import subprocess
import sys
root = Path(__file__).resolve().parent.parent
# The previous explicit command named these families. Discovery still runs
# every bridge test, and each named family must keep a test module.
REQUIRED_BRIDGE_FAMILIES = ("mitmproxy", "native", "pwndbg")
def fail(code, **details):
print(json.dumps({"code": code, **details}, sort_keys=True), file=sys.stderr)
raise SystemExit(1)
def family_tests(family):
return [
path
for path in root.joinpath("bridge", family).rglob("test_*.py")
if path.is_file()
]
missing_families = [
family for family in REQUIRED_BRIDGE_FAMILIES if not family_tests(family)
]
if missing_families:
fail("missing_bridge_family", families=missing_families)
bridge_tests = sorted(root.joinpath("bridge").rglob("test_*.py"))
if not bridge_tests:
fail("missing_bridge_tests")
if not list(root.joinpath("tests/support/analysis-efficiency").glob("test_*.py")):
fail("missing_analysis_efficiency_tests")
modules = [
".".join(path.relative_to(root).with_suffix("").parts) for path in bridge_tests
]
commands = [
[sys.executable, "-m", "unittest", "-v", *modules],
[
sys.executable, "-m", "unittest", "discover",
"-s", "tests/support/analysis-efficiency", "-p", "test_*.py", "-v",
],
]
for command in commands:
result = subprocess.run(command, cwd=root, check=False)
if result.returncode:
sys.exit(result.returncode)
+18 -18
View File
@@ -13,7 +13,7 @@ import { parseEvidence, createEvidence } from "../../src/domain/evidence.js";
import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js";
import { connectLocalToolsMcp } from "../fixtures/localToolsMcp.js";
import { parseMcpToolError } from "../fixtures/mcpToolError.js";
import { cliTest } from "../support/cli/cliFixture.js";
import { cliTest, runFileCli } from "../support/cli/cliFixture.js";
import {
cudaFixture,
peGpuFixture,
@@ -88,7 +88,7 @@ cliTest(
const bytes = Buffer.concat([header, compressed]),
path = join(root, `v${version}`);
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -147,7 +147,7 @@ cliTest(
const bytes = hipFixture(payload, bundleId),
path = join(root, name);
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -197,7 +197,7 @@ cliTest(
}
const path = join(root, "raw-bitcode");
await writeFile(path, bitcode);
const raw = await cli.run({
const raw = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -235,7 +235,7 @@ cliTest(
]),
path = join(root, name);
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -319,7 +319,7 @@ cliTest(
]),
path = join(root, name);
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -360,7 +360,7 @@ cliTest(
]),
path = join(root, "large-parameter.cubin");
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -397,7 +397,7 @@ cliTest(
const bytes = hipFixture(payload, "hip-amdgcn-amd-amdhsa--gfx906+xnack"),
path = join(root, "feature.bundle");
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -442,7 +442,7 @@ cliTest(
root = await createTestTempDirectory("rea-ptx-callprototype-"),
path = join(root, "prototype.ptx");
await writeFile(path, source);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -492,7 +492,7 @@ cliTest(
const bytes = cubinFixture({ extendedSectionIndex: true });
const path = join(root, "extended.cubin");
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -540,7 +540,7 @@ cliTest(
malformedBytes.writeBigUInt64LE(4n, sectionTable + indexTable! * 64 + 32);
const malformedPath = join(root, "short-shndx.cubin");
await writeFile(malformedPath, malformedBytes);
const malformed = await cli.run({
const malformed = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
malformedPath,
@@ -571,7 +571,7 @@ cliTest(
const text = `.version 7.0\n.target sm_80\n.address_size 64\n.visible .entry kernel() {\n.reg .b32 ${register};\nmov.u32 ${register}, 0;\nret;\n}\n`;
const path = join(root, `${register.slice(1)}.ptx`);
await writeFile(path, text);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -608,7 +608,7 @@ cliTest(
const text = `.version 8.8\n.target sm_80\n.address_size 64\n.entry %kernel(.param .u32 input) {\nret;\n}\n`;
const path = join(root, "identifiers.ptx");
await writeFile(path, text);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -671,7 +671,7 @@ cliTest(
const text = `${preamble}${declaration}\n`,
path = join(root, `${name}.ptx`);
await writeFile(path, text);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -721,7 +721,7 @@ cliTest(
const text = `.version 8.8\n.target compute_80\n.address_size 64\n.entry kernel() {\nret;\n}\n`;
const path = join(root, "target.ptx");
await writeFile(path, text);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -751,7 +751,7 @@ cliTest(
const text = `.version 1.3\n.target sm_80\n.address_size 64\n.entry kernel {\n.param .u32 input;\nret;\n}\n`;
const path = join(root, "legacy.ptx");
await writeFile(path, text);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
@@ -790,7 +790,7 @@ cliTest(
malformedPath,
`.version 1.3\n.target sm_80\n.entry kernel {\n.param .u32 input\n`,
);
const malformed = await cli.run({
const malformed = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
malformedPath,
@@ -1273,7 +1273,7 @@ cliTest(
] as const) {
const path = join(root, name);
await writeFile(path, bytes);
const response = await cli.run({
const response = await runFileCli(cli, {
arguments: [
"inspect-gpu-code-objects",
path,
+17 -17
View File
@@ -53,7 +53,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
`${base}[0,1].value,${base}.1.value`,
`${base}.1.value,${base}[0,1].value`,
]) {
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -72,7 +72,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
},
});
}
const refused = await cli.run({
const refused = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -105,7 +105,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
} finally {
database.close();
}
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -383,7 +383,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
},
},
});
const cliResult = await cli.run({
const cliResult = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -529,7 +529,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
}
const { call } = await connectLocalToolsMcp();
for (const limit of [1001, Number.MAX_SAFE_INTEGER]) {
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -752,7 +752,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
const wal = before[1];
expect(db).toBeDefined();
expect(wal).toBeDefined();
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -818,7 +818,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
],
},
};
const corrupt = await cli.run({
const corrupt = await runFileCli(cli, {
arguments: ["inspect-sqlite-database", selected, "--json"],
environment: { REA_LOG_LEVEL: "silent" },
});
@@ -854,7 +854,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
},
},
});
const ambiguous = await cli.run({
const ambiguous = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
valid,
@@ -874,7 +874,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
})
).isError,
).toBe(true);
const missing = await cli.run({
const missing = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
valid,
@@ -924,7 +924,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
producer.close();
}
const before = await sourceState(path);
const control = await cli.run({
const control = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1011,7 +1011,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
const path = createSqliteApplicationCollationFixture(root);
const before = await sourceState(path);
const { call } = await connectLocalToolsMcp();
const schema = await cli.run({
const schema = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1050,7 +1050,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
expect(schemaResponse.isError).not.toBe(true);
expect(schemaResponse.structuredContent).toMatchObject(expectedSchema);
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1125,7 +1125,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
},
};
for (const columns of [["b", "a"], undefined]) {
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1146,7 +1146,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
});
expect(parseMcpToolError(response).error).toMatchObject(expectedError);
}
const base = await cli.run({
const base = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1203,7 +1203,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
"selected_table",
"database",
] as const) {
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1348,7 +1348,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
native.close();
}
const cliResult = await cli.run({
const cliResult = await runFileCli(cli, {
arguments: [
"inspect-sqlite-database",
path,
@@ -1427,7 +1427,7 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)(
producer.close();
}
const original = await readFile(path);
const result = await cli.run({
const result = await runFileCli(cli, {
arguments: ["inspect-sqlite-database", path, "--json"],
environment: { REA_LOG_LEVEL: "silent" },
});
@@ -103,6 +103,27 @@ async function classify(
return z.record(z.string(), z.unknown()).parse(JSON.parse(result.stdout));
}
it.each([
["bridge/native/rea_lldb_tracer.py", "python"],
["bridge/pwndbg/test_core_context.py", "python"],
["tests/support/analysis-efficiency/test_signal_oracle.py", "python"],
["scripts/run-python-tests.py", "python"],
["src/captures/CaptureProvider.ts", "packet_capture"],
["tests/boundary/mcp/packetCaptureMcp.test.ts", "packet_capture"],
["src/signals/RecordingProvider.ts", "signal_recording"],
["tests/boundary/mcp/signalRecordingMcp.test.ts", "signal_recording"],
["src/process/LinuxResourceLimits.ts", "packet_capture"],
["src/process/LinuxResourceLimits.ts", "signal_recording"],
["scripts/ci/require-provider-tests.mjs", "packet_capture"],
["scripts/ci/require-provider-tests.mjs", "signal_recording"],
])("selects %s for %s", async (path, scope) => {
const { directory, base } = await fixture();
const head = await change(directory, [path]);
expect(await classify(directory, base, head)).toMatchObject({
[scope]: true,
});
});
it("reuses only scheduled qualification and preserves every explicit full request", async () => {
const { directory, base } = await fixture();
const reused = await classify(directory, base, base, "schedule", false, true);
@@ -213,6 +213,7 @@ it("holds native and provider fan-out behind static validation while portable fe
"changes",
"static",
"static-deep",
"python",
"build",
"test-shard",
"test-source",
@@ -382,6 +383,8 @@ it("binds provider build reuse to the current run while keeping standalone dispa
"web_runtime",
"web_source_map",
"agentenv",
"packet_capture",
"signal_recording",
];
const buildTerms = z
.string()
@@ -574,6 +577,9 @@ it.each([
["workflows", "static"],
["ghidra_bridge", "ghidra_bridge"],
["agentenv", "agentenv"],
["python", "python"],
["packet_capture", "packet_capture"],
["signal_recording", "signal_recording"],
] as const) {
// GitHub exposes only direct dependencies in the required job's needs context.
const needs: Record<
@@ -60,6 +60,7 @@ const workflow = async () =>
test: z.object({ steps: stepsSchema }),
build: z.object({ steps: stepsSchema }),
docs: z.object({ steps: stepsSchema }),
python: z.object({ if: z.string(), steps: stepsSchema }),
static: z.object({ steps: stepsSchema }),
"static-deep": z.object({
if: z.string(),
@@ -415,9 +416,27 @@ it("runs the existing onboarding and Python producer guards from their selected
expect(scripts["check:ci:fast:tasks"]).toBe(
"node scripts/run-turbo.mjs run typecheck lint format:check",
);
expect(scripts["test:python"]).toContain(
"bridge/native/test_rea_lldb_tracer.py",
expect(scripts["test:python"]).toBe("python3 scripts/run-python-tests.py");
expect(jobs.python.if).toContain("outputs.python == 'true'");
expect(jobs.python.if).toContain("tier == 'fast'");
const downloadCache = jobs.python.steps.findIndex(
(step) => step.uses === "./.github/actions/npm-cache",
);
const install = jobs.python.steps.findIndex((step) => step.run === "npm ci");
const prepare = jobs.python.steps.findIndex(
(step) => step.run === "npm run test:prepare",
);
const suite = jobs.python.steps.findIndex(
(step) => step.run === "npm run test:python",
);
const cache = jobs.python.steps.findIndex(
(step) => step.uses === "./.github/actions/portable-build-cache",
);
expect(downloadCache).toBeGreaterThanOrEqual(0);
expect(install).toBeGreaterThan(downloadCache);
expect(cache).toBeGreaterThan(install);
expect(prepare).toBeGreaterThan(cache);
expect(suite).toBeGreaterThan(prepare);
});
it("exports portable timing history and removes it on a failed, partial, or invalid rerun", async () => {
@@ -713,3 +732,56 @@ it("records failed omissions from full reports without treating partial or unhan
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
});
it("discovers Python bridge tests and fails on a newly added failing producer test", async () => {
const root = await createTestTempDirectory("rea-python-inventory-");
await mkdir(join(root, "scripts"), { recursive: true });
await mkdir(join(root, "bridge", "example"), { recursive: true });
await mkdir(join(root, "tests", "support", "analysis-efficiency"), {
recursive: true,
});
await writeFile(
join(root, "scripts", "run-python-tests.py"),
await readFile(
new URL("../../../scripts/run-python-tests.py", import.meta.url),
),
);
const passing =
"import unittest\nclass Example(unittest.TestCase):\n def test_case(self): pass\n";
for (const family of ["mitmproxy", "native", "pwndbg"]) {
await mkdir(join(root, "bridge", family), { recursive: true });
await writeFile(join(root, "bridge", family, "capture.py"), "VALUE = 1\n");
await writeFile(join(root, "bridge", family, "test_case.py"), passing);
}
await writeFile(join(root, "bridge", "example", "test_first.py"), passing);
await writeFile(
join(root, "tests", "support", "analysis-efficiency", "test_example.py"),
passing,
);
const command = ["scripts/run-python-tests.py"];
const discovered = await execute("python3", command, { cwd: root });
expect(discovered.stderr).toContain("bridge.mitmproxy.test_case");
expect(discovered.stderr).toContain("bridge.native.test_case");
expect(discovered.stderr).toContain("bridge.pwndbg.test_case");
expect(discovered.stderr).toContain("OK");
await writeFile(
join(root, "bridge", "example", "test_second.py"),
"import unittest\nclass Example(unittest.TestCase):\n def test_case(self): self.fail('new test executed')\n",
);
await expect(
execute("python3", command, { cwd: root }),
).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("new test executed"),
});
await rm(join(root, "bridge", "mitmproxy", "test_case.py"));
const missingFamily = await execute("python3", command, { cwd: root }).then(
() => undefined,
(error: { code?: unknown; stderr?: unknown }) => error,
);
expect(missingFamily).toMatchObject({ code: 1 });
expect(JSON.parse(String(missingFamily?.stderr))).toEqual({
code: "missing_bridge_family",
families: ["mitmproxy"],
});
});
@@ -0,0 +1,215 @@
import { execFileSync } from "node:child_process";
import {
mkdirSync,
mkdtempSync,
readFileSync,
rmSync,
writeFileSync,
} from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
const script = fileURLToPath(
new URL("../../scripts/ci/require-provider-tests.mjs", import.meta.url),
);
const cleanupCases = [
"capture",
"tool",
"limiter",
"kernel",
"field-tool",
"field-limiter",
"field-kernel",
].map(
(item) =>
`retains failed ${item} descriptors and retries them before further acquisition and shutdown`,
);
const commonLimits = [
"enforces both address-space limits before any producer allocation",
"preserves lower inherited file/CPU limits in the real launch",
];
const testFile = (path: string, names: string[]) => ({
name: resolve(path),
assertionResults: names.map((fullName) => ({ fullName, status: "passed" })),
});
const report = (lane: "packet" | "signal") => ({
success: true,
testResults:
lane === "packet"
? [
testFile("tests/boundary/captures/resourceLimits.test.ts", [
...commonLimits,
"reports a real kernel file-limit termination with its effective ceiling",
]),
testFile(
"tests/boundary/captures/captureDescriptorCleanup.test.ts",
cleanupCases,
),
]
: [
testFile(
"tests/boundary/signals/resourceLimits.test.ts",
commonLimits,
),
testFile(
"tests/boundary/signals/recordingDescriptorCleanup.test.ts",
cleanupCases,
),
],
});
const run = (lane: string, value: unknown) => {
mkdirSync(join(process.cwd(), ".cache"), { recursive: true });
const directory = mkdtempSync(
join(process.cwd(), ".cache", "rea-required-cases-"),
);
try {
const path = join(directory, "report.json");
writeFileSync(path, JSON.stringify(value));
return execFileSync(process.execPath, [script, lane, path], {
cwd: process.cwd(),
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
} finally {
rmSync(directory, { recursive: true, force: true });
}
};
const outsideReportFailure = (reportPath: string) => {
try {
execFileSync(process.execPath, [script, "packet", reportPath], {
cwd: process.cwd(),
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
});
return undefined;
} catch (error) {
return error;
}
};
it.each(["packet", "signal"] as const)(
"requires the %s resource and descriptor cases without banning unrelated skips",
(lane) => {
const result = report(lane);
result.testResults.push({
name: resolve("tests/unrelated.test.ts"),
assertionResults: [{ fullName: "host-specific", status: "pending" }],
});
expect(run(lane, result)).toContain("cases passed");
for (const resultIndex of [0, 1]) {
const skipped = report(lane);
skipped.testResults[resultIndex]!.assertionResults[0]!.status = "pending";
expect(() => run(lane, skipped)).toThrow(
"Required provider test did not pass",
);
const failed = report(lane);
failed.testResults[resultIndex]!.assertionResults[0]!.status = "failed";
expect(() => run(lane, failed)).toThrow(
"Required provider test did not pass",
);
const removed = report(lane);
removed.testResults[resultIndex]!.assertionResults.shift();
expect(() => run(lane, removed)).toThrow(
"Required provider test did not pass",
);
const duplicate = report(lane);
duplicate.testResults[resultIndex]!.assertionResults.push(
duplicate.testResults[resultIndex]!.assertionResults[0]!,
);
expect(() => run(lane, duplicate)).toThrow(
"Required provider test did not pass",
);
}
const missingFile = report(lane);
missingFile.testResults.pop();
expect(() => run(lane, missingFile)).toThrow(
"Expected exactly one provider test result",
);
expect(() => run(lane, { success: true, testResults: [] })).toThrow();
expect(() => run(lane, { success: false, testResults: [] })).toThrow();
},
);
it.each([
["parent", resolve(process.cwd(), "..", "rea-provider-report.json")],
["sibling prefix", `${resolve(process.cwd())}-outside/report.json`],
["absolute outside", resolve(tmpdir(), "rea-provider-report.json")],
] as const)(
"rejects a %s report path outside the checkout",
(_label, reportPath) => {
const failure = z
.object({ status: z.literal(1), stderr: z.string() })
.parse(outsideReportFailure(reportPath));
const encoded = failure.stderr.match(
/\{"code":"report_path_outside_checkout"\}/u,
)?.[0];
expect(JSON.parse(encoded ?? "")).toEqual({
code: "report_path_outside_checkout",
});
},
);
it("rejects unknown lanes and missing or malformed reports", () => {
expect(() => run("other", report("packet"))).toThrow(
"Expected provider test lane",
);
expect(() => run("packet", null)).toThrow(
"missing a successful test inventory",
);
expect(() =>
execFileSync(process.execPath, [script, "packet", "/missing/report.json"], {
stdio: "pipe",
}),
).toThrow();
});
it.each([
[
"real-packet-capture.yml",
"tshark",
"packet",
"tests/boundary/captures/resourceLimits.test.ts",
"tests/boundary/captures/captureDescriptorCleanup.test.ts",
],
[
"real-signal-recording.yml",
"sigrok",
"signal",
"tests/boundary/signals/resourceLimits.test.ts",
"tests/boundary/signals/recordingDescriptorCleanup.test.ts",
],
] as const)(
"keeps required case validation in %s",
(workflow, job, lane, limits, cleanup) => {
const file = new URL(
`../../.github/workflows/${workflow}`,
import.meta.url,
);
const parsed = z
.object({
jobs: z.record(
z.string(),
z.object({
steps: z.array(z.object({ run: z.string().optional() })),
}),
),
})
.parse(parse(readFileSync(file, "utf8")));
const step = parsed.jobs[job]?.steps.find((item) =>
item.run?.includes("scripts/ci/require-provider-tests.mjs"),
);
expect(step?.run).toContain('rm -f "$report"');
expect(step?.run).toContain("node scripts/run-exact-tests.mjs run");
expect(step?.run).toContain("--reporter=json");
expect(step?.run).toContain('--outputFile.json="$report"');
expect(step?.run).toContain(limits);
expect(step?.run).toContain(cleanup);
expect(step?.run).toContain(
`node scripts/ci/require-provider-tests.mjs ${lane} "$report"`,
);
},
);