diff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.js index 261595600..70085d107 100644 --- a/scripts/dev/gateguard-eval.js +++ b/scripts/dev/gateguard-eval.js @@ -62,6 +62,9 @@ function validateStep(scenario, step, index) { if (!step.payload || typeof step.payload !== 'object') fail(`${where}: missing payload`); if (typeof step.payload.tool_name !== 'string') fail(`${where}: payload.tool_name must be a string`); if (!EXPECTATIONS.has(step.expect)) fail(`${where}: expect must be deny or allow`); + if (step.expectOnWindows !== undefined && step.expectOnWindows !== 'deny') { + fail(`${where}: expectOnWindows may only strengthen the expectation to deny`); + } if (typeof step.mustDeny !== 'boolean') fail(`${where}: mustDeny must be a boolean`); if (typeof step.redundant !== 'boolean') fail(`${where}: redundant must be a boolean`); if (step.mustDeny && step.expect !== 'deny') fail(`${where}: mustDeny requires expect deny`); @@ -86,7 +89,7 @@ function validateScenario(scenario, file) { } /** Load and validate every scenario file in a corpus directory, sorted by file name. */ -function loadCorpus(dir = DEFAULT_CORPUS_DIR) { +function loadCorpus(dir = DEFAULT_CORPUS_DIR, platform = process.platform) { return fs .readdirSync(dir) .filter(name => name.endsWith('.json')) @@ -94,7 +97,12 @@ function loadCorpus(dir = DEFAULT_CORPUS_DIR) { .map(name => { const scenario = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8')); validateScenario(scenario, name); - return { ...scenario, file: name }; + // Folded Windows filenames cannot prove case-sensitive type coverage. + // Normalize the shared corpus so assertions, summaries and SARIF agree. + const steps = scenario.steps.map(step => platform === 'win32' && step.expectOnWindows === 'deny' + ? { ...step, expect: 'deny', mustDeny: true, redundant: false } + : step); + return { ...scenario, steps, file: name }; }); } diff --git a/tests/fixtures/gateguard-scenarios/21-bypass-filter-forms.json b/tests/fixtures/gateguard-scenarios/21-bypass-filter-forms.json index d4bdc73e1..a79358630 100644 --- a/tests/fixtures/gateguard-scenarios/21-bypass-filter-forms.json +++ b/tests/fixtures/gateguard-scenarios/21-bypass-filter-forms.json @@ -1022,6 +1022,7 @@ }, { "id": "rg-type-includes-target", + "expectOnWindows": "deny", "transcript": [ { "type": "user", diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.js index 690c888e8..fed1c19c8 100644 --- a/tests/scripts/gateguard-eval.test.js +++ b/tests/scripts/gateguard-eval.test.js @@ -67,6 +67,44 @@ function results(log) { console.log('\n=== Testing gateguard-eval ===\n'); +test('the corpus preserves POSIX type credit and requires denial for folded Windows names', () => { + const posix = loadCorpus(undefined, 'linux'); + const windows = loadCorpus(undefined, 'win32'); + const typeStep = corpus => corpus.find(scenario => scenario.name === 'bypass-filter-forms') + .steps.find(candidate => candidate.id === 'rg-type-includes-target'); + assert.strictEqual(typeStep(posix).expect, 'allow'); + assert.strictEqual(typeStep(posix).redundant, true); + assert.strictEqual(typeStep(windows).expect, 'deny'); + assert.strictEqual(typeStep(windows).mustDeny, true); + assert.strictEqual(typeStep(windows).redundant, false); + posix.forEach((scenario, index) => scenario.steps.forEach((candidate, at) => { + if (candidate.mustDeny) assert.deepStrictEqual(windows[index].steps[at], candidate); + })); + const scenario = windows.find(candidate => candidate.name === 'bypass-filter-forms'); + const expected = typeStep(windows); + const run = decision => [{ scenario: { ...scenario, steps: [expected] }, results: [{ + decision, kind: decision === 'deny' ? 'deny' : 'credit', latencyMs: 0, reason: '' + }] }]; + assert.strictEqual(summarize(run('deny')).totals.mismatches, 0); + assert.strictEqual(summarize(run('deny')).totals.redundantDenials, 0); + assert.strictEqual(summarize(run('allow')).totals.mustDenyBypasses, 1); +}); + +test('Windows corpus overrides only strengthen denial and reject invalid values on every host', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-platform-corpus-')); + try { + const scenario = loadCorpus()[0]; + for (const expectOnWindows of ['allow', 'invalid', false]) { + fs.writeFileSync(path.join(dir, 'fixture.json'), JSON.stringify({ + ...scenario, steps: [{ ...scenario.steps[0], expectOnWindows }] + })); + for (const platform of ['linux', 'win32']) assert.throws(() => loadCorpus(dir, platform), /expectOnWindows/); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test('--sarif takes a file path and keeps the other options', () => { const options = parseArgs(['--json', '--sarif', 'out/gate.sarif', '--baseline', 'main']); assert.strictEqual(options.format, 'json');