diff --git a/.github/workflows/maintenance.yml b/.github/workflows/maintenance.yml index ca724b328..964570fda 100644 --- a/.github/workflows/maintenance.yml +++ b/.github/workflows/maintenance.yml @@ -54,3 +54,22 @@ jobs: stale-pr-message: 'This PR is stale due to inactivity.' days-before-stale: 30 days-before-close: 7 + + closed-reference-audit: + name: Audit Closed-Issue PR References + runs-on: ubuntu-latest + permissions: + contents: read + issues: read + pull-requests: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '20.x' + - name: Verify cited PRs in closed issues are actually merged + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/ci/audit-closed-references.js diff --git a/scripts/ci/audit-closed-references.js b/scripts/ci/audit-closed-references.js new file mode 100644 index 000000000..0f05bfca9 --- /dev/null +++ b/scripts/ci/audit-closed-references.js @@ -0,0 +1,336 @@ +#!/usr/bin/env node +/** + * Audit recently closed issues whose asserted delivery claims cite a PR that + * is not actually merged. + * + * Motivated by #3223: an issue closed as "fixed in #N" while #N was closed + * without merging leaves a completion claim that does not hold up. This audit + * runs weekly from .github/workflows/maintenance.yml, scans issue text and + * close-window comments for *asserted* delivery claims, verifies each cited + * PR's merge state, and reports findings to the workflow step summary. + * + * Evidence contract (per the #3522 review): + * - Only asserted delivery claims ("delivered through #N", "fixed in #N") + * produce findings. Historical mentions, quoted text, fenced examples, + * negated claims, and tracking-only references are reported separately as + * context — a merged PR alone also does not certify issue completion. + * - Observed states are reported distinctly: merged, open-unmerged, + * closed-unmerged, and unknown (404, rate limit, transient failures). An + * unavailable PR warrants manual verification; it does not establish that + * a claim failed. + * - Collection is paginated and its completeness is visible: skipped pages, + * rate limits, and retrieval failures turn the report into an explicitly + * incomplete audit rather than an all-clear. + * + * Read-only by design: findings are advisory, the script never comments on or + * closes anything, and always exits 0 so the scheduled run never breaks. + */ + +'use strict'; + +const fs = require('fs'); +const { execFileSync } = require('child_process'); + +const DEFAULT_WINDOW_DAYS = 30; +const MAX_COMMENT_REQUESTS = 200; +const MAX_PR_CHECKS = 100; + +const CLAIM_PATTERN = + /\b(?:delivered|delivers|fixed|fixes|shipped|ships|landed|lands|resolved|resolves|implemented|implements|completed|completes|closed|closes|merged|merges)\s+(?:through|in|via|by|with|as|into)\s+(?:PR\s+)?#(\d+)\b/gi; +const PR_REF_PATTERN = /\bPR\s+#(\d+)\b/gi; +const ANY_REF_PATTERN = /(?:^|[^\w&])#(\d+)\b/g; +const NEGATION_PATTERN = + /\b(?:not|never|cannot|can't|won't|doesn't|does\s+not|no\s+longer|isn't|wasn't)\s+(?:\w+\s+){0,2}$/i; + +function runGh(args, env) { + const options = { encoding: 'utf8', env, timeout: 60000, maxBuffer: 64 * 1024 * 1024 }; + if (env.AUDIT_GH_SCRIPT) { + // Testability hook: run a fixture script instead of the real gh CLI. + return execFileSync(process.execPath, [env.AUDIT_GH_SCRIPT, ...args], options); + } + return execFileSync('gh', args, options); +} + +function describeGhError(error) { + const stderr = (error && error.stderr) || ''; + const http = stderr.match(/HTTP (\d{3})/); + if (http) return `HTTP ${http[1]}`; + return String((error && error.message) || error).split('\n')[0]; +} + +// Pure: fenced code blocks, inline code, and blockquoted lines are examples or +// quotation, not assertions made by the author — strip them before matching. +function stripNonEvidence(text) { + if (!text) return ''; + return text + .replace(/```[\s\S]*?```/g, ' ') + .replace(/`[^`\n]*`/g, ' ') + .split('\n') + .filter((line) => !/^\s*>/.test(line)) + .join('\n'); +} + +// Pure: a claim pattern preceded by negation words is not an assertion. +function isNegated(clean, matchIndex) { + const before = clean.slice(Math.max(0, matchIndex - 60), matchIndex); + return NEGATION_PATTERN.test(before); +} + +// Pure: classify every #N in one text as an asserted delivery claim or a +// contextual reference. Deduplicates per (pr, kind). In a closing comment a +// bare "PR #N" reference is the closing evidence; in issue text it stays +// context. +function extractRefs(text, source, sourceUrl) { + const clean = stripNonEvidence(text); + if (!clean) return []; + const refs = []; + const claimMatches = [...clean.matchAll(CLAIM_PATTERN)]; + if (source === 'closing-comment') claimMatches.push(...clean.matchAll(PR_REF_PATTERN)); + for (const match of claimMatches) { + if (isNegated(clean, match.index)) continue; + refs.push({ pr: Number(match[1]), kind: 'claim', source, sourceUrl, snippet: match[0] }); + } + const ranges = claimMatches.map((m) => [m.index, m.index + m[0].length]); + for (const match of clean.matchAll(ANY_REF_PATTERN)) { + const insideClaim = ranges.some(([start, end]) => match.index >= start && match.index < end); + if (insideClaim || isNegated(clean, match.index)) continue; + refs.push({ pr: Number(match[1]), kind: 'reference', source, sourceUrl, snippet: match[0].trim() }); + } + const seen = new Set(); + return refs.filter((ref) => (seen.has(ref.pr + ':' + ref.kind) ? false : (seen.add(ref.pr + ':' + ref.kind), true))); +} + +// Pure: issues closed within the look-back window; PRs are not issues. +function selectRecentClosedIssues(issues, nowMs, windowDays) { + const cutoff = nowMs - windowDays * 24 * 60 * 60 * 1000; + return issues.filter( + (issue) => + issue.state === 'closed' && + !issue.pull_request && + typeof issue.closed_at === 'string' && + Date.parse(issue.closed_at) >= cutoff + ); +} + +// Pure: comment objects (body + html_url) inside the window, starting at the +// issue's closure so the closing comment always counts as evidence. +function selectWindowComments(comments, issueClosedAt, windowDays, nowMs) { + const cutoff = nowMs - windowDays * 24 * 60 * 60 * 1000; + return comments + .filter((comment) => typeof comment.body === 'string') + .filter((comment) => Date.parse(comment.created_at) >= Math.max(cutoff, Date.parse(issueClosedAt) - 60_000)); +} + +// Pure: one entry per issue with its claims and contextual references. +function extractEvidence(issue, comments, nowMs) { + const refs = [ + ...extractRefs([issue.title, issue.body].filter(Boolean).join('\n'), 'issue-body', issue.html_url), + ]; + for (const comment of selectWindowComments(comments, issue.closed_at, DEFAULT_WINDOW_DAYS, nowMs)) { + refs.push(...extractRefs(comment.body, 'closing-comment', comment.html_url)); + } + return refs; +} + +// Collect issues + evidence. Fetch failures are surfaced as incomplete notes, +// never silently swallowed into an empty list. +function collectEntries(fetchIssues, fetchComments, nowMs, windowDays) { + const { items: issues, error } = fetchIssues(); + if (error) return { entries: [], incomplete: [`issue list unavailable: ${error}`] }; + const incomplete = []; + const entries = []; + for (const issue of selectRecentClosedIssues(issues, nowMs, windowDays)) { + const { items: comments, error: commentError } = fetchComments(issue.number); + if (commentError) incomplete.push(`#${issue.number} comments unavailable: ${commentError}`); + const refs = extractEvidence(issue, comments, nowMs); + if (refs.length) entries.push({ issue: issue.number, issueUrl: issue.html_url, refs }); + } + return { entries, incomplete }; +} + +function uniqueClaimPrs(entries) { + return [...new Set(entries.flatMap((entry) => entry.refs.filter((r) => r.kind === 'claim').map((r) => r.pr)))]; +} + +function collectReferences(entries) { + const seen = new Set(); + const refs = []; + for (const entry of entries) { + for (const ref of entry.refs.filter((r) => r.kind === 'reference')) { + const key = entry.issue + ':' + ref.pr; + if (seen.has(key)) continue; + seen.add(key); + refs.push({ issue: entry.issue, pr: ref.pr }); + } + } + return refs; +} + +// Pure: one verdict row per asserted claim. Merged passes; open/closed +// unmerged are findings; unknown is unverifiable and needs manual verification. +function buildVerdicts(entries, pullStates) { + const rows = []; + for (const entry of entries) { + for (const ref of entry.refs.filter((r) => r.kind === 'claim')) { + const state = pullStates.get(ref.pr) || { observed: 'unknown', reason: 'not fetched' }; + rows.push({ + issue: entry.issue, pr: ref.pr, snippet: ref.snippet, source: ref.source, + sourceUrl: ref.sourceUrl, observed: state.observed, reason: state.reason || null, + verdict: state.observed === 'merged' ? 'pass' : state.observed === 'unknown' ? 'unverifiable' : 'unmerged', + }); + } + } + return rows; +} + +function summaryIncomplete(lines, incomplete) { + if (!incomplete.length) return; + lines.push('', '**Audit incomplete** — some evidence could not be collected; absence of findings below is not an all-clear:'); + for (const note of incomplete) lines.push(`- ${note}`); +} + +function summaryFindings(lines, unmerged) { + if (!unmerged.length) return; + lines.push('', '### Findings — asserted delivery claims whose cited PR is not merged'); + for (const v of unmerged) { + lines.push(`- #${v.issue} claims "${v.snippet}" (source: ${v.source}, ${v.sourceUrl}) → #${v.pr}: ${v.observed}.`); + } +} + +function summaryUnverifiable(lines, unverifiable) { + if (!unverifiable.length) return; + lines.push('', '### Unverifiable — manual verification required'); + for (const v of unverifiable) { + lines.push(`- #${v.issue} claims "${v.snippet}" (${v.sourceUrl}) → #${v.pr}: unavailable (${v.reason}). An unavailable PR does not establish that the claim failed.`); + } +} + +// Traceability: every checked claim is listed with its evidence link, passes +// included — an all-clear line alone cannot be audited. +function summaryPasses(lines, passes) { + if (!passes.length) return; + lines.push('', '### Checked claims — cited PR is merged'); + for (const v of passes) { + lines.push(`- #${v.issue} claims "${v.snippet}" (source: ${v.source}, ${v.sourceUrl}) → #${v.pr}: merged.`); + } +} + +// Pure: markdown lines for the step summary. +function buildSummary(verdicts, references, incomplete) { + const lines = ['## Closed-issue delivery-claim audit', '', `Observed: ${new Date().toISOString()}`]; + summaryIncomplete(lines, incomplete); + const unmerged = verdicts.filter((v) => v.verdict === 'unmerged'); + const unverifiable = verdicts.filter((v) => v.verdict === 'unverifiable'); + lines.push('', `${verdicts.length} asserted delivery claim(s) found: ${unmerged.length} unmerged finding(s), ${unverifiable.length} unverifiable.`); + summaryFindings(lines, unmerged); + summaryUnverifiable(lines, unverifiable); + summaryPasses(lines, verdicts.filter((v) => v.verdict === 'pass')); + if (references.length) { + lines.push('', '### Contextual references (not delivery claims)'); + const byIssue = new Map(); + for (const r of references) { + if (!byIssue.has(r.issue)) byIssue.set(r.issue, []); + byIssue.get(r.issue).push('#' + r.pr); + } + for (const [issue, prs] of byIssue) { + lines.push(`- #${issue}: ${[...new Set(prs)].join(', ')}`); + } + } + if (!verdicts.length && !references.length && !incomplete.length) { + lines.push('No asserted delivery claims found in recently closed issues.'); + } + if (verdicts.length && !unmerged.length) { + lines.push('', 'No unmerged findings. A merged cited PR supports the claim; it does not by itself certify issue completion, current code after a revert, or package publication.'); + } + return lines; +} + +// Paginated gh api fetch: --paginate --slurp yields one array per page. +function fetchAllPages(args, env) { + try { + const raw = JSON.parse(runGh(['api', '--paginate', '--slurp', args], env)); + return { items: Array.isArray(raw) ? raw.flat() : [], error: null }; + } catch (error) { + return { items: [], error: describeGhError(error) }; + } +} + +// Comment fetching is bounded by a request budget; exhaustion is reported as +// incompleteness instead of passing as an empty list. +function makeCommentFetcher(env) { + let budget = MAX_COMMENT_REQUESTS; + return (number) => { + if (budget-- <= 0) return { items: [], error: 'comment request budget exhausted' }; + return fetchAllPages(`repos/${env.GITHUB_REPOSITORY}/issues/${number}/comments?per_page=100`, env); + }; +} + +// Four observed states, kept distinct: merged / open-unmerged / +// closed-unmerged / unknown (404, rate limit, transient failure). +function fetchPullState(env, number) { + try { + const pr = JSON.parse(runGh(['api', `repos/${env.GITHUB_REPOSITORY}/pulls/${number}`], env)); + if (pr.merged === true) return { observed: 'merged' }; + if (pr.state === 'open') return { observed: 'open-unmerged' }; + if (pr.state === 'closed') return { observed: 'closed-unmerged' }; + return { observed: 'unknown', reason: 'unexpected PR payload' }; + } catch (error) { + return { observed: 'unknown', reason: describeGhError(error) }; + } +} + +function fetchPullStates(env, prNumbers, incomplete) { + const states = new Map(); + if (prNumbers.length > MAX_PR_CHECKS) { + incomplete.push(`${prNumbers.length - MAX_PR_CHECKS} cited PR(s) not checked: PR check limit of ${MAX_PR_CHECKS} reached`); + } + for (const number of prNumbers.slice(0, MAX_PR_CHECKS)) { + states.set(number, fetchPullState(env, number)); + } + return states; +} + +function main() { + const env = process.env; + if (!env.GITHUB_REPOSITORY || !env.GITHUB_STEP_SUMMARY) { + console.error('audit-closed-references: requires GITHUB_REPOSITORY and GITHUB_STEP_SUMMARY'); + process.exitCode = 1; + return; + } + const issuesResult = fetchAllPages( + `repos/${env.GITHUB_REPOSITORY}/issues?state=closed&sort=updated&direction=desc&per_page=100`, env + ); + const fetchComments = makeCommentFetcher(env); + const { entries, incomplete } = collectEntries( + () => issuesResult, fetchComments, Date.now(), DEFAULT_WINDOW_DAYS + ); + const pullStates = fetchPullStates(env, uniqueClaimPrs(entries), incomplete); + const verdicts = buildVerdicts(entries, pullStates); + const references = collectReferences(entries); + const report = buildSummary(verdicts, references, incomplete).join('\n'); + fs.appendFileSync(env.GITHUB_STEP_SUMMARY, report + '\n'); + console.log(report); + // Advisory by design: findings land in the summary; the run stays green so + // the weekly audit is never the thing that fails. +} + +module.exports = { + DEFAULT_WINDOW_DAYS, + MAX_COMMENT_REQUESTS, + CLAIM_PATTERN, + stripNonEvidence, + isNegated, + extractRefs, + selectRecentClosedIssues, + selectWindowComments, + extractEvidence, + collectEntries, + uniqueClaimPrs, + collectReferences, + buildVerdicts, + buildSummary, + fetchAllPages, + fetchPullState, +}; + +if (require.main === module) main(); diff --git a/tests/ci/audit-closed-references.test.js b/tests/ci/audit-closed-references.test.js new file mode 100644 index 000000000..050c2bed6 --- /dev/null +++ b/tests/ci/audit-closed-references.test.js @@ -0,0 +1,258 @@ +#!/usr/bin/env node +/** + * Contract + unit tests for the closed-issue delivery-claim audit. + * + * Contract side: the maintenance workflow must run the audit on the weekly + * schedule with job-level read-only permissions. Unit side: the evidence + * contract from the #3522 review — asserted claims vs contextual references, + * four observed PR states, and visible collection completeness. A CLI fixture + * (stub gh script) exercises pagination and retrieval failures end to end. + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { execFileSync } = require('child_process'); + +const audit = require('../../scripts/ci/audit-closed-references.js'); + +const WORKFLOW_PATH = path.join(__dirname, '..', '..', '.github', 'workflows', 'maintenance.yml'); +const SCRIPT_PATH = path.join(__dirname, '..', '..', 'scripts', 'ci', 'audit-closed-references.js'); + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + return true; + } catch (error) { + console.log(` ✗ ${name}`); + console.log(` Error: ${error.message}`); + return false; + } +} + +function run() { + console.log('\n=== Testing closed-issue delivery-claim audit ===\n'); + let passed = 0; + let failed = 0; + const check = (name, fn) => (test(name, fn) ? passed++ : failed++); + + const source = fs.readFileSync(WORKFLOW_PATH, 'utf8'); + const script = fs.readFileSync(SCRIPT_PATH, 'utf8'); + + check('maintenance workflow runs the audit on schedule', () => { + assert.match(source, /closed-reference-audit:/); + assert.match(source, /node scripts\/ci\/audit-closed-references\.js/); + assert.match(source, /GH_TOKEN:\s*\$\{\{ github\.token \}\}/); + }); + + check('audit job is scoped to read permissions', () => { + const job = source.split('closed-reference-audit:')[1] || ''; + assert.match(job, /permissions:\s*\n\s*contents:\s*read\s*\n\s*issues:\s*read\s*\n\s*pull-requests:\s*read/); + }); + + check('audit stays read-only: no comment or close calls', () => { + assert.doesNotMatch(script, /-X\s*(POST|PATCH|PUT)/); + assert.doesNotMatch(script, /method:\s*['"](POST|PATCH|PUT|DELETE)/); + }); + + check('asserted delivery claims are extracted with wording variants', () => { + const text = 'Delivered through #3500; also fixed in #3133 and shipped via #3495 and PR #2100.'; + const claims = audit.extractRefs(text, 'closing-comment', 'http://x/1').filter((r) => r.kind === 'claim'); + assert.deepStrictEqual(claims.map((r) => r.pr), [3500, 3133, 3495, 2100]); + }); + + check('historical mentions, quotes, fences, and negations are not claims', () => { + assert.deepStrictEqual(audit.extractRefs('see discussion in #100', 'issue-body', 'u').filter((r) => r.kind === 'claim'), []); + assert.deepStrictEqual(audit.extractRefs('> fixed in #100', 'issue-body', 'u').filter((r) => r.kind === 'claim'), []); + assert.deepStrictEqual(audit.extractRefs('```\nfixed in #100\n```', 'issue-body', 'u').filter((r) => r.kind === 'claim'), []); + assert.deepStrictEqual(audit.extractRefs('was not fixed in #100', 'issue-body', 'u').filter((r) => r.kind === 'claim'), []); + assert.deepStrictEqual(audit.extractRefs('cannot be closed by #100', 'issue-body', 'u').filter((r) => r.kind === 'claim'), []); + }); + + const NOW = Date.parse('2026-10-12T12:00:00Z'); + const days = (n) => new Date(NOW - n * 24 * 60 * 60 * 1000).toISOString(); + + check('#3408 fixture: closing claim #3500 is found, historical #3060 stays context', () => { + const issue = { + number: 3408, state: 'closed', closed_at: days(1), + html_url: 'http://x/3408', title: 'detector gap', + body: 'Same area as #3060, historical context only.', + }; + const comments = [{ + body: 'Delivered through #3500', + html_url: 'http://x/3408#c3500', + created_at: days(1), + }]; + const refs = audit.extractEvidence(issue, comments, NOW); + const claims = refs.filter((r) => r.kind === 'claim'); + const context = refs.filter((r) => r.kind === 'reference'); + assert.deepStrictEqual(claims.map((r) => r.pr), [3500]); + assert.deepStrictEqual(context.map((r) => r.pr), [3060]); + assert.strictEqual(claims[0].sourceUrl, 'http://x/3408#c3500'); + const entries = [{ issue: 3408, issueUrl: issue.html_url, refs }]; + const verdicts = audit.buildVerdicts(entries, new Map([[3500, { observed: 'merged' }]])); + const summary = audit.buildSummary(verdicts, audit.collectReferences(entries), []).join('\n'); + assert.match(summary, /#3500/); + assert.match(summary, /#3408: [^\n]*#3060/); + assert.doesNotMatch(summary, /#3060[^\n]*not merged/); + assert.doesNotMatch(summary, /#3060[^\n]*unmerged/); + }); + + check('four observed states stay distinct', () => { + const entries = [ + { issue: 1, issueUrl: 'u', refs: [{ pr: 11, kind: 'claim', source: 'closing-comment', sourceUrl: 'u', snippet: 'delivered through #11' }] }, + { issue: 2, issueUrl: 'u', refs: [{ pr: 22, kind: 'claim', source: 'closing-comment', sourceUrl: 'u', snippet: 'fixed in #22' }] }, + { issue: 3, issueUrl: 'u', refs: [{ pr: 33, kind: 'claim', source: 'closing-comment', sourceUrl: 'u', snippet: 'shipped via #33' }] }, + { issue: 4, issueUrl: 'u', refs: [{ pr: 44, kind: 'claim', source: 'closing-comment', sourceUrl: 'u', snippet: 'landed in #44' }] }, + ]; + const states = new Map([ + [11, { observed: 'merged' }], + [22, { observed: 'open-unmerged' }], + [33, { observed: 'closed-unmerged' }], + [44, { observed: 'unknown', reason: 'HTTP 404' }], + ]); + const verdicts = audit.buildVerdicts(entries, states); + assert.deepStrictEqual(verdicts.map((v) => v.verdict), ['pass', 'unmerged', 'unmerged', 'unverifiable']); + const summary = audit.buildSummary(verdicts, [], []).join('\n'); + assert.match(summary, /#22: open-unmerged/); + assert.match(summary, /#33: closed-unmerged/); + assert.match(summary, /manual verification required/); + assert.match(summary, /HTTP 404/); + assert.match(summary, /does not establish that the claim failed/); + }); + + check('unknown evidence is advisory, never a failed-claim finding', () => { + const entries = [{ issue: 5, issueUrl: 'u', refs: [{ pr: 55, kind: 'claim', source: 'issue-body', sourceUrl: 'u', snippet: 'fixed in #55' }] }]; + const verdicts = audit.buildVerdicts(entries, new Map([[55, { observed: 'unknown', reason: 'HTTP 404' }]])); + assert.strictEqual(verdicts[0].verdict, 'unverifiable'); + assert.match(audit.buildSummary(verdicts, [], []).join('\n'), /Unverifiable — manual verification required/); + }); + + check('window filter keeps only recent closed issues and drops PRs', () => { + const issues = [ + { number: 1, state: 'closed', closed_at: days(3) }, + { number: 2, state: 'closed', closed_at: days(45) }, + { number: 3, state: 'open', closed_at: days(1) }, + { number: 4, state: 'closed', closed_at: days(2), pull_request: { url: 'x' } }, + { number: 5, state: 'closed', closed_at: null }, + ]; + const kept = audit.selectRecentClosedIssues(issues, NOW, 30).map((i) => i.number); + assert.deepStrictEqual(kept, [1]); + }); + + check('comment window starts at issue closure (close comments count)', () => { + const comments = [ + { body: 'closed as delivered through #3133', html_url: 'c1', created_at: days(10) }, + { body: 'old pre-close chatter #999', html_url: 'c2', created_at: days(40) }, + { body: 'post-close follow-up landed in #3500', html_url: 'c3', created_at: days(1) }, + ]; + const kept = audit.selectWindowComments(comments, days(11), 30, NOW); + assert.deepStrictEqual(kept.map((c) => c.html_url), ['c1', 'c3']); + }); + + check('collection failures surface as incomplete, never as all-clear', () => { + const fetchIssues = () => ({ items: [{ number: 11, state: 'closed', closed_at: days(2), html_url: 'u', title: 'gap', body: 'fixed in #77' }] }); + const fetchComments = () => ({ items: [], error: 'HTTP 403' }); + const { entries, incomplete } = audit.collectEntries(fetchIssues, fetchComments, NOW, 30); + assert.strictEqual(entries.length, 1); // issue body evidence survives the comment failure + assert.match(entries[0].refs.map((r) => r.pr).join(','), /77/); + assert.match(incomplete[0], /#11 comments unavailable: HTTP 403/); + const summary = audit.buildSummary([], audit.collectReferences(entries), incomplete).join('\n'); + assert.match(summary, /Audit incomplete/); + assert.match(summary, /absence of findings below is not an all-clear/); + }); + + check('issues list failure yields an explicitly empty, incomplete audit', () => { + const { entries, incomplete } = audit.collectEntries(() => ({ items: [], error: 'HTTP 502' }), () => ({ items: [], error: null }), NOW, 30); + assert.deepStrictEqual(entries, []); + assert.match(incomplete[0], /issue list unavailable: HTTP 502/); + assert.match(audit.buildSummary([], [], incomplete).join('\n'), /Audit incomplete/); + }); + + // CLI fixture: a stub gh script exercises pagination (issue on page two) + // and a retrieval failure, end to end through the real main(). + check('CLI fixture: page-two issue is audited and failed comments report incomplete', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'audit-cli-')); + const stub = path.join(tmp, 'stub-gh.js'); + fs.writeFileSync(stub, ` + const args = process.argv.slice(2); + const url = args.find((a) => a.includes('/issues?state=closed')) || ''; + if (url) { + const page1 = [{ number: 1, state: 'closed', closed_at: '2026-10-11T00:00:00Z', html_url: 'u1', title: 'a', body: 'no refs here' }]; + const page2 = [{ number: 88, state: 'closed', closed_at: '2026-10-11T00:00:00Z', html_url: 'u88', title: 'page two', body: 'no refs either' }]; + process.stdout.write(JSON.stringify([page1, page2])); + return; + } + if (args.some((a) => a.includes('/issues/88/comments'))) { + process.stderr.write('gh: HTTP 500'); + process.exit(1); + } + if (args.some((a) => a.includes('/issues/1/comments'))) { + process.stdout.write(JSON.stringify([{ body: 'Delivered through #3500', html_url: 'c1', created_at: '2026-10-11T00:00:00Z' }])); + return; + } + if (args.some((a) => a.includes('/pulls/'))) { + process.stdout.write(JSON.stringify({ state: 'closed', merged: true })); + return; + } + process.stderr.write('unexpected args'); + process.exit(2); + `); + const summaryPath = path.join(tmp, 'summary.md'); + const env = { + ...process.env, + GITHUB_REPOSITORY: 'example/repo', + GITHUB_STEP_SUMMARY: summaryPath, + AUDIT_GH_SCRIPT: stub, + }; + execFileSync(process.execPath, [SCRIPT_PATH], { env, encoding: 'utf8' }); + const summary = fs.readFileSync(summaryPath, 'utf8'); + assert.match(summary, /Delivered through #3500/); // page-one issue's claim was audited + assert.match(summary, /#88 comments unavailable: HTTP 500/); + assert.match(summary, /Audit incomplete/); + assert.match(summary, /Observed: /); + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + check('CLI fixture: 101 distinct cited PRs report the unchecked claim and incomplete audit', () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'audit-cap-')); + try { + const stub = path.join(tmp, 'stub-gh.js'); + fs.writeFileSync(stub, ` + const args = process.argv.slice(2); + if (args.some((a) => a.includes('/issues?state=closed'))) { + const body = Array.from({ length: 101 }, (_, i) => 'fixed in #' + (i + 1)).join('; '); + const issue = { number: 99, state: 'closed', closed_at: new Date().toISOString(), html_url: 'u99', title: 'delivery', body }; + process.stdout.write(JSON.stringify([[issue]])); + } else if (args.some((a) => a.includes('/issues/99/comments'))) { + process.stdout.write(JSON.stringify([[]])); + } else if (args.some((a) => a.includes('/pulls/'))) { + process.stdout.write(JSON.stringify({ state: 'closed', merged: true })); + } else { + process.stderr.write('unexpected args'); + process.exit(2); + } + `); + const summaryPath = path.join(tmp, 'summary.md'); + execFileSync(process.execPath, [SCRIPT_PATH], { + env: { ...process.env, GITHUB_REPOSITORY: 'example/repo', GITHUB_STEP_SUMMARY: summaryPath, AUDIT_GH_SCRIPT: stub }, + encoding: 'utf8', + }); + const summary = fs.readFileSync(summaryPath, 'utf8'); + assert.match(summary, /\*\*Audit incomplete\*\*/); + assert.match(summary, /1 cited PR\(s\) not checked: PR check limit of 100 reached/); + assert.match(summary, /101 asserted delivery claim\(s\) found: 0 unmerged finding\(s\), 1 unverifiable/); + assert.match(summary, /#101: unavailable \(not fetched\)/); + assert.match(summary, /#100: merged/); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + + console.log(`\nPassed: ${passed} Failed: ${failed}`); + if (failed > 0) process.exit(1); +} + +run();