mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-26 18:35:05 +00:00
ci: audit closed-issue delivery claims — cited PRs must actually be merged (#3522)
* ci: audit closed-issue delivery claims — cited PRs must actually be merged Follow-up to #3223: an issue closed as 'fixed in #N' while #N was closed without merging (or does not exist) leaves a completion claim that does not hold. The weekly maintenance workflow now runs a read-only audit over issues closed in the last 30 days, extracts delivery-claim PR references from issue bodies and close-window comments, verifies each cited PR's merge state, and reports findings to the step summary. Design constraints: - Read-only: no comments, no closes, no labels; findings are advisory and the run always stays green so the weekly audit never blocks maintenance. - Unverifiable counts as unmerged: a 404 PR or an API failure fails the verdict, because an unverifiable claim is exactly the failure mode. - Pure decision functions are exported for tests; a first live run against this repository surfaced one real finding (#3408 cites #3060, which does not resolve), included here as the motivating sample. 8 unit/contract tests: extraction, windowing (close comments included), the #3223 unmerged-citation failure mode, the all-clear summary, workflow contract, and a no-write guard. * ci: rework audit evidence contract — claims vs context, four PR states, visible completeness Revision per the #3522 review (reviewer ran the CLI through 19 controlled cases plus a bounded live check of #3408): - Bind findings to asserted delivery claims. Closing wording ('Delivered through #N'), claim verbs (fixed/shipped/landed/resolved/...), and bare 'PR #N' inside closing comments are claims; historical mentions, quoted lines, fenced examples, negated claims, and tracking-only references are reported as contextual references and never produce findings. On the live #3408 the audit now recognizes the closing claim #3500 (merged) instead of flagging the historical #3060 text. - Separate observed states: merged / open-unmerged / closed-unmerged / unknown (404, rate limit, transient). Unknown claims are advisory manual- verification items, not failed claims; each row carries the claim snippet, its source link, and the observation time. Checked claims are listed even when passing, so the report is traceable rather than an opaque all-clear. - Paginate issue and comment collection (--paginate --slurp), bound comment requests (200) and PR checks (100), and surface skipped pages, rate limits, and retrieval failures as an explicit incomplete audit instead of an empty all-clear. A live-run maxBuffer overflow on the paginated issue list was caught and fixed (64MB); the audit reported it as incomplete before the fix. - Scope the job to read access: job-level contents/issues/pull-requests read. 13 contract/unit tests including a CLI fixture (stub gh) that places the relevant issue on page two and forces a comment-fetch failure. * ci: report incomplete audit at PR check cap --------- Co-authored-by: chenhz01 <chenhz01@users.noreply.github.com> Co-authored-by: haelyra <49814733+haelyra@users.noreply.github.com>
This commit is contained in:
3 files changed
+613
No files matched your search
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Audit recently closed issues whose asserted delivery claims cite a PR that
|
||||
* is not actually merged.
|
||||
*
|
||||
* Motivated by #3223: an issue closed as "fixed in #N" while #N was closed
|
||||
* without merging leaves a completion claim that does not hold up. This audit
|
||||
* runs weekly from .github/workflows/maintenance.yml, scans issue text and
|
||||
* close-window comments for *asserted* delivery claims, verifies each cited
|
||||
* PR's merge state, and reports findings to the workflow step summary.
|
||||
*
|
||||
* Evidence contract (per the #3522 review):
|
||||
* - Only asserted delivery claims ("delivered through #N", "fixed in #N")
|
||||
* produce findings. Historical mentions, quoted text, fenced examples,
|
||||
* negated claims, and tracking-only references are reported separately as
|
||||
* context — a merged PR alone also does not certify issue completion.
|
||||
* - Observed states are reported distinctly: merged, open-unmerged,
|
||||
* closed-unmerged, and unknown (404, rate limit, transient failures). An
|
||||
* unavailable PR warrants manual verification; it does not establish that
|
||||
* a claim failed.
|
||||
* - Collection is paginated and its completeness is visible: skipped pages,
|
||||
* rate limits, and retrieval failures turn the report into an explicitly
|
||||
* incomplete audit rather than an all-clear.
|
||||
*
|
||||
* Read-only by design: findings are advisory, the script never comments on or
|
||||
* closes anything, and always exits 0 so the scheduled run never breaks.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const { execFileSync } = require('child_process');
|
||||
|
||||
const DEFAULT_WINDOW_DAYS = 30;
|
||||
const MAX_COMMENT_REQUESTS = 200;
|
||||
const MAX_PR_CHECKS = 100;
|
||||
|
||||
const CLAIM_PATTERN =
|
||||
/\b(?:delivered|delivers|fixed|fixes|shipped|ships|landed|lands|resolved|resolves|implemented|implements|completed|completes|closed|closes|merged|merges)\s+(?:through|in|via|by|with|as|into)\s+(?:PR\s+)?#(\d+)\b/gi;
|
||||
const PR_REF_PATTERN = /\bPR\s+#(\d+)\b/gi;
|
||||
const ANY_REF_PATTERN = /(?:^|[^\w&])#(\d+)\b/g;
|
||||
const NEGATION_PATTERN =
|
||||
/\b(?:not|never|cannot|can't|won't|doesn't|does\s+not|no\s+longer|isn't|wasn't)\s+(?:\w+\s+){0,2}$/i;
|
||||
|
||||
function runGh(args, env) {
|
||||
const options = { encoding: 'utf8', env, timeout: 60000, maxBuffer: 64 * 1024 * 1024 };
|
||||
if (env.AUDIT_GH_SCRIPT) {
|
||||
// Testability hook: run a fixture script instead of the real gh CLI.
|
||||
return execFileSync(process.execPath, [env.AUDIT_GH_SCRIPT, ...args], options);
|
||||
}
|
||||
return execFileSync('gh', args, options);
|
||||
}
|
||||
|
||||
function describeGhError(error) {
|
||||
const stderr = (error && error.stderr) || '';
|
||||
const http = stderr.match(/HTTP (\d{3})/);
|
||||
if (http) return `HTTP ${http[1]}`;
|
||||
return String((error && error.message) || error).split('\n')[0];
|
||||
}
|
||||
|
||||
// Pure: fenced code blocks, inline code, and blockquoted lines are examples or
|
||||
// quotation, not assertions made by the author — strip them before matching.
|
||||
function stripNonEvidence(text) {
|
||||
if (!text) return '';
|
||||
return text
|
||||
.replace(/```[\s\S]*?```/g, ' ')
|
||||
.replace(/`[^`\n]*`/g, ' ')
|
||||
.split('\n')
|
||||
.filter((line) => !/^\s*>/.test(line))
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
// Pure: a claim pattern preceded by negation words is not an assertion.
|
||||
function isNegated(clean, matchIndex) {
|
||||
const before = clean.slice(Math.max(0, matchIndex - 60), matchIndex);
|
||||
return NEGATION_PATTERN.test(before);
|
||||
}
|
||||
|
||||
// Pure: classify every #N in one text as an asserted delivery claim or a
|
||||
// contextual reference. Deduplicates per (pr, kind). In a closing comment a
|
||||
// bare "PR #N" reference is the closing evidence; in issue text it stays
|
||||
// context.
|
||||
function extractRefs(text, source, sourceUrl) {
|
||||
const clean = stripNonEvidence(text);
|
||||
if (!clean) return [];
|
||||
const refs = [];
|
||||
const claimMatches = [...clean.matchAll(CLAIM_PATTERN)];
|
||||
if (source === 'closing-comment') claimMatches.push(...clean.matchAll(PR_REF_PATTERN));
|
||||
for (const match of claimMatches) {
|
||||
if (isNegated(clean, match.index)) continue;
|
||||
refs.push({ pr: Number(match[1]), kind: 'claim', source, sourceUrl, snippet: match[0] });
|
||||
}
|
||||
const ranges = claimMatches.map((m) => [m.index, m.index + m[0].length]);
|
||||
for (const match of clean.matchAll(ANY_REF_PATTERN)) {
|
||||
const insideClaim = ranges.some(([start, end]) => match.index >= start && match.index < end);
|
||||
if (insideClaim || isNegated(clean, match.index)) continue;
|
||||
refs.push({ pr: Number(match[1]), kind: 'reference', source, sourceUrl, snippet: match[0].trim() });
|
||||
}
|
||||
const seen = new Set();
|
||||
return refs.filter((ref) => (seen.has(ref.pr + ':' + ref.kind) ? false : (seen.add(ref.pr + ':' + ref.kind), true)));
|
||||
}
|
||||
|
||||
// Pure: issues closed within the look-back window; PRs are not issues.
|
||||
function selectRecentClosedIssues(issues, nowMs, windowDays) {
|
||||
const cutoff = nowMs - windowDays * 24 * 60 * 60 * 1000;
|
||||
return issues.filter(
|
||||
(issue) =>
|
||||
issue.state === 'closed' &&
|
||||
!issue.pull_request &&
|
||||
typeof issue.closed_at === 'string' &&
|
||||
Date.parse(issue.closed_at) >= cutoff
|
||||
);
|
||||
}
|
||||
|
||||
// Pure: comment objects (body + html_url) inside the window, starting at the
|
||||
// issue's closure so the closing comment always counts as evidence.
|
||||
function selectWindowComments(comments, issueClosedAt, windowDays, nowMs) {
|
||||
const cutoff = nowMs - windowDays * 24 * 60 * 60 * 1000;
|
||||
return comments
|
||||
.filter((comment) => typeof comment.body === 'string')
|
||||
.filter((comment) => Date.parse(comment.created_at) >= Math.max(cutoff, Date.parse(issueClosedAt) - 60_000));
|
||||
}
|
||||
|
||||
// Pure: one entry per issue with its claims and contextual references.
|
||||
function extractEvidence(issue, comments, nowMs) {
|
||||
const refs = [
|
||||
...extractRefs([issue.title, issue.body].filter(Boolean).join('\n'), 'issue-body', issue.html_url),
|
||||
];
|
||||
for (const comment of selectWindowComments(comments, issue.closed_at, DEFAULT_WINDOW_DAYS, nowMs)) {
|
||||
refs.push(...extractRefs(comment.body, 'closing-comment', comment.html_url));
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
// Collect issues + evidence. Fetch failures are surfaced as incomplete notes,
|
||||
// never silently swallowed into an empty list.
|
||||
function collectEntries(fetchIssues, fetchComments, nowMs, windowDays) {
|
||||
const { items: issues, error } = fetchIssues();
|
||||
if (error) return { entries: [], incomplete: [`issue list unavailable: ${error}`] };
|
||||
const incomplete = [];
|
||||
const entries = [];
|
||||
for (const issue of selectRecentClosedIssues(issues, nowMs, windowDays)) {
|
||||
const { items: comments, error: commentError } = fetchComments(issue.number);
|
||||
if (commentError) incomplete.push(`#${issue.number} comments unavailable: ${commentError}`);
|
||||
const refs = extractEvidence(issue, comments, nowMs);
|
||||
if (refs.length) entries.push({ issue: issue.number, issueUrl: issue.html_url, refs });
|
||||
}
|
||||
return { entries, incomplete };
|
||||
}
|
||||
|
||||
function uniqueClaimPrs(entries) {
|
||||
return [...new Set(entries.flatMap((entry) => entry.refs.filter((r) => r.kind === 'claim').map((r) => r.pr)))];
|
||||
}
|
||||
|
||||
function collectReferences(entries) {
|
||||
const seen = new Set();
|
||||
const refs = [];
|
||||
for (const entry of entries) {
|
||||
for (const ref of entry.refs.filter((r) => r.kind === 'reference')) {
|
||||
const key = entry.issue + ':' + ref.pr;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
refs.push({ issue: entry.issue, pr: ref.pr });
|
||||
}
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
// Pure: one verdict row per asserted claim. Merged passes; open/closed
|
||||
// unmerged are findings; unknown is unverifiable and needs manual verification.
|
||||
function buildVerdicts(entries, pullStates) {
|
||||
const rows = [];
|
||||
for (const entry of entries) {
|
||||
for (const ref of entry.refs.filter((r) => r.kind === 'claim')) {
|
||||
const state = pullStates.get(ref.pr) || { observed: 'unknown', reason: 'not fetched' };
|
||||
rows.push({
|
||||
issue: entry.issue, pr: ref.pr, snippet: ref.snippet, source: ref.source,
|
||||
sourceUrl: ref.sourceUrl, observed: state.observed, reason: state.reason || null,
|
||||
verdict: state.observed === 'merged' ? 'pass' : state.observed === 'unknown' ? 'unverifiable' : 'unmerged',
|
||||
});
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function summaryIncomplete(lines, incomplete) {
|
||||
if (!incomplete.length) return;
|
||||
lines.push('', '**Audit incomplete** — some evidence could not be collected; absence of findings below is not an all-clear:');
|
||||
for (const note of incomplete) lines.push(`- ${note}`);
|
||||
}
|
||||
|
||||
function summaryFindings(lines, unmerged) {
|
||||
if (!unmerged.length) return;
|
||||
lines.push('', '### Findings — asserted delivery claims whose cited PR is not merged');
|
||||
for (const v of unmerged) {
|
||||
lines.push(`- #${v.issue} claims "${v.snippet}" (source: ${v.source}, ${v.sourceUrl}) → #${v.pr}: ${v.observed}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function summaryUnverifiable(lines, unverifiable) {
|
||||
if (!unverifiable.length) return;
|
||||
lines.push('', '### Unverifiable — manual verification required');
|
||||
for (const v of unverifiable) {
|
||||
lines.push(`- #${v.issue} claims "${v.snippet}" (${v.sourceUrl}) → #${v.pr}: unavailable (${v.reason}). An unavailable PR does not establish that the claim failed.`);
|
||||
}
|
||||
}
|
||||
|
||||
// Traceability: every checked claim is listed with its evidence link, passes
|
||||
// included — an all-clear line alone cannot be audited.
|
||||
function summaryPasses(lines, passes) {
|
||||
if (!passes.length) return;
|
||||
lines.push('', '### Checked claims — cited PR is merged');
|
||||
for (const v of passes) {
|
||||
lines.push(`- #${v.issue} claims "${v.snippet}" (source: ${v.source}, ${v.sourceUrl}) → #${v.pr}: merged.`);
|
||||
}
|
||||
}
|
||||
|
||||
// Pure: markdown lines for the step summary.
|
||||
function buildSummary(verdicts, references, incomplete) {
|
||||
const lines = ['## Closed-issue delivery-claim audit', '', `Observed: ${new Date().toISOString()}`];
|
||||
summaryIncomplete(lines, incomplete);
|
||||
const unmerged = verdicts.filter((v) => v.verdict === 'unmerged');
|
||||
const unverifiable = verdicts.filter((v) => v.verdict === 'unverifiable');
|
||||
lines.push('', `${verdicts.length} asserted delivery claim(s) found: ${unmerged.length} unmerged finding(s), ${unverifiable.length} unverifiable.`);
|
||||
summaryFindings(lines, unmerged);
|
||||
summaryUnverifiable(lines, unverifiable);
|
||||
summaryPasses(lines, verdicts.filter((v) => v.verdict === 'pass'));
|
||||
if (references.length) {
|
||||
lines.push('', '### Contextual references (not delivery claims)');
|
||||
const byIssue = new Map();
|
||||
for (const r of references) {
|
||||
if (!byIssue.has(r.issue)) byIssue.set(r.issue, []);
|
||||
byIssue.get(r.issue).push('#' + r.pr);
|
||||
}
|
||||
for (const [issue, prs] of byIssue) {
|
||||
lines.push(`- #${issue}: ${[...new Set(prs)].join(', ')}`);
|
||||
}
|
||||
}
|
||||
if (!verdicts.length && !references.length && !incomplete.length) {
|
||||
lines.push('No asserted delivery claims found in recently closed issues.');
|
||||
}
|
||||
if (verdicts.length && !unmerged.length) {
|
||||
lines.push('', 'No unmerged findings. A merged cited PR supports the claim; it does not by itself certify issue completion, current code after a revert, or package publication.');
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
// Paginated gh api fetch: --paginate --slurp yields one array per page.
|
||||
function fetchAllPages(args, env) {
|
||||
try {
|
||||
const raw = JSON.parse(runGh(['api', '--paginate', '--slurp', args], env));
|
||||
return { items: Array.isArray(raw) ? raw.flat() : [], error: null };
|
||||
} catch (error) {
|
||||
return { items: [], error: describeGhError(error) };
|
||||
}
|
||||
}
|
||||
|
||||
// Comment fetching is bounded by a request budget; exhaustion is reported as
|
||||
// incompleteness instead of passing as an empty list.
|
||||
function makeCommentFetcher(env) {
|
||||
let budget = MAX_COMMENT_REQUESTS;
|
||||
return (number) => {
|
||||
if (budget-- <= 0) return { items: [], error: 'comment request budget exhausted' };
|
||||
return fetchAllPages(`repos/${env.GITHUB_REPOSITORY}/issues/${number}/comments?per_page=100`, env);
|
||||
};
|
||||
}
|
||||
|
||||
// Four observed states, kept distinct: merged / open-unmerged /
|
||||
// closed-unmerged / unknown (404, rate limit, transient failure).
|
||||
function fetchPullState(env, number) {
|
||||
try {
|
||||
const pr = JSON.parse(runGh(['api', `repos/${env.GITHUB_REPOSITORY}/pulls/${number}`], env));
|
||||
if (pr.merged === true) return { observed: 'merged' };
|
||||
if (pr.state === 'open') return { observed: 'open-unmerged' };
|
||||
if (pr.state === 'closed') return { observed: 'closed-unmerged' };
|
||||
return { observed: 'unknown', reason: 'unexpected PR payload' };
|
||||
} catch (error) {
|
||||
return { observed: 'unknown', reason: describeGhError(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function fetchPullStates(env, prNumbers, incomplete) {
|
||||
const states = new Map();
|
||||
if (prNumbers.length > MAX_PR_CHECKS) {
|
||||
incomplete.push(`${prNumbers.length - MAX_PR_CHECKS} cited PR(s) not checked: PR check limit of ${MAX_PR_CHECKS} reached`);
|
||||
}
|
||||
for (const number of prNumbers.slice(0, MAX_PR_CHECKS)) {
|
||||
states.set(number, fetchPullState(env, number));
|
||||
}
|
||||
return states;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const env = process.env;
|
||||
if (!env.GITHUB_REPOSITORY || !env.GITHUB_STEP_SUMMARY) {
|
||||
console.error('audit-closed-references: requires GITHUB_REPOSITORY and GITHUB_STEP_SUMMARY');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const issuesResult = fetchAllPages(
|
||||
`repos/${env.GITHUB_REPOSITORY}/issues?state=closed&sort=updated&direction=desc&per_page=100`, env
|
||||
);
|
||||
const fetchComments = makeCommentFetcher(env);
|
||||
const { entries, incomplete } = collectEntries(
|
||||
() => issuesResult, fetchComments, Date.now(), DEFAULT_WINDOW_DAYS
|
||||
);
|
||||
const pullStates = fetchPullStates(env, uniqueClaimPrs(entries), incomplete);
|
||||
const verdicts = buildVerdicts(entries, pullStates);
|
||||
const references = collectReferences(entries);
|
||||
const report = buildSummary(verdicts, references, incomplete).join('\n');
|
||||
fs.appendFileSync(env.GITHUB_STEP_SUMMARY, report + '\n');
|
||||
console.log(report);
|
||||
// Advisory by design: findings land in the summary; the run stays green so
|
||||
// the weekly audit is never the thing that fails.
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_WINDOW_DAYS,
|
||||
MAX_COMMENT_REQUESTS,
|
||||
CLAIM_PATTERN,
|
||||
stripNonEvidence,
|
||||
isNegated,
|
||||
extractRefs,
|
||||
selectRecentClosedIssues,
|
||||
selectWindowComments,
|
||||
extractEvidence,
|
||||
collectEntries,
|
||||
uniqueClaimPrs,
|
||||
collectReferences,
|
||||
buildVerdicts,
|
||||
buildSummary,
|
||||
fetchAllPages,
|
||||
fetchPullState,
|
||||
};
|
||||
|
||||
if (require.main === module) main();
|
||||
Reference in new issue
Block a user