diff --git a/scripts/hooks/gateguard-fact-force.js b/scripts/hooks/gateguard-fact-force.js index 977ec90d1..52ce78b3a 100644 --- a/scripts/hooks/gateguard-fact-force.js +++ b/scripts/hooks/gateguard-fact-force.js @@ -2117,6 +2117,7 @@ function creditNote(match, filePath) { const MISS_DETAIL_MAX_CHARS = 60; const MISS_REASON_TEXT = Object.freeze({ 'same-batch': 'it was sent in the same batch as this call, so its result was not seen yet', + lookup: 'it only looked this file up by its own name, so it found nothing that uses it', excluded: 'its filters exclude this file', 'out-of-scope': 'its search path does not contain this file', 'stdin-only': 'it searched piped input, not the tree', diff --git a/scripts/lib/gateguard-code-lexer.js b/scripts/lib/gateguard-code-lexer.js index 29676d51f..f89c9e101 100644 --- a/scripts/lib/gateguard-code-lexer.js +++ b/scripts/lib/gateguard-code-lexer.js @@ -301,7 +301,11 @@ function codeSignature(text, spec) { return signature; } +// JavaScript ends a line at U+2028 and U+2029, and C# also at U+0085; the lexer splits only at \n and \r. +const UNICODE_LINE_TERMINATORS = /[\u0085\u2028\u2029]/; + function isTrivialEdit(oldString, newString, spec) { + if (UNICODE_LINE_TERMINATORS.test(oldString) || UNICODE_LINE_TERMINATORS.test(newString)) return false; if (spec.lineComment === '//' && (oldString.includes('??/') || newString.includes('??/'))) return false; const before = codeSignature(oldString, spec); if (before === null) return false; diff --git a/scripts/lib/gateguard-search-evidence.js b/scripts/lib/gateguard-search-evidence.js index 92d8806e2..e8d4cf086 100644 --- a/scripts/lib/gateguard-search-evidence.js +++ b/scripts/lib/gateguard-search-evidence.js @@ -12,7 +12,9 @@ const { grepToolFilters, shellSearchFilters, filtersAdmitTarget, - includesAdmitTarget + includesAdmitTarget, + pathPatternFor, + confinedToTargetName } = require('./gateguard-search-filters'); // --- Search commands and flags --- @@ -141,6 +143,11 @@ function globLiteralPrefix(pattern) { return joined || (literal.length > 0 && pattern.startsWith('/') ? '/' : ''); } +function grepDetail(input) { + const flags = [['--glob', input.glob], ['--type', input.type]].filter(([, value]) => typeof value === 'string' && value); + return [input.pattern].concat(flags.map(([flag, value]) => `${flag} ${value}`)).join(' '); +} + function stringsOf(...values) { return values.filter(value => typeof value === 'string' && value); } @@ -165,16 +172,22 @@ function operandScopePath(arg) { return literal.join('/') || '/'; } -function shellSearchScopes(parsed, base) { - if (parsed.operands.length === 0) return base === null ? [] : [base]; +// see docs/gateguard/design-notes.md#path-patterns +function shellSearchScopes(kind, parsed, base) { + if (parsed.operands.length === 0) return { scopes: base === null ? [] : [base], patterns: [] }; const scopes = []; + const patterns = []; + // A shell glob is expanded before the search runs; a git pathspec's `*` also matches `/`. + const options = { crossSegments: kind.startsWith('git '), ancestors: true }; for (const operand of parsed.operands) { if (!operand || operand === '-' || /^[$~]/.test(operand)) continue; const scoped = operandScopePath(operand); - if (isAbsoluteShellPath(scoped)) scopes.push([scoped]); - else if (base !== null) scopes.push(base.concat(scoped)); + const root = isAbsoluteShellPath(scoped) ? [] : base; + if (root === null) continue; + scopes.push(root.concat(scoped)); + patterns.push(operand.startsWith(':') ? null : pathPatternFor(root, operand, options)); } - return scopes; + return { scopes, patterns }; } function isRecursiveFlag(kind, arg) { @@ -272,6 +285,12 @@ function evidenceInScope(item, ctx) { return true; } +// A search credits by stem when it names the stem, saw the target, and was not a lookup of the target itself. +function stemCredits(item, ctx, stem) { + if (!includesAdmitTarget(item, ctx) || confinedToTargetName(item, ctx)) return false; + return item.texts.some(text => stem.test(text.toLowerCase())); +} + function evidenceNamesDir(item, ctx) { return item.dirs.some( parts => ctx.resolveDir(...parts) === ctx.targetDir && (!item.dirsMustExist || isDirectoryNow(ctx.resolveNative(...parts))) @@ -281,7 +300,9 @@ function evidenceNamesDir(item, ctx) { // --- Prior-search credit --- const MIN_GENERIC_MISS_STEM = 3; -const MISS_RANK = Object.freeze({ 'same-batch': 0, excluded: 1, 'out-of-scope': 2, 'stdin-only': 3, 'not-a-search': 4, 'generic-stem': 5 }); +const MISS_RANK = Object.freeze({ + 'same-batch': 0, lookup: 1, excluded: 2, 'out-of-scope': 3, 'stdin-only': 4, 'not-a-search': 5, 'generic-stem': 6 +}); /** Build the prior-search matcher around the hook's shell segmenter. */ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGMENT_SEPARATORS }) { @@ -297,13 +318,15 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME return [{ texts: [input.pattern], dirs: [], scope, scopeExact: true, operands: [lookup], detail: input.pattern }]; } const dirs = prefix || explicit ? [scope] : []; - return [{ texts: [input.pattern], dirs, scope, detail: input.pattern }]; + // see docs/gateguard/design-notes.md#path-patterns + const pathPattern = pathPatternFor([explicit || '.'], input.pattern); + return [{ texts: [input.pattern], dirs, scope, detail: input.pattern, pathPattern }]; } if (search.name === 'Grep') { if (typeof input.pattern !== 'string' || !input.pattern) return []; const explicit = typeof input.path === 'string' && input.path ? input.path : ''; - const detail = typeof input.glob === 'string' && input.glob ? `${input.pattern} --glob ${input.glob}` : input.pattern; - const filters = grepToolFilters(input.glob); + const detail = grepDetail(input); + const filters = grepToolFilters(input.glob, input.type); return [{ texts: stringsOf(input.pattern, ...filters.positives), dirs: explicit ? [[explicit]] : [], @@ -316,7 +339,7 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME if (search.name === 'LS') { return stringsOf(input.path).map(dir => ({ texts: [], dirs: [[dir]], scope: [dir], detail: dir })); } - return shellSearchEvidence(input.command, shellDirsTrusted, misses); + return shellSearchEvidence(input.command, search.name, shellDirsTrusted, misses); } function segmentPipeFlags(input) { @@ -351,9 +374,11 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME return flags; } - function shellSearchEvidence(command, dirsTrusted, misses) { - if (typeof command !== 'string' || !command || command.length > MAX_SEARCH_COMMAND_CHARS) return []; - if (AMBIGUOUS_SHELL_PATTERN.test(command)) return []; + function shellSearchEvidence(rawCommand, toolName, dirsTrusted, misses) { + if (typeof rawCommand !== 'string' || !rawCommand || rawCommand.length > MAX_SEARCH_COMMAND_CHARS) return []; + if (AMBIGUOUS_SHELL_PATTERN.test(rawCommand)) return []; + // PowerShell escapes with a backtick, so a backslash there is a path separator, not an escape. + const command = toolName === 'PowerShell' ? rawCommand.replace(/\\/g, '\\\\') : rawCommand; const pipeFlags = segmentPipeFlags(command); const evidence = []; let trusted = dirsTrusted; @@ -381,7 +406,7 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME miss('stdin-only'); return; } - const scopes = shellSearchScopes(parsed, base); + const { scopes, patterns } = shellSearchScopes(kind, parsed, base); if (scopes.length === 0) return; const filters = shellSearchFilters(kind, args); // see docs/gateguard/design-notes.md#search-filters @@ -392,6 +417,7 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME dirs, dirsMustExist: true, scopes, + scopePatterns: patterns, operands: parsed.operands.map(op => [op]), detail: tokens.join(' '), exclusions: filters.exclusions, @@ -455,7 +481,7 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME if (shellDirsTrusted === null) shellDirsTrusted = shellDirsTrustedFor(scan); for (const item of cachedEvidence(search, shellDirsTrusted).items) { if (!evidenceInScope(item, ctx) || !filtersAdmitTarget(item, ctx, stem)) continue; - const byStem = stem !== null && includesAdmitTarget(item, ctx) && item.texts.some(text => stem.test(text.toLowerCase())); + const byStem = stem !== null && stemCredits(item, ctx, stem); const byDir = Boolean(allowDirMatch) && evidenceNamesDir(item, ctx); if (byStem || byDir) { return { name: search.name, detail: item.detail, callsAgo: search.callsAgo }; @@ -484,7 +510,9 @@ function createSearchEvidence({ quoteAwareSegments, commandBasename, SHELL_SEGME const inScope = evidenceInScope(item, ctx); const admitted = filtersAdmitTarget(item, ctx, stem); const included = includesAdmitTarget(item, ctx); - if (inScope && admitted && (byDir || included)) return sameBatch ? 'same-batch' : null; + const lookup = included && confinedToTargetName(item, ctx); + if (inScope && admitted && (byDir || (included && !lookup))) return sameBatch ? 'same-batch' : null; + if (inScope && admitted && lookup) return 'lookup'; if (!inScope) return 'out-of-scope'; return 'excluded'; } diff --git a/scripts/lib/gateguard-search-filters.js b/scripts/lib/gateguard-search-filters.js index 2077aff7f..4d0cf4881 100644 --- a/scripts/lib/gateguard-search-filters.js +++ b/scripts/lib/gateguard-search-filters.js @@ -6,8 +6,8 @@ const POWERSHELL_SEARCHES = new Set(['get-childitem', 'gci', 'select-string', 's const VALUE_FLAGS = new Set([ '-e', '-f', '-g', '-t', '-T', '-m', '-A', '-B', '-C', - '--regexp', '--file', '--type', '--glob', '--iglob', '--include', '--exclude', '--exclude-dir', '--exclude-from', - '--ignore-file', '--max-count', '--replace', + '--regexp', '--file', '--type', '--type-not', '--type-add', '--type-clear', '--glob', '--iglob', '--include', + '--exclude', '--exclude-dir', '--exclude-from', '--ignore-file', '--max-count', '--replace', '-name', '-iname', '-path', '-ipath', '-type', '-maxdepth', '-mindepth', '-newer', '-regex', '-size', '-user', '-group', '-perm' ]); @@ -15,7 +15,65 @@ const LS_VALUE_FLAGS = new Set(['-I', '--ignore', '--hide']); const TREE_VALUE_FLAGS = new Set(['-L', '-P', '-I', '-o']); -const FD_VALUE_FLAGS = new Set([...VALUE_FLAGS, '-E']); +const FD_VALUE_FLAGS = new Set([...VALUE_FLAGS, '-E', '--extension']); + +// see docs/gateguard/design-notes.md#type-filters +// ripgrep 14.1 built-in types (`rg --type-list`) for common languages; any other name is unreadable. +const RG_TYPE_GLOBS = Object.freeze({ + c: ['*.[chH]', '*.[chH].in', '*.cats'], + cmake: ['*.cmake', 'CMakeLists.txt'], + config: ['*.cfg', '*.conf', '*.config', '*.ini'], + cpp: ['*.[ChH]', '*.[ChH].in', '*.[ch]pp', '*.[ch]pp.in', '*.[ch]xx', '*.[ch]xx.in', '*.cc', '*.cc.in', '*.hh', '*.hh.in', '*.inl'], + cs: ['*.cs'], + csharp: ['*.cs'], + css: ['*.css', '*.scss'], + csv: ['*.csv'], + dart: ['*.dart'], + docker: ['*Dockerfile*'], + elixir: ['*.eex', '*.ex', '*.exs', '*.heex', '*.leex', '*.livemd'], + erlang: ['*.erl', '*.hrl'], + go: ['*.go'], + graphql: ['*.graphql', '*.graphqls'], + h: ['*.h', '*.hh', '*.hpp'], + html: ['*.ejs', '*.htm', '*.html'], + java: ['*.java', '*.jsp', '*.jspx', '*.properties'], + js: ['*.cjs', '*.js', '*.jsx', '*.mjs', '*.vue'], + json: ['*.json', '*.sarif', 'composer.lock'], + jsonl: ['*.jsonl'], + kotlin: ['*.kt', '*.kts'], + lua: ['*.lua'], + make: ['*.mak', '*.mk', '[Gg][Nn][Uu]makefile', '[Gg][Nn][Uu]makefile.am', '[Gg][Nn][Uu]makefile.in', '[Mm]akefile', '[Mm]akefile.am', '[Mm]akefile.in'], + markdown: ['*.markdown', '*.md', '*.mdown', '*.mdwn', '*.mdx', '*.mkd', '*.mkdn'], + md: ['*.markdown', '*.md', '*.mdown', '*.mdwn', '*.mdx', '*.mkd', '*.mkdn'], + php: ['*.php', '*.php3', '*.php4', '*.php5', '*.php7', '*.php8', '*.pht', '*.phtml'], + protobuf: ['*.proto'], + ps: ['*.cdxml', '*.ps1', '*.ps1xml', '*.psd1', '*.psm1'], + py: ['*.py', '*.pyi'], + python: ['*.py', '*.pyi'], + ruby: ['*.gemspec', '*.rb', '*.rbw', '.irbrc', 'Gemfile', 'Rakefile', 'config.ru'], + rust: ['*.rs'], + scala: ['*.sbt', '*.scala'], + sh: [ + '*.bash', '*.bashrc', '*.csh', '*.cshrc', '*.ksh', '*.kshrc', '*.sh', '*.tcsh', '*.zsh', '.bash_login', '.bash_logout', + '.bash_profile', '.bashrc', '.cshrc', '.kshrc', '.login', '.logout', '.profile', '.tcshrc', '.zlogin', '.zlogout', + '.zprofile', '.zshenv', '.zshrc', 'bash_login', 'bash_logout', 'bash_profile', 'bashrc', 'profile', 'zlogin', 'zlogout', + 'zprofile', 'zshenv', 'zshrc' + ], + sql: ['*.psql', '*.sql'], + svelte: ['*.svelte'], + swift: ['*.swift'], + toml: ['*.toml', 'Cargo.lock'], + ts: ['*.cts', '*.mts', '*.ts', '*.tsx'], + txt: ['*.txt'], + typescript: ['*.cts', '*.mts', '*.ts', '*.tsx'], + vue: ['*.vue'], + xml: ['*.dtd', '*.rng', '*.sch', '*.xhtml', '*.xjb', '*.xml', '*.xml.dist', '*.xsd', '*.xsl', '*.xslt'], + yaml: ['*.yaml', '*.yml'] +}); + +function rgTypeGlobs(name) { + return Object.prototype.hasOwnProperty.call(RG_TYPE_GLOBS, name) ? RG_TYPE_GLOBS[name] : null; +} const POWERSHELL_FILTER_PARAMS = new Set(['filter', 'include', 'exclude']); @@ -246,6 +304,8 @@ function includeAdmitsTarget(include, item, ctx) { // see docs/gateguard/design-notes.md#include-filters function includesAdmitTarget(item, ctx) { + if (item.pathPattern && !pathPatternAdmitsTarget(item.pathPattern, ctx)) return false; + if (!operandPatternsAdmitTarget(item, ctx)) return false; const includes = Array.isArray(item.includes) ? item.includes : []; if (includes.length === 0) return true; if (includes.length > MAX_FILTERS_PER_SEARCH) return false; @@ -256,8 +316,70 @@ function isBasenameGlob(glob) { return !/[\\/]/.test(String(glob).replace(/^(?:\*\*[\\/])+/, '')); } -function grepToolFilters(glob) { +// --- Path patterns --- +// see docs/gateguard/design-notes.md#path-patterns + +const PATH_GLOB_CHARS = /[*?[{]/; + +/** + * The pattern a path argument applies, split at its literal directory prefix, + * or null when it has no glob characters. `root` is the prefix as path parts. + */ +function pathPatternFor(root, pattern, { crossSegments = false, ancestors = false } = {}) { + const segments = String(pattern).split(/[\\/]/); + const firstGlob = segments.findIndex(segment => PATH_GLOB_CHARS.test(segment)); + if (firstGlob === -1) return null; + const literal = segments.slice(0, firstGlob).join('/') || (firstGlob > 0 ? '/' : ''); + return { root: literal ? root.concat(literal) : root, glob: segments.slice(firstGlob).join('/'), crossSegments, ancestors }; +} + +function pathPatternAdmitsTarget(pattern, ctx) { + const root = ctx.resolveDir(...pattern.root).replace(/\/+$/, ''); + if (!isInsideDir(ctx.targetKey, root)) return false; + const rel = ctx.targetKey.slice(root.length + 1).toLowerCase(); + let glob = normalizeFilterGlob(pattern.glob); + if (pattern.crossSegments) glob = glob.replace(/\*+/g, '**'); + const candidates = [rel]; + if (pattern.ancestors) { + for (let at = rel.lastIndexOf('/'); at > 0; at = rel.lastIndexOf('/', at - 1)) candidates.push(rel.slice(0, at)); + } + return candidates.some(text => globMatches(glob, text, false) === true); +} + +// One alternative per path operand: a literal operand admits what its scope covers, a pattern what it matches. +function operandPatternsAdmitTarget(item, ctx) { + const patterns = Array.isArray(item.scopePatterns) ? item.scopePatterns : []; + if (!patterns.some(Boolean) || !Array.isArray(item.scopes)) return true; + return item.scopes.some((parts, index) => { + if (patterns[index]) return pathPatternAdmitsTarget(patterns[index], ctx); + const scope = ctx.resolveDir(...parts); + return scope === ctx.targetKey || isInsideDir(ctx.targetKey, scope); + }); +} + +function literalBasename(glob) { + const last = normalizeFilterGlob(glob).split('/').pop(); + return last && !PATH_GLOB_CHARS.test(last) ? last : null; +} + +// see docs/gateguard/design-notes.md#search-scope +/** Whether every include or path pattern that limits the search names only the target's own file name. */ +function confinedToTargetName(item, ctx) { + const name = (ctx.targetKey.split('/').pop() || '').toLowerCase(); + const namesOnly = globs => globs.length > 0 && globs.every(glob => literalBasename(glob) === name); + if (namesOnly(Array.isArray(item.includes) ? item.includes : [])) return true; + if (item.pathPattern && namesOnly([item.pathPattern.glob])) return true; + const patterns = Array.isArray(item.scopePatterns) ? item.scopePatterns : []; + return patterns.length > 0 && patterns.every(Boolean) && namesOnly(patterns.map(pattern => pattern.glob)); +} + +function grepToolFilters(glob, type) { const filters = { positives: [], includes: [], exclusions: [], opaque: false }; + if (typeof type === 'string' && type) { + const globs = rgTypeGlobs(type); + if (globs) filters.includes.push(...globs); + else filters.opaque = true; + } if (typeof glob !== 'string' || !glob) return filters; for (const part of splitTopLevelCommas(glob)) { if (part.startsWith('!')) filters.exclusions.push(part.slice(1)); @@ -293,6 +415,13 @@ function applyShellFilterArg(kind, args, i, filters) { const include = value => { if (value) filters.includes.push(value); }; + const type = (name, indexes, excluded) => { + const globs = rgTypeGlobs(name); + if (!globs) filters.opaque = true; + else if (excluded) globs.forEach(glob => exclude(glob, indexes)); + else globs.forEach(include); + }; + const extension = value => include(value ? `*.${String(value).replace(/^\./, '')}` : ''); if (kind.startsWith('git ') && GIT_EXCLUDE_PATHSPEC.test(arg)) { exclude(arg.replace(GIT_EXCLUDE_PATHSPEC, ''), [i]); return 0; @@ -303,25 +432,7 @@ function applyShellFilterArg(kind, args, i, filters) { const hasNext = eq <= 0 && i + 1 < args.length; const value = eq > 0 ? arg.slice(eq + 1) : hasNext ? args[i + 1] : ''; const indexes = eq > 0 || !hasNext ? [i] : [i, i + 1]; - const consumed = indexes.length - 1; - if (OPAQUE_FILTER_FLAGS.has(name)) { - filters.opaque = true; - return consumed; - } - if (name === '--glob' || name === '--iglob') { - if (value.startsWith('!')) exclude(value.slice(1), indexes); - else include(value); - return consumed; - } - if (name === '--include') { - include(value); - return consumed; - } - if (EXCLUDE_LONG_FLAGS.has(name) && (kind === 'ls' || !LS_ONLY_EXCLUDE_FLAGS.has(name))) { - exclude(value, indexes); - return consumed; - } - return 0; + return applyLongFilter(kind, name, value, indexes, { filters, exclude, include, type, extension }) ? indexes.length - 1 : 0; } if (!/^-[A-Za-z]/.test(arg)) return 0; const letters = shortFilterLetters(kind); @@ -336,23 +447,48 @@ function applyShellFilterArg(kind, args, i, filters) { const value = attached || args[i + 1] || ''; const indexes = attached || i + 1 >= args.length ? [i] : [i, i + 1]; const consumed = indexes.length - 1; - applyShortFilter(kind, letter, value, indexes, { filters, exclude, include }); + applyShortFilter(kind, letter, value, indexes, { filters, exclude, include, type, extension }); return consumed; } return 0; } +/** Apply one long filter flag; false when the flag is not a filter. */ +function applyLongFilter(kind, name, value, indexes, { filters, exclude, include, type, extension }) { + if (OPAQUE_FILTER_FLAGS.has(name) || (kind === 'rg' && (name === '--type-add' || name === '--type-clear'))) { + filters.opaque = true; + } else if (kind === 'rg' && (name === '--type' || name === '--type-not')) { + type(value, indexes, name === '--type-not'); + } else if (kind === 'fd' && name === '--extension') { + extension(value); + } else if (name === '--glob' || name === '--iglob') { + if (value.startsWith('!')) exclude(value.slice(1), indexes); + else include(value); + } else if (name === '--include') { + include(value); + } else if (EXCLUDE_LONG_FLAGS.has(name) && (kind === 'ls' || !LS_ONLY_EXCLUDE_FLAGS.has(name))) { + exclude(value, indexes); + } else { + return false; + } + return true; +} + function shortFilterLetters(kind) { - if (kind === 'rg') return new Set(['g']); - if (kind === 'fd') return new Set(['E']); + if (kind === 'rg') return new Set(['g', 't', 'T']); + if (kind === 'fd') return new Set(['E', 'e']); if (kind === 'ls') return new Set(['I']); if (kind === 'tree') return new Set(['I', 'P']); if (kind === 'git ls-files') return new Set(['x', 'X']); return new Set(); } -function applyShortFilter(kind, letter, value, indexes, { filters, exclude, include }) { - if (kind === 'rg') { +function applyShortFilter(kind, letter, value, indexes, { filters, exclude, include, type, extension }) { + if (kind === 'rg' && (letter === 't' || letter === 'T')) { + type(value, indexes, letter === 'T'); + } else if (kind === 'fd' && letter === 'e') { + extension(value); + } else if (kind === 'rg') { if (value.startsWith('!')) exclude(value.slice(1), indexes); else include(value); } else if (kind === 'tree' && letter === 'P') { @@ -455,5 +591,7 @@ module.exports = { grepToolFilters, shellSearchFilters, filtersAdmitTarget, - includesAdmitTarget + includesAdmitTarget, + pathPatternFor, + confinedToTargetName };