Files
reverse-engineering-anything/src/reference/ReferenceSourceReaderTraversal.ts
T
morluto 65f78edce5 perf(reference): overlap reference-source identity lookups in traversal order (#11585)
The reference-source reader made about nine serial filesystem calls per
directory. It now runs its read-only identity lookups ahead of their turn,
bounded to four running at once (libuv's default filesystem concurrency):
opening admissions of the next pending directories, metadata of the next
listed names, and closing admissions of directories already enumerated.
Results are applied in traversal order: policy and exclusion callbacks keep
their order, the first error in traversal order is returned, a failure a
started lookup observed is retained, and every started lookup settles before
the read returns. Running opening admissions are counted, so older siblings
buried by deeper levels cannot raise concurrency with tree depth.

Measured on macOS, Node 24.22.1: the 130,000-directory regression's
reference-source read 27.5 s -> 17.2 s; a 20,000-directory flat tree
2.1-2.3 s -> 1.2 s; an 8,641-file source tree 1.41 s -> 1.31 s.

Refs #11365
2026-11-16 07:57:51 +08:00

59 lines
1.9 KiB
TypeScript

import type { BigIntStats } from "node:fs";
import type { ArtifactResourceScope } from "../resources/index.js";
import type {
ReferenceSourceEntry,
ReferenceSourceExclusionKind,
ReferenceSourceResult,
ReferenceSourceRootFailure,
} from "./ReferenceSourceReaderTypes.js";
import type { DirectoryValidation } from "./ReferenceSourceReaderValidate.js";
/** Traversal state private to the reader; not part of its contract. */
export type PendingDirectory = {
readonly path: string;
/**
* Identity admission started ahead of this directory's turn. It is an
* observation only; traversal consumes it in its original order.
*/
validation?: Promise<DirectoryValidation>;
};
export type TraversalState = {
readonly resources: ArtifactResourceScope;
readonly root: string;
readonly rootIdentity: BigIntStats;
readonly signal?: AbortSignal;
/** Prepare policy after identity admission and opening, before enumeration; refusal closes the owned handle. */
readonly beforeDirectory?: (
path: string,
) => Promise<ReferenceSourceResult<undefined>>;
readonly shouldExclude?: (
path: string,
kind: ReferenceSourceExclusionKind,
) => boolean;
readonly entries: ReferenceSourceEntry[];
readonly exclusions: Array<{
readonly path: string;
readonly kind: ReferenceSourceExclusionKind;
}>;
readonly pending: PendingDirectory[];
/** Closing admissions of enumerated directories, oldest first. */
readonly settling: Promise<void>[];
/** Opening admissions started ahead of their turn and still running. */
openingLookups: number;
rootFailure?: ReferenceSourceRootFailure;
rootFailureCause?: { readonly value: unknown };
bytesRead: number;
};
export type StableFileRequest = {
readonly resources: ArtifactResourceScope;
readonly root: string;
readonly rootIdentity: BigIntStats;
readonly absolute: string;
readonly path: string;
readonly expected: BigIntStats;
readonly signal?: AbortSignal;
};