mirror of
https://github.com/morluto/rea.git
synced 2026-11-16 11:43:20 +00:00
The reference-source reader made about nine serial filesystem calls per directory. It now runs its read-only identity lookups ahead of their turn, bounded to four running at once (libuv's default filesystem concurrency): opening admissions of the next pending directories, metadata of the next listed names, and closing admissions of directories already enumerated. Results are applied in traversal order: policy and exclusion callbacks keep their order, the first error in traversal order is returned, a failure a started lookup observed is retained, and every started lookup settles before the read returns. Running opening admissions are counted, so older siblings buried by deeper levels cannot raise concurrency with tree depth. Measured on macOS, Node 24.22.1: the 130,000-directory regression's reference-source read 27.5 s -> 17.2 s; a 20,000-directory flat tree 2.1-2.3 s -> 1.2 s; an 8,641-file source tree 1.41 s -> 1.31 s. Refs #11365
732 lines
24 KiB
TypeScript
732 lines
24 KiB
TypeScript
import {
|
|
mkdir,
|
|
lstat as fsLstat,
|
|
open as fsOpen,
|
|
opendir as fsOpendir,
|
|
writeFile,
|
|
type FileHandle,
|
|
} from "node:fs/promises";
|
|
import { dirname, join } from "node:path";
|
|
import { setTimeout as delay } from "node:timers/promises";
|
|
import type { Dirent } from "node:fs";
|
|
|
|
import { afterEach, beforeEach, expect, it, onTestFinished, vi } from "vitest";
|
|
|
|
import { ArtifactResourceScope } from "../../../src/resources/ArtifactResourceScope.js";
|
|
import { importReferenceSource } from "../../../src/application/ReferenceSourceImport.js";
|
|
import { referenceSourceImportAnalysisError } from "../../../src/application/ReferenceSourceImportTypes.js";
|
|
import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js";
|
|
import { readReferenceSource } from "../../../src/reference/ReferenceSourceReader.js";
|
|
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
|
import { err, ok } from "../../../src/domain/result.js";
|
|
|
|
vi.mock("node:fs/promises", async (importOriginal) => {
|
|
const actual = await importOriginal<typeof import("node:fs/promises")>();
|
|
return {
|
|
...actual,
|
|
open: vi.fn(actual.open),
|
|
lstat: vi.fn(actual.lstat),
|
|
opendir: vi.fn(actual.opendir),
|
|
};
|
|
});
|
|
|
|
const openMock = vi.mocked(fsOpen);
|
|
const lstatMock = vi.mocked(fsLstat);
|
|
const opendirMock = vi.mocked(fsOpendir);
|
|
let actual: typeof import("node:fs/promises");
|
|
|
|
interface DirectoryOperations {
|
|
read(): Promise<Dirent | null>;
|
|
close(): Promise<void>;
|
|
}
|
|
|
|
beforeEach(async () => {
|
|
actual =
|
|
await vi.importActual<typeof import("node:fs/promises")>(
|
|
"node:fs/promises",
|
|
);
|
|
openMock.mockImplementation((...args) => actual.open(...args));
|
|
lstatMock.mockImplementation((...args) => actual.lstat(...args));
|
|
opendirMock.mockImplementation((...args) => actual.opendir(...args));
|
|
});
|
|
|
|
afterEach(() => {
|
|
openMock.mockReset();
|
|
lstatMock.mockReset();
|
|
opendirMock.mockReset();
|
|
});
|
|
|
|
it.each([
|
|
"root-admission",
|
|
"directory-validation",
|
|
"entry-metadata",
|
|
"file-finalization",
|
|
] as const)(
|
|
"retains physical metadata failure during cancellation at %s",
|
|
async (phase) => {
|
|
const root = await createTestTempDirectory("rea-reference-metadata-abort-");
|
|
const file = join(root, "source.ts");
|
|
if (phase === "entry-metadata" || phase === "file-finalization")
|
|
await writeFile(file, "observed");
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close());
|
|
const controller = new AbortController();
|
|
const physical = Object.assign(new Error("authored metadata EIO"), {
|
|
code: "EIO",
|
|
});
|
|
let armed = phase === "root-admission";
|
|
if (phase === "file-finalization")
|
|
openMock.mockImplementation(async (...args) => {
|
|
const handle = await actual.open(...args);
|
|
const read = handle.read.bind(handle);
|
|
vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => {
|
|
const result = await read(...readArgs);
|
|
if (result.bytesRead === 0) armed = true;
|
|
return result;
|
|
});
|
|
return handle;
|
|
});
|
|
lstatMock.mockImplementation(async (...args) => {
|
|
if (!armed || (phase === "entry-metadata" && String(args[0]) !== file))
|
|
return actual.lstat(...args);
|
|
controller.abort("during failed metadata lookup");
|
|
throw physical;
|
|
});
|
|
const result = await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
beforeDirectory: async () => {
|
|
if (phase !== "file-finalization") armed = true;
|
|
return ok(undefined);
|
|
},
|
|
});
|
|
if (result.ok) throw new Error("Metadata failure was ignored");
|
|
if (phase === "root-admission") {
|
|
expect(result.error).toMatchObject({ code: "io", cause: physical });
|
|
} else if (phase === "directory-validation") {
|
|
expect(result.error.code).toBe("cancelled");
|
|
expect(result.error.partial?.rootFailure).toMatchObject({
|
|
code: "io",
|
|
message: expect.stringContaining("EIO"),
|
|
});
|
|
} else {
|
|
expect(result.error.code).toBe("cancelled");
|
|
expect(result.error.partial?.entries).toContainEqual(
|
|
expect.objectContaining({ path: "source.ts", code: "io" }),
|
|
);
|
|
}
|
|
},
|
|
);
|
|
|
|
it.each([
|
|
["refusal", false],
|
|
["refusal", true],
|
|
["cancel-after-policy", false],
|
|
["cancel-after-policy", true],
|
|
["refusal-late-cancel", true],
|
|
] as const)(
|
|
"owns directory cleanup before policy %s (close fails=%s)",
|
|
async (outcome, closeFails) => {
|
|
const root = await createTestTempDirectory("rea-reference-policy-owner-");
|
|
await writeFile(join(root, "unexamined.ts"), "export const value = 1;\n");
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close().catch(() => undefined));
|
|
const controller = new AbortController();
|
|
const reason = new Error("cancelled at the policy boundary");
|
|
const policyCause = new Error("scoped policy read failed");
|
|
const closeFailure = new Error("directory close rejected after policy");
|
|
let closeCalls = 0;
|
|
let readCalls = 0;
|
|
opendirMock.mockImplementation(async (...args) => {
|
|
const directory = await actual.opendir(...args);
|
|
const operations: DirectoryOperations = directory;
|
|
const read = operations.read.bind(directory);
|
|
vi.spyOn(operations, "read").mockImplementation(async () => {
|
|
readCalls += 1;
|
|
return read();
|
|
});
|
|
const close = operations.close.bind(directory);
|
|
vi.spyOn(operations, "close").mockImplementation(async () => {
|
|
closeCalls += 1;
|
|
if (outcome === "refusal-late-cancel") controller.abort(reason);
|
|
await close();
|
|
if (closeFails) throw closeFailure;
|
|
});
|
|
return directory;
|
|
});
|
|
const result = await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
beforeDirectory: async () => {
|
|
expect(opendirMock).toHaveBeenCalledTimes(1);
|
|
if (outcome === "cancel-after-policy") {
|
|
controller.abort(reason);
|
|
return ok(undefined);
|
|
}
|
|
return err({
|
|
tag: "reference-source-reader",
|
|
code: "io",
|
|
message: "Scoped policy could not be read",
|
|
cause: policyCause,
|
|
});
|
|
},
|
|
});
|
|
if (result.ok) throw new Error("Policy outcome was ignored");
|
|
expect(result.error.code).toBe(
|
|
outcome === "cancel-after-policy" ? "cancelled" : "io",
|
|
);
|
|
if (outcome !== "cancel-after-policy") {
|
|
expect(result.error.message).toBe("Scoped policy could not be read");
|
|
expect(result.error.cause).toBe(policyCause);
|
|
}
|
|
expect(result.error.partial).toMatchObject({ entries: [], bytesRead: 0 });
|
|
expect(readCalls).toBe(0);
|
|
expect(openMock).not.toHaveBeenCalled();
|
|
expect(closeCalls).toBe(1);
|
|
expect(resources.hasPending).toBe(closeFails);
|
|
if (closeFails) {
|
|
expect(result.error.cleanup).toEqual({
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
});
|
|
await expect(resources.close()).rejects.toMatchObject({
|
|
cleanup: { resources: [root] },
|
|
});
|
|
} else {
|
|
expect(result.error.cleanup).toBeUndefined();
|
|
await resources.close();
|
|
}
|
|
expect(closeCalls).toBe(1);
|
|
},
|
|
);
|
|
|
|
it.each([false, true])(
|
|
"preserves admission EIO and retains its failed-close owner (cancelled=%s)",
|
|
async (cancelled) => {
|
|
const root = await createTestTempDirectory(
|
|
"rea-reference-admission-owner-",
|
|
);
|
|
const file = join(root, "source.ts");
|
|
await writeFile(file, "export const answer = 42;\n");
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close().catch(() => undefined));
|
|
const statFailure = Object.assign(new Error("injected admission EIO"), {
|
|
code: "EIO",
|
|
});
|
|
const closeFailure = new Error("native close rejected after invalidation");
|
|
const controller = new AbortController();
|
|
let closeCalls = 0;
|
|
|
|
openMock.mockImplementation(async (...args) => {
|
|
const handle = await actual.open(...args);
|
|
vi.spyOn(handle, "stat").mockImplementationOnce(async () => {
|
|
if (cancelled) controller.abort("during failed admission");
|
|
throw statFailure;
|
|
});
|
|
const close = handle.close.bind(handle);
|
|
vi.spyOn(handle, "close").mockImplementation(async () => {
|
|
closeCalls += 1;
|
|
await close();
|
|
throw closeFailure;
|
|
});
|
|
return handle;
|
|
});
|
|
|
|
const result = await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
});
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
tag: "reference-source-reader",
|
|
code: "io",
|
|
message: expect.stringContaining("EIO"),
|
|
cleanup: {
|
|
resources: [file],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
partial: {
|
|
entries: [
|
|
{
|
|
status: "failed",
|
|
kind: "file",
|
|
path: "source.ts",
|
|
code: "io",
|
|
message: expect.stringContaining("EIO"),
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
expect(closeCalls).toBe(1);
|
|
|
|
await expect(resources.close()).rejects.toMatchObject({
|
|
cleanup: {
|
|
resources: [file],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
});
|
|
expect(closeCalls).toBe(1);
|
|
},
|
|
);
|
|
|
|
const assertSourceCleanupRetry = async (enclosingGit: boolean) => {
|
|
const workspace = await createTestTempDirectory("rea-reference-file-owner-");
|
|
const root = join(workspace, "source");
|
|
await mkdir(root);
|
|
if (enclosingGit) {
|
|
const gitDirectory = join(workspace, ".git");
|
|
await mkdir(gitDirectory);
|
|
await writeFile(
|
|
join(gitDirectory, "HEAD"),
|
|
"0123456789abcdef0123456789abcdef01234567\n",
|
|
);
|
|
}
|
|
const file = join(root, "source.ts");
|
|
const bytes = "export const answer = 'retained';\n";
|
|
await writeFile(file, bytes);
|
|
const resources = new ArtifactResourceScope();
|
|
const closeFailure = new Error("injected close before native close");
|
|
let allowClose = false;
|
|
onTestFinished(async () => {
|
|
allowClose = true;
|
|
await resources.close().catch(() => undefined);
|
|
});
|
|
let closeCalls = 0;
|
|
let openCalls = 0;
|
|
let sourceHandle: FileHandle | undefined;
|
|
|
|
openMock.mockImplementation(async (...args) => {
|
|
if (args[0] !== file) return actual.open(...args);
|
|
openCalls += 1;
|
|
const handle = await actual.open(...args);
|
|
sourceHandle = handle;
|
|
const close = handle.close.bind(handle);
|
|
vi.spyOn(handle, "close").mockImplementation(async () => {
|
|
closeCalls += 1;
|
|
if (!allowClose) throw closeFailure;
|
|
await close();
|
|
});
|
|
return handle;
|
|
});
|
|
|
|
const first = await importReferenceSource(
|
|
{
|
|
root,
|
|
caller: "reference-source-cleanup-owner-test",
|
|
policy: { secretPatterns: [] },
|
|
},
|
|
resources,
|
|
);
|
|
expect(first).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "io",
|
|
cleanup: {
|
|
resources: [file],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
partial: {
|
|
entries: [
|
|
{
|
|
status: "read",
|
|
kind: "file",
|
|
path: "source.ts",
|
|
bytes: Buffer.from(bytes),
|
|
},
|
|
],
|
|
bytesRead: Buffer.byteLength(bytes),
|
|
},
|
|
},
|
|
});
|
|
expect(openCalls).toBe(1);
|
|
expect(closeCalls).toBe(1);
|
|
expect(sourceHandle?.fd).toBeGreaterThanOrEqual(0);
|
|
|
|
const blocked = await readReferenceSource(root, resources);
|
|
expect(blocked).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "io",
|
|
cleanup: { resources: [file] },
|
|
},
|
|
});
|
|
expect(openCalls).toBe(1);
|
|
expect(closeCalls).toBe(2);
|
|
expect(sourceHandle?.fd).toBeGreaterThanOrEqual(0);
|
|
|
|
allowClose = true;
|
|
await resources.close();
|
|
expect(openCalls).toBe(1);
|
|
expect(closeCalls).toBe(3);
|
|
expect(sourceHandle?.fd).toBe(-1);
|
|
};
|
|
|
|
it.each([false, true])(
|
|
"retains partial file bytes and retries the same live owner before new work (enclosing Git repository: %s)",
|
|
assertSourceCleanupRetry,
|
|
);
|
|
|
|
it.each([
|
|
[false, false, "reader"],
|
|
[false, true, "reader"],
|
|
[true, false, "reader"],
|
|
[true, true, "reader"],
|
|
[true, false, "import"],
|
|
[true, true, "import"],
|
|
] as const)(
|
|
"preserves directory EIO through close (late cancellation=%s, close fails=%s, surface=%s)",
|
|
async (lateCancellation, closeFails, surface) => {
|
|
const root = await createTestTempDirectory(
|
|
"rea-reference-directory-read-owner-",
|
|
);
|
|
const file = join(root, "source.ts");
|
|
const bytes = "observed before directory EIO\n";
|
|
await writeFile(file, bytes);
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close().catch(() => undefined));
|
|
const readFailure = Object.assign(new Error("injected directory EIO"), {
|
|
code: "EIO",
|
|
});
|
|
const closeFailure = new Error(
|
|
"directory close rejected after invalidation",
|
|
);
|
|
const controller = new AbortController();
|
|
const reason = { selected: "after directory read failed" };
|
|
let readCalls = 0;
|
|
let closeCalls = 0;
|
|
|
|
opendirMock.mockImplementation(async (...args) => {
|
|
const directory = await actual.opendir(...args);
|
|
const operations: DirectoryOperations = directory;
|
|
const read = operations.read.bind(directory);
|
|
vi.spyOn(operations, "read").mockImplementation(async () => {
|
|
if (readCalls++ > 0) throw readFailure;
|
|
return read();
|
|
});
|
|
const close = operations.close.bind(directory);
|
|
vi.spyOn(operations, "close").mockImplementation(async () => {
|
|
closeCalls += 1;
|
|
if (lateCancellation) controller.abort(reason);
|
|
await close();
|
|
if (closeFails) throw closeFailure;
|
|
});
|
|
return directory;
|
|
});
|
|
|
|
const result =
|
|
surface === "reader"
|
|
? await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
})
|
|
: await importReferenceSource(
|
|
{
|
|
root,
|
|
caller: "reference-cancellation-test",
|
|
policy: { secretPatterns: [] },
|
|
signal: controller.signal,
|
|
},
|
|
resources,
|
|
);
|
|
if (result.ok)
|
|
throw new Error("Directory and cleanup failures were ignored");
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "io",
|
|
message: expect.stringContaining(readFailure.message),
|
|
...(closeFails
|
|
? {
|
|
cleanup: {
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
}
|
|
: {}),
|
|
partial: {
|
|
bytesRead: Buffer.byteLength(bytes),
|
|
},
|
|
},
|
|
});
|
|
if (lateCancellation)
|
|
expect(result.error.secondaryCancellation?.cause).toBe(reason);
|
|
expect(result.error.cause).toBe(readFailure);
|
|
if (
|
|
surface === "import" &&
|
|
result.error.tag === "reference-source-import"
|
|
) {
|
|
const projected = projectAnalysisError(
|
|
referenceSourceImportAnalysisError(result.error),
|
|
);
|
|
expect(projected.details?.secondary_failures).toEqual([
|
|
expect.objectContaining({ category: "cancelled" }),
|
|
]);
|
|
}
|
|
expect(result.error.partial?.entries).toContainEqual(
|
|
expect.objectContaining({
|
|
status: "read",
|
|
kind: "file",
|
|
path: "source.ts",
|
|
bytes: Buffer.from(bytes),
|
|
}),
|
|
);
|
|
expect(result.error.partial?.rootFailure).toEqual({
|
|
code: "io",
|
|
message: expect.stringContaining(readFailure.message),
|
|
});
|
|
expect(closeCalls).toBe(1);
|
|
|
|
if (closeFails)
|
|
await expect(resources.close()).rejects.toMatchObject({
|
|
cleanup: {
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
});
|
|
else {
|
|
expect(result.error.cleanup).toBeUndefined();
|
|
await resources.close();
|
|
}
|
|
expect(closeCalls).toBe(1);
|
|
},
|
|
);
|
|
|
|
it("keeps cancelled traversal typed when native directory close rejects after invalidation", async () => {
|
|
const root = await createTestTempDirectory("rea-reference-directory-owner-");
|
|
await mkdir(join(root, "child"));
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close().catch(() => undefined));
|
|
const controller = new AbortController();
|
|
const closeFailure = new Error("native directory close rejected");
|
|
let closeCalls = 0;
|
|
let openedPath: string | undefined;
|
|
|
|
opendirMock.mockImplementation(async (...args) => {
|
|
const directory = await actual.opendir(...args);
|
|
const operations: DirectoryOperations = directory;
|
|
openedPath = String(args[0]);
|
|
const read = operations.read.bind(directory);
|
|
vi.spyOn(operations, "read").mockImplementation(async () => {
|
|
const child = await read();
|
|
if (child !== null) controller.abort();
|
|
return child;
|
|
});
|
|
const close = operations.close.bind(directory);
|
|
vi.spyOn(operations, "close").mockImplementation(async () => {
|
|
closeCalls += 1;
|
|
await close();
|
|
throw closeFailure;
|
|
});
|
|
return directory;
|
|
});
|
|
|
|
const result = await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
});
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: {
|
|
code: "cancelled",
|
|
cleanup: {
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
partial: { entries: [] },
|
|
},
|
|
});
|
|
expect(openedPath).toBe(root);
|
|
expect(closeCalls).toBe(1);
|
|
|
|
await expect(resources.close()).rejects.toMatchObject({
|
|
cleanup: {
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
});
|
|
expect(closeCalls).toBe(1);
|
|
});
|
|
|
|
it("preserves null cancellation and retained directory cleanup across retry", async () => {
|
|
const root = await createTestTempDirectory("rea-reference-null-close-");
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close().catch(() => undefined));
|
|
const controller = new AbortController();
|
|
const closeFailure = new Error("directory close temporarily denied");
|
|
let closeCalls = 0;
|
|
let releaseFixture: (() => Promise<void>) | undefined;
|
|
onTestFinished(() => releaseFixture?.());
|
|
opendirMock.mockImplementation(async (...args) => {
|
|
const directory = await actual.opendir(...args);
|
|
const operations: DirectoryOperations = directory;
|
|
const close = operations.close.bind(directory);
|
|
releaseFixture = close;
|
|
vi.spyOn(operations, "close").mockImplementation(async () => {
|
|
closeCalls += 1;
|
|
if (closeCalls === 1) throw closeFailure;
|
|
await close();
|
|
});
|
|
return directory;
|
|
});
|
|
const result = await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
beforeDirectory: async () => {
|
|
controller.abort(null);
|
|
return ok(undefined);
|
|
},
|
|
});
|
|
if (result.ok) throw new Error("Expected cancellation and cleanup failure");
|
|
expect(result.error.code).toBe("cancelled");
|
|
expect(result.error.cause).toBeNull();
|
|
expect(result.error.cleanup).toEqual({
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
});
|
|
expect(result.error.partial).toMatchObject({ entries: [], bytesRead: 0 });
|
|
expect(resources.hasPending).toBe(true);
|
|
expect(resources.cleanupResources).toEqual([root]);
|
|
expect(closeCalls).toBe(1);
|
|
await expect(resources.close()).rejects.toMatchObject({
|
|
cleanup: {
|
|
resources: [root],
|
|
reason: expect.stringContaining(closeFailure.message),
|
|
},
|
|
});
|
|
expect(resources.hasPending).toBe(true);
|
|
expect(resources.cleanupResources).toEqual([root]);
|
|
expect(closeCalls).toBe(1);
|
|
});
|
|
|
|
// Directory admissions run ahead of their turn (#11365). Each later lookup
|
|
// below finishes sooner than the one before it, so completion order reverses
|
|
// start order; policy must still run, and refuse first, in traversal order.
|
|
it("runs directory policy in traversal order when prefetched admissions finish out of order", async () => {
|
|
const root = await createTestTempDirectory("rea-reference-ordered-");
|
|
for (const name of ["a", "b", "c", "d", "e", "f"])
|
|
await mkdir(join(root, name));
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close());
|
|
let started = 0;
|
|
lstatMock.mockImplementation(async (...args) => {
|
|
if (dirname(String(args[0])) === root) {
|
|
started += 1;
|
|
await delay(Math.max(0, 80 - started * 10));
|
|
}
|
|
return actual.lstat(...args);
|
|
});
|
|
const listed: string[] = [];
|
|
const policy: string[] = [];
|
|
const result = await readReferenceSource(root, resources, {
|
|
shouldExclude: (path, kind) => {
|
|
if (kind === "directory") listed.push(path);
|
|
return false;
|
|
},
|
|
beforeDirectory: async (path) => {
|
|
policy.push(path);
|
|
return path === "."
|
|
? ok(undefined)
|
|
: err({
|
|
tag: "reference-source-reader",
|
|
code: "io",
|
|
message: `refused ${path}`,
|
|
});
|
|
},
|
|
});
|
|
expect(listed).toHaveLength(6);
|
|
expect(policy).toEqual([".", listed[0]]);
|
|
expect(result).toMatchObject({
|
|
ok: false,
|
|
error: { code: "io", message: `refused ${listed[0]}` },
|
|
});
|
|
});
|
|
|
|
it("settles every lookup started ahead of its turn before a cancelled read returns", async () => {
|
|
const root = await createTestTempDirectory("rea-reference-settled-");
|
|
for (const name of ["a", "b", "c", "d", "e", "f"]) {
|
|
await mkdir(join(root, name));
|
|
await writeFile(join(root, name, "source.ts"), "observed");
|
|
}
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close());
|
|
// Each later lookup takes longer, so lookups started ahead of the cancelled
|
|
// directory are still running when it stops.
|
|
let started = 0;
|
|
let inFlight = 0;
|
|
lstatMock.mockImplementation(async (...args) => {
|
|
started += 1;
|
|
inFlight += 1;
|
|
try {
|
|
await delay(started * 3);
|
|
return await actual.lstat(...args);
|
|
} finally {
|
|
inFlight -= 1;
|
|
}
|
|
});
|
|
const controller = new AbortController();
|
|
const result = await readReferenceSource(root, resources, {
|
|
signal: controller.signal,
|
|
beforeDirectory: async (path) => {
|
|
if (path !== ".") controller.abort("stop at the first child");
|
|
return ok(undefined);
|
|
},
|
|
});
|
|
expect(inFlight).toBe(0);
|
|
expect(result).toMatchObject({ ok: false, error: { code: "cancelled" } });
|
|
});
|
|
|
|
// Children pushed during descent bury older siblings whose admissions are
|
|
// still running; the number running at once stays bounded at every depth.
|
|
it("bounds running directory admissions while slow siblings are buried by deeper levels", async () => {
|
|
const root = await createTestTempDirectory("rea-reference-bounded-");
|
|
const levels = 6;
|
|
let parent = root;
|
|
for (let level = 0; level < levels; level += 1) {
|
|
for (const sibling of [1, 2, 3])
|
|
await mkdir(join(parent, `s${String(level)}-${String(sibling)}`));
|
|
parent = join(parent, `d${String(level)}`);
|
|
await mkdir(parent);
|
|
}
|
|
const resources = new ArtifactResourceScope();
|
|
onTestFinished(() => resources.close());
|
|
// List names in byte order, so each level descends ("d") before its
|
|
// siblings ("s") are popped.
|
|
opendirMock.mockImplementation(async (...args) => {
|
|
const directory = await actual.opendir(...args);
|
|
const children: Dirent[] = [];
|
|
for await (const child of directory) children.push(child);
|
|
children.sort((left, right) =>
|
|
Buffer.compare(
|
|
Buffer.from(left.name as unknown as Uint8Array),
|
|
Buffer.from(right.name as unknown as Uint8Array),
|
|
),
|
|
);
|
|
const operations: DirectoryOperations = directory;
|
|
vi.spyOn(operations, "read").mockImplementation(async () =>
|
|
Promise.resolve(children.shift() ?? null),
|
|
);
|
|
vi.spyOn(operations, "close").mockImplementation(async () => undefined);
|
|
return directory;
|
|
});
|
|
// A directory's second lookup is its opening admission (the first is its
|
|
// parent's metadata lookup); hold the siblings' admissions open.
|
|
const lookups = new Map<string, number>();
|
|
let running = 0;
|
|
let peak = 0;
|
|
lstatMock.mockImplementation(async (...args) => {
|
|
const path = String(args[0]);
|
|
const count = (lookups.get(path) ?? 0) + 1;
|
|
lookups.set(path, count);
|
|
if (count !== 2 || !/\/s\d+-\d+$/u.test(path)) return actual.lstat(...args);
|
|
running += 1;
|
|
peak = Math.max(peak, running);
|
|
try {
|
|
await delay(100);
|
|
return await actual.lstat(...args);
|
|
} finally {
|
|
running -= 1;
|
|
}
|
|
});
|
|
const result = await readReferenceSource(root, resources);
|
|
if (!result.ok) throw result.error;
|
|
expect(result.value.entries).toHaveLength(levels * 4);
|
|
// Four started ahead plus the popped directory's own admission.
|
|
expect(peak).toBeLessThanOrEqual(5);
|
|
});
|