reverse-engineering-anything/src/artifacts/inventory/scanCanonical.ts

import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";

import { AsarArtifactReader } from "../AsarArtifactReader.js";

import { lstat } from "node:fs/promises";
import type { Stats } from "node:fs";
import type { RootInventorySource } from "./classify.js";

import {
  ArtifactReaderFailure,
  type ArtifactReader,
  type ArtifactReaderTraversal,
} from "../ArtifactReader.js";
import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js";
import {
  artifactInventoryResultSchema,
  type ArtifactInventoryResult,
  type ArtifactNode,
  type ArtifactOccurrence,
  type IntegrityContradiction,
} from "../../domain/artifactGraph.js";
import {
  createArtifactEdges,
  createRootNode,
  materializeDirectoryNodes,
  rekeyOccurrences,
  rootOccurrenceFor,
  type MutableOccurrence,
} from "./ArtifactGraphConstruction.js";
import {
  artifactContradictionId,
  artifactGraphDigest,
  artifactManifestId,
} from "../../domain/artifactIdentity.js";
import { classifyAndHashRootForInventory } from "./classify.js";
import { hostExecutableMarking } from "../hostExecutableMarking.js";
import type { HashResult } from "../ArtifactHash.js";
import {
  hashStableRootArtifact,
  hashStableRootArtifactHandle,
} from "./hashStableRootArtifact.js";
import { createReader, inventoryCoverage } from "./reader.js";
import {
  scanReader,
  type PendingIntegrityContradiction,
} from "./scanReader.js";
import {
  STRICT_INTEGRITY_POLICY,
  type ArtifactInventoryOptions,
} from "./types.js";

export const scanCanonicalArtifactInventory = async (
  path: string,
  options: ArtifactInventoryOptions,
  readerFactory: typeof createReader = createReader,
): Promise<ArtifactInventorySnapshot> =>
  options.resourceScope.run(() =>
    scanCanonicalArtifactInventoryInScope(path, options, readerFactory),
  );

export const scanCanonicalArtifactInventoryInScope = async (
  path: string,
  options: ArtifactInventoryOptions,
  readerFactory: typeof createReader = createReader,
): Promise<ArtifactInventorySnapshot> => {
  const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
  const metadata = await lstat(path);
  const {
    format: rootFormat,
    digest: rootDigest,
    rootSource,
  } = await classifyAndHashRootForInventory(
    path,
    metadata.isDirectory(),
    metadata,
    options,
  );
  let reader: ArtifactReader | undefined;
  let readerCreationFailure: { readonly cause: unknown } | undefined;
  try {
    reader = readerFactory(
      path,
      rootFormat,
      options.environment ?? {},
      rootSource,
    );
  } catch (cause: unknown) {
    readerCreationFailure = { cause };
  }
  const ownedReaders: ArtifactReader[] = reader === undefined ? [] : [reader];
  let outcome: InventoryScanOutcome;
  try {
    if (readerCreationFailure !== undefined) throw readerCreationFailure.cause;
    if (reader instanceof AsarArtifactReader)
      await reader.prepareContainer(rootDigest?.sha256, options.signal);
    const {
      nodes,
      occurrences,
      pendingContradictions,
      caseCollisionLimitation,
      unknownParentOccurrences,
      traversal,
    } = await scanReader(reader, ownedReaders, options.signal, integrity);
    outcome = {
      kind: "completed",
      snapshot: await buildInventorySnapshot({
        path,
        metadata,
        rootFormat,
        rootDigest,
        rootSource,
        resourceScope: options.resourceScope,
        reader,
        signal: options.signal,
        nodes,
        occurrences,
        pendingContradictions,
        caseCollisionLimitation,
        unknownParentOccurrences,
        traversal,
      }),
    };
  } catch (cause: unknown) {
    outcome = { kind: "failed", cause };
  }
  const cleanupFailure = await cleanupInventoryOwners({
    path,
    options,
    rootSource,
    ownedReaders,
    outcome,
  });
  if (cleanupFailure !== undefined) throw cleanupFailure;
  if (outcome.kind === "failed") throw outcome.cause;
  return outcome.snapshot;
};

type InventoryScanOutcome =
  | {
      readonly kind: "completed";
      readonly snapshot: ArtifactInventorySnapshot;
    }
  | { readonly kind: "failed"; readonly cause: unknown };

const cleanupInventoryOwners = async ({
  path,
  options,
  rootSource,
  ownedReaders,
  outcome,
}: {
  readonly path: string;
  readonly options: ArtifactInventoryOptions;
  readonly rootSource: RootInventorySource | undefined;
  readonly ownedReaders: readonly ArtifactReader[];
  readonly outcome: InventoryScanOutcome;
}): Promise<ArtifactReaderFailure | undefined> => {
  let cleanupFailure: ArtifactReaderFailure | undefined;
  const owners: ArtifactResourceOwner[] = [...ownedReaders]
    .reverse()
    .map((reader) => ({
      kind: "reader" as const,
      reader,
      resource: `artifact reader for ${path}`,
    }));
  if (rootSource !== undefined)
    owners.push({
      kind: "file-handle",
      handle: rootSource.owner,
      resource: `root artifact descriptor for ${path}`,
    });
  for (const owner of owners) {
    const attempt = await options.resourceScope.release(owner);
    if (attempt.kind === "released") continue;
    cleanupFailure = options.resourceScope.cleanupFailure(
      cleanupFailure === undefined
        ? outcome
        : { kind: "failed", cause: cleanupFailure },
      [attempt],
      outcome.kind === "completed"
        ? { kind: "artifact-inventory", inventory: outcome.snapshot }
        : undefined,
    );
  }
  return cleanupFailure;
};

interface SnapshotBuildInput {
  readonly resourceScope: ArtifactInventoryOptions["resourceScope"];
  readonly path: string;
  readonly metadata: Stats;
  readonly rootFormat: ArtifactOccurrence["artifact_format"];
  readonly rootDigest: HashResult | null;
  readonly rootSource: RootInventorySource | undefined;
  readonly reader: ArtifactReader | undefined;
  readonly signal: AbortSignal | undefined;
  readonly nodes: Map<string, ArtifactNode>;
  readonly occurrences: MutableOccurrence[];
  readonly pendingContradictions: PendingIntegrityContradiction[];
  readonly caseCollisionLimitation: string | undefined;
  readonly unknownParentOccurrences: ReadonlySet<MutableOccurrence>;
  readonly traversal: ArtifactReaderTraversal;
}

const buildInventorySnapshot = async (
  input: SnapshotBuildInput,
): Promise<ArtifactInventorySnapshot> => {
  const { path, metadata, rootFormat, rootDigest, rootSource, reader, signal } =
    input;
  materializeDirectoryNodes(input.occurrences, input.nodes);
  const rootNode = createRootNode({
    digest: rootDigest,
    occurrences: input.occurrences,
  });
  input.nodes.set(rootNode.artifact_id, rootNode);
  rekeyOccurrences(rootNode.artifact_id, input.occurrences);
  const rootOccurrence = rootOccurrenceFor(rootNode, {
    size: metadata.size,
    executable: hostExecutableMarking(path, metadata),
    format: rootFormat,
    path,
  });
  for (const occurrence of input.occurrences)
    if (
      occurrence.parent_occurrence_id === null &&
      !input.unknownParentOccurrences.has(occurrence)
    )
      occurrence.parent_occurrence_id = rootOccurrence.occurrence_id;
  input.occurrences.unshift(rootOccurrence);

  const occurrenceById = new Map(
    input.occurrences.map((occurrence) => [
      occurrence.occurrence_id,
      occurrence,
    ]),
  );
  const integrityContradictions = buildIntegrityContradictions(
    input.pendingContradictions,
    occurrenceById,
    rootNode,
  );
  const edgeResult = createArtifactEdges(
    rootNode.artifact_id,
    input.occurrences,
    reader?.provenance()[0],
  );
  const { edges } = edgeResult;
  const orderedNodes = sortNodes([...input.nodes.values()]);
  const orderedOccurrences = sortOccurrences(input.occurrences);
  const orderedEdges = sortEdges(edges).map((edge, ordinal) => ({
    ...edge,
    ordinal,
  }));

  await verifyRootDigest(
    path,
    rootDigest,
    rootSource,
    input.resourceScope,
    signal,
  );

  const graphSha256 = artifactGraphDigest({
    nodes: orderedNodes,
    occurrences: orderedOccurrences,
    edges: orderedEdges,
    contradictions: integrityContradictions,
  });
  const manifest = buildManifest({
    rootNode,
    rootFormat,
    graphSha256,
    orderedNodes,
    orderedOccurrences,
    orderedEdges,
  });
  const traversal = inventoryCoverage(rootFormat, reader, input.traversal);
  const coverage =
    edgeResult.limitations.length === 0
      ? traversal.coverage
      : { ...traversal.coverage, status: "unknown" as const };
  return {
    manifest,
    nodes: orderedNodes,
    occurrences: orderedOccurrences,
    edges: orderedEdges,
    provenance: reader?.provenance() ?? [],
    integrity_contradictions: integrityContradictions,
    coverage,
    limitations: buildLimitations(
      [...traversal.limitations, ...edgeResult.limitations],
      integrityContradictions,
      input.caseCollisionLimitation,
    ),
  };
};

const buildIntegrityContradictions = (
  pending: readonly PendingIntegrityContradiction[],
  occurrenceById: ReadonlyMap<string, MutableOccurrence>,
  rootNode: ArtifactNode,
): IntegrityContradiction[] =>
  pending.map((contradiction): IntegrityContradiction => {
    const occurrence = contradiction.occurrence;
    const parent =
      occurrence.parent_occurrence_id === null
        ? undefined
        : occurrenceById.get(occurrence.parent_occurrence_id);
    const parentArtifactId = parent?.artifact_id ?? rootNode.artifact_id;
    return {
      contradiction_id: artifactContradictionId({
        rootArtifactId: rootNode.artifact_id,
        logicalPath: occurrence.logical_path,
        declaredSha256: contradiction.declaredSha256,
        observedSha256: contradiction.observedSha256,
      }),
      occurrence_id: occurrence.occurrence_id,
      parent_artifact_id: parentArtifactId,
      logical_path: occurrence.logical_path,
      declared_sha256: contradiction.declaredSha256,
      observed_sha256: contradiction.observedSha256,
      entry_kind: contradiction.entryKind,
      unpacked: contradiction.unpacked,
      trust: "observed-untrusted",
      provenance: "container-integrity-metadata-versus-observed-bytes",
      limitations: [
        "Observed bytes contradict declared integrity metadata and cannot support equivalence.",
      ],
    };
  });

const verifyRootDigest = async (
  path: string,
  rootDigest: HashResult | null,
  rootSource: RootInventorySource | undefined,
  resourceScope: ArtifactInventoryOptions["resourceScope"],
  signal: AbortSignal | undefined,
): Promise<void> => {
  if (rootDigest === null) return;
  const verified =
    rootSource === undefined
      ? await hashStableRootArtifact(path, resourceScope, signal)
      : await hashStableRootArtifactHandle(
          path,
          rootSource.handle,
          rootSource.initial,
          signal,
        );
  if (
    verified.sha256 !== rootDigest.sha256 ||
    verified.bytes !== rootDigest.bytes
  )
    throw new ArtifactReaderFailure(
      "integrity",
      "Root artifact changed during inventory",
    );
};

const sortNodes = (nodes: ArtifactNode[]): ArtifactNode[] =>
  nodes.sort((left, right) =>
    compareUnicodeCodePoints(left.artifact_id, right.artifact_id),
  );

const sortOccurrences = (
  occurrences: MutableOccurrence[],
): MutableOccurrence[] =>
  occurrences.sort((left, right) =>
    compareUnicodeCodePoints(left.logical_path, right.logical_path),
  );

const sortEdges = <T extends { edge_id: string }>(edges: T[]): T[] =>
  edges.sort((left, right) =>
    compareUnicodeCodePoints(left.edge_id, right.edge_id),
  );

const buildManifest = ({
  rootNode,
  rootFormat,
  graphSha256,
  orderedNodes,
  orderedOccurrences,
  orderedEdges,
}: {
  readonly rootNode: ArtifactNode;
  readonly rootFormat: ArtifactOccurrence["artifact_format"];
  readonly graphSha256: string;
  readonly orderedNodes: readonly ArtifactNode[];
  readonly orderedOccurrences: readonly unknown[];
  readonly orderedEdges: readonly { edge_id: string }[];
}): ArtifactInventoryResult["manifest"] =>
  artifactInventoryResultSchema.shape.manifest.parse({
    manifest_id: artifactManifestId(rootNode.artifact_id, graphSha256),
    root_artifact_id: rootNode.artifact_id,
    root_sha256: rootNode.sha256,
    root_format: rootFormat,
    graph_sha256: graphSha256,
    node_count: orderedNodes.length,
    occurrence_count: orderedOccurrences.length,
    edge_count: orderedEdges.length,
  });

const buildLimitations = (
  scopeLimitations: readonly string[],
  integrityContradictions: readonly IntegrityContradiction[],
  caseCollisionLimitation: string | undefined,
): string[] => [
  ...scopeLimitations,
  ...(caseCollisionLimitation === undefined ? [] : [caseCollisionLimitation]),
  ...(integrityContradictions.length === 0
    ? []
    : [
        `${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`,
      ]),
];