diff --git a/docs/specs/README.md b/docs/specs/README.md index 2b1d424f2c..4494e192e1 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -26,6 +26,7 @@ for new work, roadmap granularity and producer-owned contracts. - [Native call source coordinates](native/call-source-locations/spec.md) ([plan](native/call-source-locations/plan.md), [tasks](native/call-source-locations/tasks.md), [verification](native/call-source-locations/verification.md)) - [Go static string-header literal recovery](native/go-string-literals/spec.md) ([plan](native/go-string-literals/plan.md), [tasks](native/go-string-literals/tasks.md)) - [Windows Shell Link shortcut inspection](native/windows-shortcut/spec.md) ([plan](native/windows-shortcut/plan.md), [tasks](native/windows-shortcut/tasks.md)) +- [NTFS $MFT and USN Journal inspection](native/ntfs-mft/spec.md) ([plan](native/ntfs-mft/plan.md), [tasks](native/ntfs-mft/tasks.md)) - [Solidity terminal metadata inspection](solidity/cbor-metadata/spec.md) ([plan](solidity/cbor-metadata/plan.md), [tasks](solidity/cbor-metadata/tasks.md), [verification](solidity/cbor-metadata/verification.md)) - [Inspect a V8 code cache container](v8/code-cache-inspection/spec.md) ([plan](v8/code-cache-inspection/plan.md), [tasks](v8/code-cache-inspection/tasks.md)) - [Deno compile executable inventory and extraction](artifacts/deno-standalone/spec.md) ([plan](artifacts/deno-standalone/plan.md), [tasks](artifacts/deno-standalone/tasks.md)) diff --git a/docs/specs/native/ntfs-mft/plan.md b/docs/specs/native/ntfs-mft/plan.md new file mode 100644 index 0000000000..83f03c27ab --- /dev/null +++ b/docs/specs/native/ntfs-mft/plan.md @@ -0,0 +1,39 @@ +# Implementation plan + +## Boundary and contract + +Follow the existing artifact readers. `src/domain/windows/ntfsMft.ts` owns FILE +record bytes and coverage; `src/domain/artifactFormatMetadata.ts` owns the +`ntfs-mft` result schema. `NtfsMftArtifactReader` reads the admitted file +through `ArtifactFileSource` and exposes its facts through `formatMetadata()`. +A record stream is its own content, so the root occurrence carries every fact +and no members are enumerated. + +Recognition lives in `hasMftFileRecordSignature` and is applied in +`classifyArtifactBytes`, next to the other unambiguous magic probes. + +## Incremental work and validation + +1. Define the first-step contract, limits and observations; add parser fixtures. +2. Implement the FILE record header and update-sequence fixup parse with + explicit limitations. +3. Add the reader, format descriptor and byte classification. +4. Decode attribute records, then `$FILE_NAME` attributes so a record's parent + reference, names and timestamps become explicit; keep hard links separate. +5. Parse `$UsnJrnl:$J` records and correlate them to records by full file + reference, including sequence numbers, reporting unresolved references. +6. Document the workflow in a native Windows artifact guide. + +## Effects and compatibility + +The workflow reads one caller-selected local file. It adds no target execution, +injection, mutation, protection recovery, network access, volume mount or live +filesystem access, so no maintainer scope decision is required. +`inspect_artifact` routing for every existing format is unchanged, and the new +format id appears only for bytes whose first record carries the `FILE` +signature. + +## References + +- https://learn.microsoft.com/en-us/windows/win32/fileio/master-file-table +- https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc diff --git a/docs/specs/native/ntfs-mft/spec.md b/docs/specs/native/ntfs-mft/spec.md new file mode 100644 index 0000000000..40c13c37c2 --- /dev/null +++ b/docs/specs/native/ntfs-mft/spec.md @@ -0,0 +1,48 @@ +# NTFS $MFT and USN Journal inspection + +This specification records an implementation increment and its original problem +statement. For current behavior and limitations, see the artifact inventory +guide. + +## Problem and outcome + +REA has no reader for acquired NTFS metadata artifacts. An analyst holding a +captured `$MFT` stream cannot ask which FILE records it contains, whether their +update-sequence fixups survived acquisition, or which records are directories, +extensions of another record, or no longer in use. Add offline, read-only +inventory of a selected `$MFT` file, delivered in steps so each one merges on +its own. + +## Requirements + +- Recognize a `$MFT` stream by the `FILE` signature of its first record, under + any file name, and route it to a dedicated artifact reader. +- Validate each FILE record header: the `FILE` signature, sequence number, flags, + used and total byte counts, base record reference, and the update-sequence + array's offset, count and array bounds. +- Verify the update-sequence fixups: every sector trailer the array covers must + hold the array's sequence number. A mismatch keeps the record's header facts + and is reported per record and in the metadata count. +- Report a missing, misplaced or short update-sequence array as unverified + rather than as a clean record, and a trailing partial record as a byte count. +- Stop at the first record that does not begin with the `FILE` signature and + name its offset, rather than guessing at the remaining bytes. +- Bound the whole file at a fixed 512 MiB ceiling and report the ceiling as a + `limit` failure with its boundary. +- Parse only the records' headers and fixups. Attributes, filenames, data + streams and the `$UsnJrnl:$J` journal are not decoded in this step, so no path, + file size, file reference correlation or change history is claimed. + +## Acceptance + +Synthesized fixtures cover a clean record, a failed fixup, a missing update +sequence array, an array shorter than the record's sector count, extension and +directory records, a record whose signature is wrong mid-stream, a trailing +partial record, an undeclared record size, a 4 KiB record, a stream shorter than +one header, bytes without the `FILE` signature, and the file-size ceiling. +Inventory tests cover the root occurrence's metadata and the coverage status +when a fixup fails, and classification tests cover several file names for the +same bytes. + +Not covered in this step: real `$MFT` files written by a Windows host, USN +journal records, attribute decoding, and any correlation between the two. diff --git a/docs/specs/native/ntfs-mft/tasks.md b/docs/specs/native/ntfs-mft/tasks.md new file mode 100644 index 0000000000..36e2b8eb36 --- /dev/null +++ b/docs/specs/native/ntfs-mft/tasks.md @@ -0,0 +1,11 @@ +# Tasks + +- [x] Define the first-step scope, contract and limits. +- [x] Parse FILE record headers and update-sequence fixups with synthesized + fixtures and explicit limitations. +- [x] Add the reader, format descriptor and byte classification. +- [ ] Decode attribute records, then `$FILE_NAME` attributes with parent + references, names, hard links and timestamps. +- [ ] Parse `$UsnJrnl:$J` records and correlate them to records by full file + reference. +- [ ] Verify against real `$MFT` and journal captures from a Windows host. diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 422b940be0..30af6d1af4 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -215,6 +215,7 @@ export interface ArtifactReader { | "dfuse" | "esp-flash-image" | "fit" + | "ntfs-mft" | "file"; entries( signal?: AbortSignal, diff --git a/src/artifacts/NtfsMftArtifactReader.test.ts b/src/artifacts/NtfsMftArtifactReader.test.ts new file mode 100644 index 0000000000..64a69f9420 --- /dev/null +++ b/src/artifacts/NtfsMftArtifactReader.test.ts @@ -0,0 +1,77 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { MFT_DEFAULT_RECORD_BYTES } from "../domain/windows/ntfsMft.js"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { scanCanonicalArtifactInventory } from "../../tests/fixtures/artifactInventory.js"; + +/** A 1 KiB FILE record whose update sequence array covers both sectors. */ +const recordBytes = (options: { + readonly sequenceNumber?: number; + readonly flags?: number; +}): Buffer => { + const record = Buffer.alloc(MFT_DEFAULT_RECORD_BYTES); + record.write("FILE", 0, "latin1"); + const sequenceNumber = options.sequenceNumber ?? 1; + record.writeUInt16LE(48, 4); + record.writeUInt16LE(3, 6); + record.writeUInt16LE(sequenceNumber, 48); + record.writeUInt16LE(0xa5a5, 50); + record.writeUInt16LE(0xa5a5, 52); + record.writeUInt16LE(sequenceNumber, 16); + record.writeUInt16LE(1, 18); + record.writeUInt16LE(56, 20); + record.writeUInt16LE(options.flags ?? 0x0001, 22); + record.writeUInt32LE(56, 24); + record.writeUInt32LE(MFT_DEFAULT_RECORD_BYTES, 28); + record.writeUInt16LE(sequenceNumber, 510); + record.writeUInt16LE(sequenceNumber, 1022); + return record; +}; + +const writeMft = async (name: string, bytes: Buffer): Promise => { + const directory = await createTestTempDirectory("rea-ntfs-mft-"); + const path = join(directory, name); + await writeFile(path, bytes); + return path; +}; + +it("reports a $MFT stream's record headers as the root occurrence", async () => { + const inventory = await scanCanonicalArtifactInventory( + await writeMft("$MFT", Buffer.concat([recordBytes({}), recordBytes({ flags: 0x0003 })])), + ); + expect(inventory.manifest.root_format).toBe("ntfs-mft"); + expect(inventory.coverage.status).toBe("complete"); + // A record stream declares no members, so the root is its whole inventory. + expect(inventory.occurrences.map((row) => row.logical_path)).toEqual(["."]); + expect(inventory.occurrences[0]).toMatchObject({ + artifact_format: "ntfs-mft", + format_metadata: { + kind: "ntfs-mft", + record_bytes: MFT_DEFAULT_RECORD_BYTES, + record_count: 2, + in_use_record_count: 2, + directory_record_count: 1, + extension_record_count: 0, + fixup_mismatch_count: 0, + fixup_unverified_record_count: 0, + truncated_tail_bytes: 0, + }, + }); +}); + +it("names a failed fixup in the coverage limitations", async () => { + const record = recordBytes({ sequenceNumber: 4 }); + // A trailer that is not the sequence number marks the sector as not cleanly written. + record.writeUInt16LE(0x1234, 510); + const inventory = await scanCanonicalArtifactInventory( + await writeMft("$MFT", record), + ); + expect(inventory.manifest.root_format).toBe("ntfs-mft"); + expect(inventory.coverage.status).toBe("unknown"); + expect(inventory.limitations).toContainEqual( + expect.stringMatching(/sector trailer that is not the update sequence/u), + ); +}); diff --git a/src/artifacts/NtfsMftArtifactReader.ts b/src/artifacts/NtfsMftArtifactReader.ts new file mode 100644 index 0000000000..594946cf57 --- /dev/null +++ b/src/artifacts/NtfsMftArtifactReader.ts @@ -0,0 +1,143 @@ +import type { Readable } from "node:stream"; + +import type { ArtifactFormatMetadata } from "../domain/artifactFormatMetadata.js"; +import type { RegionDecodeFailure } from "../domain/firmware/regionDecoder.js"; +import { + createMftDecoder, + type MftImage, +} from "../domain/windows/ntfsMft.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, + type ArtifactReader, + type ArtifactReaderTraversal, + type ArtifactTraversalOptions, +} from "./ArtifactReader.js"; +import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; +import { assertArtifactReadActive } from "./ArtifactFileOperation.js"; +import { ArtifactFileSource } from "./ArtifactFileSource.js"; +import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js"; +import type { StableRegularFileDescriptor } from "../filesystem/RegularFile.js"; + +/** + * An MFT stream is its own content: the FILE records describe the source + * volume, not members inside the stream, so the root occurrence carries every + * fact and no entries are enumerated. + */ +export class NtfsMftArtifactReader implements ArtifactReader { + readonly format = "ntfs-mft" as const; + readonly #scope = new ArtifactResourceScope(); + readonly #source: ArtifactFileSource; + #prepared: Promise | undefined; + #metadata: ArtifactFormatMetadata | undefined; + #limitations: readonly string[] = []; + #closing = false; + + constructor(path: string, admitted?: StableRegularFileDescriptor) { + this.#source = new ArtifactFileSource(path, "$MFT stream", admitted); + } + + formatMetadata(): ArtifactFormatMetadata | undefined { + return this.#metadata === undefined ? undefined : { ...this.#metadata }; + } + + async prepareContainer( + expectedSha256: string, + signal?: AbortSignal, + ): Promise { + await this.#scope.run(async () => { + this.#assertOpen(signal); + await this.#source.bindSha256(expectedSha256, signal); + }); + } + + async *entries( + signal?: AbortSignal, + options?: ArtifactTraversalOptions, + ): AsyncIterable { + await this.#scope.run(async () => { + this.#assertOpen(signal); + this.#prepared ??= this.#prepare(signal); + await this.#prepared; + }); + // An MFT declares no members, so the root occurrence is the whole + // inventory; the coverage callback reports what its parse did not establish. + options?.onCoverage?.(coverageOf(this.#limitations)); + } + + async open(entry: ArtifactEntry, _signal?: AbortSignal): Promise { + return this.#scope.run(async () => { + throw new ArtifactReaderFailure( + "unavailable", + `$MFT stream declares no member to open for ${entry.path}`, + ); + }); + } + + async close(): Promise { + this.#closing = true; + await this.#scope.close(); + await this.#source.close(); + this.#metadata = undefined; + } + + provenance(): readonly [] { + return []; + } + + #assertOpen(signal?: AbortSignal): void { + assertArtifactReadActive(signal); + if (this.#closing) + throw new ArtifactReaderFailure( + "unavailable", + "$MFT stream reader is closed", + ); + } + + async #prepare(signal?: AbortSignal): Promise { + const image = await this.#decode(signal); + this.#metadata = image.metadata; + this.#limitations = image.limitations; + await this.#source.read(signal); + } + + async #decode(signal?: AbortSignal): Promise { + const { handle, initial } = await this.#source.read(signal); + const decoder = createMftDecoder(initial.size); + if (!decoder.ok) throw this.#failure(decoder.error); + if (initial.size > 0) + for await (const chunk of readFileHandleChunks(handle, { + end: initial.size - 1, + signal, + cancellationFailure: (reason) => + new ArtifactReaderFailure("cancelled", "$MFT stream read cancelled", { + cause: reason, + }), + }) as AsyncIterable) { + this.#assertOpen(signal); + const written = decoder.value.write(chunk); + if (!written.ok) throw this.#failure(written.error); + } + const image = decoder.value.finish(); + if (!image.ok) throw this.#failure(image.error); + return image.value; + } + + #failure(failure: RegionDecodeFailure): ArtifactReaderFailure { + return failure.kind === "format" + ? new ArtifactReaderFailure( + "format", + `$MFT stream is malformed: ${failure.message}`, + ) + : new ArtifactReaderFailure( + "limit", + `$MFT stream exceeds a fixed limit: ${failure.message}`, + failure.boundary === undefined ? {} : { boundary: failure.boundary }, + ); + } +} + +const coverageOf = (limitations: readonly string[]): ArtifactReaderTraversal => + limitations.length === 0 + ? { status: "complete", limitations: [] } + : { status: "unknown", limitations }; diff --git a/src/artifacts/artifactFormats.test.ts b/src/artifacts/artifactFormats.test.ts index 08143531c4..3dc0afa291 100644 --- a/src/artifacts/artifactFormats.test.ts +++ b/src/artifacts/artifactFormats.test.ts @@ -43,7 +43,8 @@ type FormatsWithReaders = | "uf2" | "dfuse" | "esp-flash-image" - | "fit"; + | "fit" + | "ntfs-mft"; const readerFormatsMatch: Equal = true; diff --git a/src/artifacts/artifactFormats.ts b/src/artifacts/artifactFormats.ts index 29a9222994..d046aa5342 100644 --- a/src/artifacts/artifactFormats.ts +++ b/src/artifacts/artifactFormats.ts @@ -40,6 +40,7 @@ import { NodeSeaArtifactReader } from "./NodeSeaArtifactReader.js"; import type { PyInstallerAdmission } from "./pyinstaller/PyInstallerAdmission.js"; import { PyInstallerArtifactReader } from "./PyInstallerArtifactReader.js"; import { Uf2ArtifactReader } from "./Uf2ArtifactReader.js"; +import { NtfsMftArtifactReader } from "./NtfsMftArtifactReader.js"; import { UImageArtifactReader } from "./UImageArtifactReader.js"; import { UnityFsArtifactReader } from "./UnityFsArtifactReader.js"; import { WxapkgArtifactReader } from "./WxapkgArtifactReader.js"; @@ -465,6 +466,14 @@ const NON_ZIP_FORMATS = { createReader: (path, options) => new FitArtifactReader(path, sourceOf(options)), }), + "ntfs-mft": described("ntfs-mft", { + ...plain, + retainClassifiedSource: true, + readerLimitation: + "MFT inventory reports each FILE record's header, flags, record extent and update-sequence fixup result. Attributes, filenames, data streams and the $UsnJrnl:$J journal are not decoded, so no path, file size or change history is established.", + createReader: (path, options) => + new NtfsMftArtifactReader(path, sourceOf(options)), + }), unknown: described("unknown", plain), } satisfies Record< Exclude, diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts index 33a19cfa3e..f475e0c105 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -576,6 +576,16 @@ it("classifies nested UF2 by block magics and DfuSe by prefix and suffix", () => ).toMatchObject({ format: "file" }); }); +it("classifies a $MFT stream by its FILE record signature, under any name", () => { + const mft = Buffer.alloc(2_048); + mft.write("FILE", 0, "latin1"); + for (const path of ["evidence/$MFT", "evidence/mft.raw", "evidence/records"]) + expect(classifyArtifactBytesAtPath(path, mft)).toEqual({ + kind: "executable", + format: "ntfs-mft", + }); +}); + it("classifies a FIT by a /images node within the first 4 KiB", () => { const images = [{ name: "kernel", data: Buffer.from("kernel") }]; const late = devicetreeFixture({ diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.ts b/src/artifacts/inventory/ArtifactGraphConstruction.ts index 27561c6772..1ef1bb46f3 100644 --- a/src/artifacts/inventory/ArtifactGraphConstruction.ts +++ b/src/artifacts/inventory/ArtifactGraphConstruction.ts @@ -11,6 +11,7 @@ import { recognizeEspImage } from "../../domain/firmware/espImage.js"; import { hasUf2Block } from "../../domain/firmware/uf2.js"; import { isDfuSeFile } from "../../domain/firmware/dfuse.js"; import { hasFitImagesNode } from "../../domain/firmware/fitImage.js"; +import { hasMftFileRecordSignature } from "../../domain/windows/ntfsMft.js"; import type { ArtifactClassificationEvidence } from "../ArtifactClassificationProbe.js"; import { asarHeaderAdmissionFailure } from "../AsarHeader.js"; import { hasDestinationPath } from "../ArtifactPaths.js"; @@ -622,6 +623,7 @@ export const classifyArtifactBytes = ( return "dfuse"; if (recognizeEspImage(prefix, fileSize) === "esp-image") return "esp-image"; if (hasFitImagesNode(prefix)) return "fit"; + if (hasMftFileRecordSignature(prefix)) return "ntfs-mft"; if (hasBinaryPlistSignature(prefix)) return "plist"; if (hasXmlPlistSignature(prefix)) return "plist"; const fatFormat = classifyFatArtifact(evidence, fileSize); diff --git a/src/domain/artifactFormatMetadata.ts b/src/domain/artifactFormatMetadata.ts index 76ae75c976..0653b0cb7d 100644 --- a/src/domain/artifactFormatMetadata.ts +++ b/src/domain/artifactFormatMetadata.ts @@ -299,6 +299,19 @@ export const hexRecordRegionMetadataSchema = z.strictObject({ last_line: offset, }); +/** NTFS $MFT record-header facts; attributes, names and the USN journal are later steps. */ +export const ntfsMftMetadataSchema = z.strictObject({ + kind: z.literal("ntfs-mft"), + record_bytes: offset, + record_count: offset, + in_use_record_count: offset, + directory_record_count: offset, + extension_record_count: offset, + fixup_mismatch_count: offset, + fixup_unverified_record_count: offset, + truncated_tail_bytes: offset, +}); + /** * U-Boot FIT root facts. Configuration references are declared names; a * reference that names no image is reported as a limitation, not removed. @@ -886,6 +899,7 @@ export const artifactFormatMetadataSchema = z.discriminatedUnion("kind", [ espImageSignatureBlockMetadataSchema, uf2MetadataSchema, uf2RegionMetadataSchema, + ntfsMftMetadataSchema, dfuSeMetadataSchema, dfuSeElementMetadataSchema, espFlashImageMetadataSchema, @@ -1036,6 +1050,10 @@ const declarationsByKind: { notCompared: "Its familyID and addresses are in its path, family_name is REA's label for the familyID, and its block counts and positions are layout; content comparison covers its bytes.", }, + "ntfs-mft": { + notCompared: + "Its fields come from the record headers in the file's own FILE records, which content comparison covers.", + }, dfuse: { notCompared: "Its fields come from the file's own prefix, target headers and suffix, which content comparison covers.", diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index f9b9728aad..83aa8741f9 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -103,6 +103,7 @@ export const ARTIFACT_FORMAT_IDS = [ "dfuse", "esp-flash-image", "fit", + "ntfs-mft", "unknown", ] as const; @@ -124,6 +125,7 @@ const artifactContentFormatSchema = artifactFormatSchema.extract([ "uf2", "dfuse", "fit", + "ntfs-mft", "mach-o-universal", "mach-o", "elf", diff --git a/src/domain/evidenceSubject.ts b/src/domain/evidenceSubject.ts index c336774249..5580bd3d2c 100644 --- a/src/domain/evidenceSubject.ts +++ b/src/domain/evidenceSubject.ts @@ -52,6 +52,7 @@ export const evidenceSubjectFormatSchema = z.enum([ "dfuse", "esp-flash-image", "fit", + "ntfs-mft", ]); /** Format-owned architecture declarations shared by Evidence and persisted executions. */ diff --git a/src/domain/windows/ntfsMft.test.ts b/src/domain/windows/ntfsMft.test.ts new file mode 100644 index 0000000000..4dd0c2ce6e --- /dev/null +++ b/src/domain/windows/ntfsMft.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from "vitest"; + +import { + MFT_DEFAULT_RECORD_BYTES, + MFT_MAX_FILE_BYTES, + createMftDecoder, + parseMftImage, +} from "./ntfsMft.js"; + +interface RecordOptions { + readonly sequenceNumber?: number; + readonly hardLinkCount?: number; + readonly flags?: number; + readonly baseRecordReference?: bigint; + readonly recordBytes?: number; + /** Write a wrong trailer value into this sector, leaving the others correct. */ + readonly corruptSector?: number; + /** Truncate the update sequence array to this many entries; 0 omits it. */ + readonly usaEntries?: number; +} + +/** One FILE record with a zeroed body and fixups applied the way NTFS writes them. */ +const recordOf = (options: RecordOptions = {}): Uint8Array => { + const recordBytes = options.recordBytes ?? MFT_DEFAULT_RECORD_BYTES; + const record = new Uint8Array(recordBytes); + const view = new DataView(record.buffer); + record.set(Buffer.from("FILE", "latin1"), 0); + const sequenceNumber = options.sequenceNumber ?? 1; + const sectors = recordBytes / 512; + + const usaEntries = options.usaEntries ?? sectors + 1; + if (usaEntries > 0) { + const usaOffset = 48; + view.setUint16(4, usaOffset, true); + view.setUint16(6, usaEntries, true); + view.setUint16(usaOffset, sequenceNumber, true); + for (let sector = 0; sector < sectors; sector += 1) { + const trailer = sector * 512 + 510; + // A saved trailer is what the sector held before the fixup replaced it. + view.setUint16(usaOffset + 2 * (sector + 1), 0xa5a5, true); + view.setUint16( + trailer, + sector >= usaEntries - 1 + ? 0xa5a5 + : sector === options.corruptSector + ? 0x1234 + : sequenceNumber, + true, + ); + } + } else { + view.setUint16(4, 0, true); + view.setUint16(6, 0, true); + } + + view.setUint16(16, sequenceNumber, true); + view.setUint16(18, options.hardLinkCount ?? 1, true); + view.setUint16(20, 56, true); + view.setUint16(22, options.flags ?? 0x0001, true); + view.setUint32(24, 56, true); + view.setUint32(28, recordBytes, true); + view.setBigUint64(32, options.baseRecordReference ?? 0n, true); + view.setUint16(40, 0, true); + return record; +}; + +const streamOf = (...records: readonly Uint8Array[]): Uint8Array => { + const total = records.reduce((sum, record) => sum + record.length, 0); + const stream = new Uint8Array(total); + let offset = 0; + for (const record of records) { + stream.set(record, offset); + offset += record.length; + } + return stream; +}; + +describe("parseMftImage", () => { + it("inventories a clean record's header and verified fixups", () => { + const parsed = parseMftImage(streamOf(recordOf({ sequenceNumber: 5 }))); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.metadata).toEqual({ + kind: "ntfs-mft", + record_bytes: MFT_DEFAULT_RECORD_BYTES, + record_count: 1, + in_use_record_count: 1, + directory_record_count: 0, + extension_record_count: 0, + fixup_mismatch_count: 0, + fixup_unverified_record_count: 0, + truncated_tail_bytes: 0, + }); + expect(parsed.value.records[0]).toMatchObject({ + index: 0, + fileOffset: 0, + sequenceNumber: 5, + inUse: true, + directory: false, + baseRecordReference: null, + firstAttributeOffset: 56, + fixups: { + sequenceNumber: 5, + verifiedSectors: 2, + coveredSectors: 2, + mismatchSector: null, + malformed: false, + }, + }); + expect(parsed.value.limitations).toEqual([]); + }); + + it("keeps a record whose sector trailer is not the sequence number, and reports it", () => { + const parsed = parseMftImage( + streamOf( + recordOf(), + recordOf({ sequenceNumber: 9, corruptSector: 1 }), + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.records[1]?.fixups).toMatchObject({ + sequenceNumber: 9, + verifiedSectors: 1, + mismatchSector: 1, + }); + expect(parsed.value.metadata.fixup_mismatch_count).toBe(1); + expect(parsed.value.limitations.join(" ")).toMatch( + /1 record\(s\) hold a sector trailer/u, + ); + }); + + it("treats a missing update sequence array as unverified rather than clean", () => { + const parsed = parseMftImage(streamOf(recordOf({ usaEntries: 0 }))); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.records[0]?.fixups).toMatchObject({ + malformed: true, + mismatchSector: null, + coveredSectors: 0, + }); + expect(parsed.value.metadata.fixup_mismatch_count).toBe(0); + expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1); + expect(parsed.value.limitations.join(" ")).toMatch( + /update sequence array that does not cover every sector/u, + ); + }); + + it("reports an array that covers fewer sectors than the record has", () => { + // A 4 KiB record holds eight sectors; a three-entry array leaves five unverified. + const parsed = parseMftImage( + streamOf(recordOf({ recordBytes: 4096, usaEntries: 3 })), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.records[0]?.fixups).toMatchObject({ + coveredSectors: 2, + verifiedSectors: 2, + mismatchSector: null, + malformed: false, + }); + expect(parsed.value.metadata.fixup_unverified_record_count).toBe(1); + }); + + it("distinguishes extension records from base records", () => { + const parsed = parseMftImage( + streamOf( + recordOf(), + recordOf({ flags: 0x0005, baseRecordReference: 0x0000000000000001n }), + ), + ); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.records[1]).toMatchObject({ + baseRecordReference: 1, + flags: { raw: 0x0005, names: ["in-use", "extension"] }, + }); + expect(parsed.value.metadata.extension_record_count).toBe(1); + }); + + it("names directory records without losing the in-use flag", () => { + const parsed = parseMftImage(streamOf(recordOf({ flags: 0x0003 }))); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.records[0]).toMatchObject({ + directory: true, + inUse: true, + flags: { names: ["in-use", "directory"] }, + }); + expect(parsed.value.metadata.directory_record_count).toBe(1); + }); + + it("stops at a record that does not begin with the FILE signature", () => { + const stream = streamOf(recordOf(), recordOf(), recordOf()); + stream.set(Buffer.from("BAAD", "latin1"), 1024); + const parsed = parseMftImage(stream); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.metadata.record_count).toBe(1); + expect(parsed.value.limitations.join(" ")).toMatch( + /record at 1024 does not begin with the FILE signature/u, + ); + }); + + it("keeps a trailing partial record out of the inventory", () => { + const stream = new Uint8Array(1024 + 300); + stream.set(recordOf(), 0); + const parsed = parseMftImage(stream); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.metadata.record_count).toBe(1); + expect(parsed.value.metadata.truncated_tail_bytes).toBe(300); + expect(parsed.value.limitations.join(" ")).toMatch( + /300 trailing byte\(s\)/u, + ); + }); + + it("falls back to the default record size when record 0 declares none", () => { + const record = recordOf(); + new DataView(record.buffer).setUint32(28, 0, true); + const parsed = parseMftImage(record); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.recordBytes).toBe(MFT_DEFAULT_RECORD_BYTES); + expect(parsed.value.limitations.join(" ")).toMatch( + /not a supported power of two/u, + ); + }); + + it("reads a 4 KiB record at its declared size", () => { + const parsed = parseMftImage(streamOf(recordOf({ recordBytes: 4096 }))); + expect(parsed.ok).toBe(true); + if (!parsed.ok) return; + expect(parsed.value.recordBytes).toBe(4096); + expect(parsed.value.metadata.record_count).toBe(1); + }); + +}); + +describe("parseMftImage refusals and limits", () => { + it("refuses a file shorter than one record header", () => { + const parsed = parseMftImage(new Uint8Array(32)); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.message).toMatch(/shorter than one 48-byte record header/u); + }); + + it("refuses bytes that do not start with the FILE signature", () => { + const parsed = parseMftImage(Buffer.alloc(2048, 0x41)); + expect(parsed.ok).toBe(false); + if (parsed.ok) return; + expect(parsed.error.message).toMatch(/does not begin with the FILE signature/u); + }); + + it("refuses a file above the fixed ceiling", () => { + const decoder = createMftDecoder(MFT_MAX_FILE_BYTES + 1); + expect(decoder.ok).toBe(false); + if (decoder.ok) return; + expect(decoder.error).toMatchObject({ kind: "limit", boundary: "file-size" }); + }); + + it("decodes the same stream through the chunked decoder", () => { + const stream = streamOf(recordOf(), recordOf({ flags: 0x0003 })); + const decoder = createMftDecoder(stream.length); + expect(decoder.ok).toBe(true); + if (!decoder.ok) return; + // Feed one byte past a record boundary so a chunk never aligns with a record. + for (const chunk of [stream.subarray(0, 1025), stream.subarray(1025)]) + expect(decoder.value.write(chunk).ok).toBe(true); + const image = decoder.value.finish(); + expect(image.ok).toBe(true); + if (!image.ok) return; + expect(image.value.metadata.record_count).toBe(2); + expect(image.value.metadata.directory_record_count).toBe(1); + }); +}); diff --git a/src/domain/windows/ntfsMft.ts b/src/domain/windows/ntfsMft.ts new file mode 100644 index 0000000000..150c0342a3 --- /dev/null +++ b/src/domain/windows/ntfsMft.ts @@ -0,0 +1,385 @@ +import { err, ok, type Result } from "../result.js"; +import type { + RegionDecodeFailure, + RegionDecoder, +} from "../firmware/regionDecoder.js"; + +/** An NTFS FILE record begins with this signature. */ +export const MFT_FILE_RECORD_MAGIC = "FILE"; +export const MFT_FILE_RECORD_MAGIC_BYTES = 4; +/** The shortest header worth reading: a FILE record header ends at `nextAttributeId`. */ +export const NTFS_MFT_MIN_RECORD_BYTES = 48; + +/** + * The `FILE` signature at file offset 0. An MFT stream is a run of fixed-size + * FILE records, so the first record's signature is the recognition fact; + * whether every record parses is the reader's question, not classification's. + */ +export const hasMftFileRecordSignature = (prefix: Uint8Array): boolean => + prefix.length >= MFT_FILE_RECORD_MAGIC_BYTES && + text(prefix.subarray(0, MFT_FILE_RECORD_MAGIC_BYTES)) === + MFT_FILE_RECORD_MAGIC; + +/** A record's declared size is a power of two from 512 upward; 1024 is NTFS's default. */ +export const MFT_DEFAULT_RECORD_BYTES = 1024; +const MFT_MIN_RECORD_BYTES = 512; +const MFT_MAX_RECORD_BYTES = 65536; +/** An update sequence array holds at least the sequence number and one fixup word. */ +const USA_MIN_ENTRIES = 2; +/** NTFS fixes up one 512-byte sector at a time. */ +const FIXUP_SECTOR_BYTES = 512; +/** + * Fixed whole-file ceiling. Records are inventoried from their headers only; + * attributes and their payloads are not decoded, so this is deliberately + * smaller than a volume's MFT can be. + */ +export const MFT_MAX_FILE_BYTES = 512 * 1024 * 1024; + +/** FILE record header fields; offsets are from the record start. */ +const RECORD_HEADER = { + updateSequenceOffset: 4, + updateSequenceCount: 6, + logFileSequenceNumber: 8, + sequenceNumber: 16, + hardLinkCount: 18, + firstAttributeOffset: 20, + flags: 22, + usedBytes: 24, + totalBytes: 28, + baseRecordReference: 32, + nextAttributeId: 40, + recordNumber: 44, +} as const; +const RECORD_HEADER_BYTES = NTFS_MFT_MIN_RECORD_BYTES; + +const FILE_RECORD_IN_USE = 0x0001; +const FILE_RECORD_IS_DIRECTORY = 0x0002; +const FILE_RECORD_NAMES = new Map([ + [FILE_RECORD_IN_USE, "in-use"], + [FILE_RECORD_IS_DIRECTORY, "directory"], + [0x0004, "extension"], + [0x0008, "view-index"], +]); + +export interface MftRecordFixups { + /** The update sequence number the array's first entry holds, or 0 when absent. */ + readonly sequenceNumber: number; + readonly entries: number; + /** Sectors whose trailer held the sequence number before a mismatch. */ + readonly verifiedSectors: number; + /** Sectors the array covers; fewer than the record's sectors leaves a tail unverified. */ + readonly coveredSectors: number; + /** The first sector whose trailer did not hold the sequence number, if any. */ + readonly mismatchSector: number | null; + /** The array itself is missing, misplaced or too short to cover one sector. */ + readonly malformed: boolean; +} + +export interface MftRecord { + /** Zero-based ordinal of this record in the stream. */ + readonly index: number; + /** File offset where the record starts. */ + readonly fileOffset: number; + readonly sequenceNumber: number; + /** Hard link count as recorded; a zero count marks a record with no names. */ + readonly hardLinkCount: number; + readonly flags: { readonly raw: number; readonly names: readonly string[] }; + readonly inUse: boolean; + readonly directory: boolean; + /** Bytes the record claims it uses, and the record size it was parsed at. */ + readonly usedBytes: number; + readonly totalBytes: number; + /** The base record this is an extension of, or `null` for a base record. */ + readonly baseRecordReference: number | null; + /** The record number the header declares; `null` when the header predates the field. */ + readonly declaredRecordNumber: number | null; + /** `$LogFile` sequence number as recorded; no log is read. */ + readonly logFileSequenceNumber: number; + /** Declared offset of the first attribute; no attribute decoding follows yet. */ + readonly firstAttributeOffset: number; + /** Declared id of the next attribute; no attribute decoding follows yet. */ + readonly nextAttributeId: number; + readonly fixups: MftRecordFixups; +} + +export interface MftImage { + readonly metadata: MftMetadata; + /** The record size the stream declared, or the fixed default when it did not. */ + readonly recordBytes: number; + readonly records: readonly MftRecord[]; + readonly limitations: readonly string[]; +} + +export interface MftMetadata { + readonly kind: "ntfs-mft"; + readonly record_bytes: number; + readonly record_count: number; + readonly in_use_record_count: number; + readonly directory_record_count: number; + readonly extension_record_count: number; + readonly fixup_mismatch_count: number; + readonly fixup_unverified_record_count: number; + readonly truncated_tail_bytes: number; +} + +const text = (ascii: Uint8Array): string => + Buffer.from(ascii).toString("latin1"); + +const failure = ( + message: string, + boundary?: RegionDecodeFailure["boundary"], +): RegionDecodeFailure => ({ + kind: "format", + message, + ...(boundary === undefined ? {} : { boundary }), +}); + +const limit = ( + message: string, + boundary?: RegionDecodeFailure["boundary"], +): RegionDecodeFailure => ({ + kind: "limit", + message, + ...(boundary === undefined ? {} : { boundary }), +}); + +const namesOf = (raw: number): readonly string[] => { + const names: string[] = []; + for (const [bit, name] of FILE_RECORD_NAMES) + if ((raw & bit) !== 0) names.push(name); + return names; +}; + +const isRecordSize = (bytes: number): boolean => + bytes >= MFT_MIN_RECORD_BYTES && + bytes <= MFT_MAX_RECORD_BYTES && + (bytes & (bytes - 1)) === 0; + +/** + * Parse FILE record headers and verify their update-sequence fixups. + * + * Only the record structure is established: attributes, their names, the + * filenames and the USN journal are left to later steps, so no path, timestamp + * or file identity beyond the record itself is claimed. A record whose fixup + * does not match keeps its header facts, because the stream still declares it, + * and the mismatch is reported on the record and counted in the metadata. + */ +export const parseMftImage = ( + bytes: Uint8Array, +): Result => { + if (bytes.length < NTFS_MFT_MIN_RECORD_BYTES) + return err( + failure( + `file is ${String(bytes.length)} bytes, shorter than one ${String(NTFS_MFT_MIN_RECORD_BYTES)}-byte record header`, + ), + ); + if (!hasMftFileRecordSignature(bytes)) + return err(failure("record 0 does not begin with the FILE signature")); + + const limitations: string[] = []; + const header = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const declaredRecordBytes = header.getUint32(RECORD_HEADER.totalBytes, true); + const recordBytes = isRecordSize(declaredRecordBytes) + ? declaredRecordBytes + : MFT_DEFAULT_RECORD_BYTES; + if (!isRecordSize(declaredRecordBytes)) + limitations.push( + `record 0 declares a record size of ${String(declaredRecordBytes)} bytes, which is not a supported power of two from ${String(MFT_MIN_RECORD_BYTES)} to ${String(MFT_MAX_RECORD_BYTES)}; records were read at the ${String(MFT_DEFAULT_RECORD_BYTES)}-byte default`, + ); + + const records: MftRecord[] = []; + for ( + let offset = 0; + offset + recordBytes <= bytes.length; + offset += recordBytes + ) { + if ( + text(bytes.subarray(offset, offset + MFT_FILE_RECORD_MAGIC_BYTES)) !== + MFT_FILE_RECORD_MAGIC + ) { + limitations.push( + `record at ${String(offset)} does not begin with the FILE signature; ${String(records.length)} record(s) were examined`, + ); + break; + } + records.push(readRecord(bytes, offset, records.length, recordBytes)); + } + + const truncatedTailBytes = bytes.length % recordBytes; + if (truncatedTailBytes !== 0) + limitations.push( + `${String(truncatedTailBytes)} trailing byte(s) do not fill a ${String(recordBytes)}-byte record and were not parsed`, + ); + + const mismatchCount = records.filter( + (record) => record.fixups.mismatchSector !== null, + ).length; + if (mismatchCount !== 0) + limitations.push( + `${String(mismatchCount)} record(s) hold a sector trailer that is not the update sequence number; their attributes were not read and no later step should trust them`, + ); + const unverifiedCount = records.filter( + (record) => + record.fixups.mismatchSector === null && + (record.fixups.malformed || + record.fixups.coveredSectors < recordBytes / FIXUP_SECTOR_BYTES), + ).length; + if (unverifiedCount !== 0) + limitations.push( + `${String(unverifiedCount)} record(s) carry an update sequence array that does not cover every sector; those trailers were neither restored nor verified`, + ); + + const metadata: MftMetadata = { + kind: "ntfs-mft", + record_bytes: recordBytes, + record_count: records.length, + in_use_record_count: records.filter((record) => record.inUse).length, + directory_record_count: records.filter((record) => record.directory).length, + extension_record_count: records.filter( + (record) => record.baseRecordReference !== null, + ).length, + fixup_mismatch_count: mismatchCount, + fixup_unverified_record_count: unverifiedCount, + truncated_tail_bytes: truncatedTailBytes, + }; + + return ok({ metadata, recordBytes, records, limitations }); +}; + +const readRecord = ( + bytes: Uint8Array, + fileOffset: number, + index: number, + recordBytes: number, +): MftRecord => { + const record = bytes.subarray(fileOffset, fileOffset + recordBytes); + const view = new DataView(record.buffer, record.byteOffset, record.byteLength); + + const flags = view.getUint16(RECORD_HEADER.flags, true); + const baseRecordReference = view.getBigUint64( + RECORD_HEADER.baseRecordReference, + true, + ); + + return { + index, + fileOffset, + sequenceNumber: view.getUint16(RECORD_HEADER.sequenceNumber, true), + hardLinkCount: view.getUint16(RECORD_HEADER.hardLinkCount, true), + flags: { raw: flags, names: namesOf(flags) }, + inUse: (flags & FILE_RECORD_IN_USE) !== 0, + directory: (flags & FILE_RECORD_IS_DIRECTORY) !== 0, + usedBytes: view.getUint32(RECORD_HEADER.usedBytes, true), + totalBytes: view.getUint32(RECORD_HEADER.totalBytes, true), + baseRecordReference: + baseRecordReference === 0n + ? null + : Number(baseRecordReference & 0xffff_ffff_ffffn), + declaredRecordNumber: + recordBytes >= RECORD_HEADER.recordNumber + 4 + ? view.getUint32(RECORD_HEADER.recordNumber, true) + : null, + logFileSequenceNumber: Number( + view.getBigUint64(RECORD_HEADER.logFileSequenceNumber, true), + ), + firstAttributeOffset: view.getUint16( + RECORD_HEADER.firstAttributeOffset, + true, + ), + nextAttributeId: view.getUint16(RECORD_HEADER.nextAttributeId, true), + fixups: verifyFixups(record, view), + }; +}; + +/** + * Each sector's last two bytes are replaced on disk by the update sequence + * number; the array's first entry is that number and the rest are the saved + * trailer words. A sector whose trailer is not the sequence number was not + * cleanly written, so the record's attributes cannot be trusted. + */ +const verifyFixups = ( + record: Uint8Array, + view: DataView, +): MftRecordFixups => { + const usaOffset = view.getUint16(RECORD_HEADER.updateSequenceOffset, true); + const usaCount = view.getUint16(RECORD_HEADER.updateSequenceCount, true); + const sectors = Math.floor(record.length / FIXUP_SECTOR_BYTES); + // Entry 0 holds the sequence number, so sector n's saved trailer is entry n+1. + const coveredSectors = Math.min(sectors, Math.max(0, usaCount - 1)); + + if ( + usaCount < USA_MIN_ENTRIES || + usaOffset < RECORD_HEADER_BYTES || + usaOffset + usaCount * 2 > record.length || + coveredSectors === 0 + ) + return { + sequenceNumber: 0, + entries: usaCount, + verifiedSectors: 0, + coveredSectors: 0, + mismatchSector: null, + malformed: true, + }; + + const sequenceNumber = view.getUint16(usaOffset, true); + for (let sector = 0; sector < coveredSectors; sector += 1) { + const trailerOffset = sector * FIXUP_SECTOR_BYTES + FIXUP_SECTOR_BYTES - 2; + if (view.getUint16(trailerOffset, true) !== sequenceNumber) + return { + sequenceNumber, + entries: usaCount, + verifiedSectors: sector, + coveredSectors, + mismatchSector: sector, + malformed: false, + }; + } + + return { + sequenceNumber, + entries: usaCount, + verifiedSectors: coveredSectors, + coveredSectors, + mismatchSector: null, + malformed: false, + }; +}; + +/** Streaming form of the same parse: the whole admitted file is buffered once. */ +class StreamingMftDecoder implements RegionDecoder { + readonly #chunks: Uint8Array[] = []; + #length = 0; + + constructor(readonly fileBytes: number) {} + + write(chunk: Uint8Array): Result { + this.#chunks.push(chunk); + this.#length += chunk.length; + return ok(undefined); + } + + finish(): Result { + const bytes = new Uint8Array(this.#length); + let written = 0; + for (const chunk of this.#chunks) { + bytes.set(chunk, written); + written += chunk.length; + } + return parseMftImage(bytes); + } +} + +export const createMftDecoder = ( + fileBytes: number, +): Result, RegionDecodeFailure> => + Number.isSafeInteger(fileBytes) && + fileBytes >= 0 && + fileBytes <= MFT_MAX_FILE_BYTES + ? ok(new StreamingMftDecoder(fileBytes)) + : err( + limit( + `file is ${String(fileBytes)} bytes, exceeding the fixed ${String(MFT_MAX_FILE_BYTES)}-byte (512 MiB) ceiling`, + "file-size", + ), + );