diff --git a/scripts/verify/baselines/raw-buffer-read.json b/scripts/verify/baselines/raw-buffer-read.json index 7697d90578..0bdc4c93fd 100644 --- a/scripts/verify/baselines/raw-buffer-read.json +++ b/scripts/verify/baselines/raw-buffer-read.json @@ -61,8 +61,6 @@ "src/domain/python/pythonBytecodeIdentity.ts": 1, "src/domain/riscvIsaDeclaration.ts": 1, "src/domain/unityfsFormat.ts": 5, - "src/domain/windows/ntfsMftAttributes.ts": 38, - "src/domain/windows/ntfsUsnJournal.ts": 23, "src/dotnet/ManagedConstantValue.ts": 12, "src/dotnet/ManagedMemberInstructionDecoder.ts": 30, "src/dotnet/ManagedMethodBodyReader.ts": 5, diff --git a/src/domain/windows/ntfsMftAttributes.test.ts b/src/domain/windows/ntfsMftAttributes.test.ts index f549c86c8d..70db2831ba 100644 --- a/src/domain/windows/ntfsMftAttributes.test.ts +++ b/src/domain/windows/ntfsMftAttributes.test.ts @@ -96,6 +96,27 @@ it("rejects a FILE_NAME attribute with an undefined namespace", () => { }); }); +it("retains a prior FILE_NAME when the next name would read into attribute padding", () => { + const value = Buffer.alloc(68); + value[64] = 1; + value.write("x", 66, "utf16le"); + const valid = resident(0x30, value); + value[64] = 2; + const malformed = resident(0x30, value); + + expect(scan(valid, malformed)).toMatchObject({ + attributes: [{ type: 0x30 }], + fileNames: [{ name: "x", fileOffset: 4120 }], + attributeCoverage: "incomplete", + fileNameCoverage: "incomplete", + attributeIssue: { + kind: "malformed", + fileOffset: 4192, + message: "FILE_NAME string exceeds its value", + }, + }); +}); + it("retains fragmented and sparse mapping pairs and lossless nonresident sizes without reading a volume", () => { const attribute = nonresident(); const original = Buffer.from(attribute); diff --git a/src/domain/windows/ntfsMftAttributes.ts b/src/domain/windows/ntfsMftAttributes.ts index f49463fe05..516aa3a7c4 100644 --- a/src/domain/windows/ntfsMftAttributes.ts +++ b/src/domain/windows/ntfsMftAttributes.ts @@ -1,4 +1,5 @@ import { z } from "zod"; +import { BinaryReader } from "../binaryContainer.js"; import { err, ok, type Result } from "../result.js"; export type MftFileReference = z.infer; @@ -159,34 +160,62 @@ export interface MftAttributeScan { readonly attributeCoverage: "complete" | "incomplete"; } -const readTimes = (value: Buffer) => ({ - createdFileTime: value.readBigUInt64LE(0).toString(), - modifiedFileTime: value.readBigUInt64LE(8).toString(), - mftChangedFileTime: value.readBigUInt64LE(16).toString(), - accessedFileTime: value.readBigUInt64LE(24).toString(), +const attributeReader = (bytes: Buffer): BinaryReader => + new BinaryReader(bytes, true, "ntfs-mft"); + +const readTimes = (value: BinaryReader, offset: number) => ({ + createdFileTime: value.word(offset, 64).toString(), + modifiedFileTime: value.word(offset + 8, 64).toString(), + mftChangedFileTime: value.word(offset + 16, 64).toString(), + accessedFileTime: value.word(offset + 24, 64).toString(), }); const readStandardInformation = ( - value: Buffer, + value: BinaryReader, ): z.infer | undefined => { - if (value.length < 48 || (value.length > 48 && value.length < 72)) - return undefined; + const size = value.bytes.length; + const extended = size >= 72; + if (size !== 48 && !extended) return undefined; return { - ...readTimes(value), - fileAttributes: value.readUInt32LE(32), - maximumVersions: value.readUInt32LE(36), - version: value.readUInt32LE(40), - classId: value.readUInt32LE(44), - ownerId: value.length >= 72 ? value.readUInt32LE(48) : null, - securityId: value.length >= 72 ? value.readUInt32LE(52) : null, - quotaCharged: - value.length >= 72 ? value.readBigUInt64LE(56).toString() : null, - usn: value.length >= 72 ? value.readBigInt64LE(64).toString() : null, + ...readTimes(value, 0), + fileAttributes: value.u32(32), + maximumVersions: value.u32(36), + version: value.u32(40), + classId: value.u32(44), + ownerId: extended ? value.u32(48) : null, + securityId: extended ? value.u32(52) : null, + quotaCharged: extended ? value.word(56, 64).toString() : null, + usn: extended ? BigInt.asIntN(64, value.word(64, 64)).toString() : null, }; }; +const readFileName = ( + value: BinaryReader, + fileOffset: number, +): Result => { + const { bytes } = value; + if (bytes.length < 66) return err("Truncated FILE_NAME value"); + const nameBytes = bytes[64]! * 2; + if (66 + nameBytes > bytes.length) + return err("FILE_NAME string exceeds its value"); + const namespace = bytes[65]!; + if (namespace > 3) + return err("FILE_NAME namespace is outside the NTFS range"); + return ok({ + fileOffset, + parent: decodeMftFileReference(value.word(0, 64)), + ...readTimes(value, 8), + namespace, + name: bytes.toString("utf16le", 66, 66 + nameBytes), + allocatedBytes: value.word(40, 64).toString(), + dataBytes: value.word(48, 64).toString(), + fileAttributes: value.u32(56), + reparseOrEa: value.u32(60), + }); +}; + const readList = ( - value: Buffer, + value: BinaryReader, fileOffset: number, ): Result< z.infer[], @@ -202,16 +231,17 @@ const readList = ( message, partialEntries: entries, }); - for (let cursor = 0; cursor < value.length;) { - if (value.length - cursor < 26) + const { bytes } = value; + for (let cursor = 0; cursor < bytes.length;) { + if (bytes.length - cursor < 26) return malformed(cursor, "Truncated attribute-list entry"); - const length = value.readUInt16LE(cursor + 4); - const nameBytes = value[cursor + 6]! * 2; - const nameOffset = value[cursor + 7]!; + const length = value.u16(cursor + 4); + const nameBytes = bytes[cursor + 6]! * 2; + const nameOffset = bytes[cursor + 7]!; if ( length < 26 || length % 8 !== 0 || - length > value.length - cursor || + length > bytes.length - cursor || (nameBytes > 0 && (nameOffset < 26 || nameOffset % 2 !== 0 || @@ -221,18 +251,18 @@ const readList = ( entries.push({ fileOffset: fileOffset + cursor, recordBytes: length, - type: value.readUInt32LE(cursor), + type: value.u32(cursor), name: nameBytes === 0 ? "" - : value.toString( + : bytes.toString( "utf16le", cursor + nameOffset, cursor + nameOffset + nameBytes, ), - lowestVcn: value.readBigUInt64LE(cursor + 8).toString(), - reference: decodeMftFileReference(value.readBigUInt64LE(cursor + 16)), - attributeId: value.readUInt16LE(cursor + 24), + lowestVcn: value.word(cursor + 8, 64).toString(), + reference: decodeMftFileReference(value.word(cursor + 16, 64)), + attributeId: value.u16(cursor + 24), }); cursor += length; } @@ -287,18 +317,25 @@ const readRuns = (bytes: Buffer, expectedClusters: bigint) => { return undefined; }; +type DecodedAttribute = { + readonly attribute: MftAttribute; + readonly fileName: MftFileName | null; +}; + const readAttribute = ( - attribute: Buffer, + reader: BinaryReader, + type: number, fileOffset: number, -): Result => { +): Result => { const malformed = (message: string): Result => err({ kind: "malformed", fileOffset, message }); + const attribute = reader.bytes; const form = attribute[8]; const headerBytes = form === 0 ? 24 : 64; if (form === undefined || form > 1 || attribute.length < headerBytes) return malformed("Invalid attribute form or header extent"); const nameBytes = attribute[9]! * 2; - const nameOffset = attribute.readUInt16LE(10); + const nameOffset = reader.u16(10); if ( nameBytes > 0 && (nameOffset < headerBytes || @@ -310,26 +347,38 @@ const readAttribute = ( const common = { fileOffset, recordBytes: attribute.length, - type: attribute.readUInt32LE(0), - id: attribute.readUInt16LE(14), - flags: attribute.readUInt16LE(12), + type, + id: reader.u16(14), + flags: reader.u16(12), name: nameBytes === 0 ? "" : attribute.toString("utf16le", nameOffset, nameEnd), }; if (form === 1 && (common.type === 0x10 || common.type === 0x30)) return malformed("Standard information and FILE_NAME must be resident"); if (form === 0) { - const valueBytes = attribute.readUInt32LE(16), - valueOffset = attribute.readUInt16LE(20); + const valueBytes = reader.u32(16), + valueOffset = reader.u16(20); if (valueOffset < nameEnd || valueBytes > attribute.length - valueOffset) return malformed( "Resident value overlaps its header/name or exceeds the attribute", ); - const value = attribute.subarray(valueOffset, valueOffset + valueBytes); + const value = reader.range( + valueOffset, + valueBytes, + "resident attribute value", + ); + const valueFileOffset = fileOffset + valueOffset; const standardInformation = - common.type === 0x10 ? readStandardInformation(value) : null; + type === 0x10 ? readStandardInformation(attributeReader(value)) : null; + if (standardInformation === undefined) + return malformed("Invalid standard-information value"); + const fileName = + type === 0x30 + ? readFileName(attributeReader(value), valueFileOffset) + : null; + if (fileName !== null && !fileName.ok) return malformed(fileName.error); const list = - common.type === 0x20 ? readList(value, fileOffset + valueOffset) : null; + type === 0x20 ? readList(attributeReader(value), valueFileOffset) : null; const listEntries = list === null ? null : list.ok ? list.value : list.error.partialEntries; const listIssue = @@ -340,24 +389,25 @@ const readAttribute = ( message: list.error.message, } : null; - if (standardInformation === undefined) - return malformed("Invalid standard-information value"); return ok({ - ...common, - storage: { - kind: "resident", - valueFileOffset: fileOffset + valueOffset, - valueBytes, - standardInformation, - listEntries, - listIssue, + attribute: { + ...common, + storage: { + kind: "resident", + valueFileOffset, + valueBytes, + standardInformation, + listEntries, + listIssue, + }, }, + fileName: fileName?.value ?? null, }); } - const lowestVcn = attribute.readBigUInt64LE(16), - highestVcn = attribute.readBigUInt64LE(24); + const lowestVcn = reader.word(16, 64), + highestVcn = reader.word(24, 64); const extended = (common.flags & 0x8001) !== 0 && lowestVcn === 0n; - const runsOffset = attribute.readUInt16LE(32); + const runsOffset = reader.u16(32); if ( runsOffset < Math.max(nameEnd, extended ? 72 : 64) || runsOffset >= attribute.length || @@ -373,30 +423,33 @@ const readAttribute = ( ); if (runs === undefined) return malformed("Invalid mapping pairs or incomplete VCN coverage"); - const dataBytes = attribute.readBigUInt64LE(48), - initializedBytes = attribute.readBigUInt64LE(56); + const dataBytes = reader.word(48, 64), + initializedBytes = reader.word(56, 64); if (lowestVcn === 0n && initializedBytes > dataBytes) return malformed("Initialized size exceeds the declared data size"); return ok({ - ...common, - storage: { - kind: "nonresident", - lowestVcn: lowestVcn.toString(), - highestVcn: highestVcn.toString(), - compressionUnit: attribute.readUInt16LE(34), - allocatedBytes: attribute.readBigUInt64LE(40).toString(), - dataBytes: dataBytes.toString(), - initializedBytes: initializedBytes.toString(), - physicalBytes: extended ? attribute.readBigUInt64LE(64).toString() : null, - mappingPairsFileOffset: fileOffset + runsOffset, - mappingPairsHex: attribute.toString( - "hex", - runsOffset, - runsOffset + runs.bytes, - ), - runCount: runs.runCount, - sparseRunCount: runs.sparseRunCount, + attribute: { + ...common, + storage: { + kind: "nonresident", + lowestVcn: lowestVcn.toString(), + highestVcn: highestVcn.toString(), + compressionUnit: reader.u16(34), + allocatedBytes: reader.word(40, 64).toString(), + dataBytes: dataBytes.toString(), + initializedBytes: initializedBytes.toString(), + physicalBytes: extended ? reader.word(64, 64).toString() : null, + mappingPairsFileOffset: fileOffset + runsOffset, + mappingPairsHex: attribute.toString( + "hex", + runsOffset, + runsOffset + runs.bytes, + ), + runCount: runs.runCount, + sparseRunCount: runs.sparseRunCount, + }, }, + fileName: null, }); }; @@ -409,6 +462,7 @@ export const readMftAttributes = ( ): MftAttributeScan => { const fileNames: MftFileName[] = [], attributes: MftAttribute[] = []; + const reader = attributeReader(record.subarray(0, usedBytes)); let complete = false, hasList = false; let attributeIssue: AttributeIssue | null = null; @@ -420,7 +474,8 @@ export const readMftAttributes = ( }; }; for (let cursor = first; cursor + 4 <= usedBytes;) { - if (record.readUInt32LE(cursor) === 0xffffffff) { + const type = reader.u32(cursor); + if (type === 0xffffffff) { complete = true; break; } @@ -428,58 +483,27 @@ export const readMftAttributes = ( malformed(cursor, "Truncated attribute header"); break; } - const length = record.readUInt32LE(cursor + 4); + const length = reader.u32(cursor + 4); if (length < 24 || length % 8 !== 0 || length > usedBytes - cursor) { malformed(cursor, "Invalid attribute length"); break; } const decoded = readAttribute( - record.subarray(cursor, cursor + length), + attributeReader(reader.range(cursor, length, "attribute record")), + type, fileOffset + cursor, ); if (!decoded.ok) { attributeIssue ??= decoded.error; break; } - const attribute = decoded.value; + const { attribute, fileName } = decoded.value; if ( attribute.storage.kind === "resident" && attribute.storage.listIssue !== null ) attributeIssue ??= attribute.storage.listIssue; - if (attribute.type === 0x30 && attribute.storage.kind === "resident") { - const valueOffset = attribute.storage.valueFileOffset - fileOffset; - const valueBytes = attribute.storage.valueBytes; - if (valueBytes < 66) { - malformed(cursor, "Truncated FILE_NAME value"); - break; - } - const nameBytes = record[valueOffset + 64]! * 2; - if (66 + nameBytes > valueBytes) { - malformed(cursor, "FILE_NAME string exceeds its value"); - break; - } - const namespace = record[valueOffset + 65]!; - if (namespace > 3) { - malformed(cursor, "FILE_NAME namespace is outside the NTFS range"); - break; - } - fileNames.push({ - fileOffset: fileOffset + valueOffset, - parent: decodeMftFileReference(record.readBigUInt64LE(valueOffset)), - ...readTimes(record.subarray(valueOffset + 8)), - namespace, - name: record.toString( - "utf16le", - valueOffset + 66, - valueOffset + 66 + nameBytes, - ), - allocatedBytes: record.readBigUInt64LE(valueOffset + 40).toString(), - dataBytes: record.readBigUInt64LE(valueOffset + 48).toString(), - fileAttributes: record.readUInt32LE(valueOffset + 56), - reparseOrEa: record.readUInt32LE(valueOffset + 60), - }); - } + if (fileName !== null) fileNames.push(fileName); hasList ||= attribute.type === 0x20; attributes.push(attribute); cursor += length; diff --git a/src/domain/windows/ntfsUsnJournal.test.ts b/src/domain/windows/ntfsUsnJournal.test.ts index 234f68a608..db3e7bc5eb 100644 --- a/src/domain/windows/ntfsUsnJournal.test.ts +++ b/src/domain/windows/ntfsUsnJournal.test.ts @@ -68,14 +68,14 @@ it("stops at a truncated record", () => { ]); }); -it("decodes V3 opaque 128-bit identifiers without narrowing to MFT references", () => { +it("preserves V3 opaque 128-bit identifiers and signed USN words", () => { const bytes = Buffer.alloc(88); bytes.writeUInt32LE(bytes.length); bytes.writeUInt16LE(3, 4); bytes.writeBigUInt64LE(0x0002_0000_0000_0007n, 8); bytes.writeBigUInt64LE(0xfedcba9876543210n, 16); bytes.writeBigUInt64LE(9007199254740993n, 24); - bytes.writeBigInt64LE(9007199254740993n, 40); + bytes.writeBigInt64LE(-9007199254740993n, 40); bytes.writeBigUInt64LE(133000000000000001n, 48); bytes.writeUInt16LE(6, 72); bytes.writeUInt16LE(76, 74); @@ -89,7 +89,7 @@ it("decodes V3 opaque 128-bit identifiers without narrowing to MFT references", name: "new", fileReference: 0xfedcba98765432100002000000000007n, parentReference: 9007199254740993n, - usn: 9007199254740993n, + usn: -9007199254740993n, timestampFileTime: 133000000000000001n, }, }, diff --git a/src/domain/windows/ntfsUsnJournal.ts b/src/domain/windows/ntfsUsnJournal.ts index daddf573e8..b3f7a0545c 100644 --- a/src/domain/windows/ntfsUsnJournal.ts +++ b/src/domain/windows/ntfsUsnJournal.ts @@ -1,3 +1,4 @@ +import { BinaryReader } from "../binaryContainer.js"; import { err, ok, type Result } from "../result.js"; const COMMON_HEADER_BYTES = 8; @@ -48,16 +49,17 @@ export function* readUsnJournal( bytes: Buffer, startFileOffset = 0, ): Generator> { + const journal = new BinaryReader(bytes, true, "ntfs-usn"); let fileOffset = startFileOffset; while (fileOffset + COMMON_HEADER_BYTES <= bytes.length) { - const length = bytes.readUInt32LE(fileOffset); - if (length === 0 && bytes.readUInt32LE(fileOffset + 4) === 0) { + const length = journal.u32(fileOffset); + if (length === 0 && journal.u32(fileOffset + 4) === 0) { const start = fileOffset; do { fileOffset += COMMON_HEADER_BYTES; } while ( fileOffset + COMMON_HEADER_BYTES <= bytes.length && - bytes.readBigUInt64LE(fileOffset) === 0n + journal.word(fileOffset, 64) === 0n ); yield err({ kind: "padding", @@ -75,48 +77,50 @@ export function* readUsnJournal( return; } - const major = bytes.readUInt16LE(fileOffset + 4); - const minor = bytes.readUInt16LE(fileOffset + 6); + const major = journal.u16(fileOffset + 4); + const minor = journal.u16(fileOffset + 6); if ((major !== 2 && major !== 3 && major !== 4) || minor !== 0) { yield err({ kind: "unsupported", fileOffset, major, minor }); - } else if (major === 4) { - yield readExtentRecord( - bytes.subarray(fileOffset, fileOffset + length), - fileOffset, - ); } else { - yield readNamedRecord( - bytes.subarray(fileOffset, fileOffset + length), - fileOffset, - major, + const record = new BinaryReader( + journal.range(fileOffset, length, "USN record"), + journal.little, + journal.format, ); + yield major === 4 + ? readExtentRecord(record, fileOffset) + : readNamedRecord(record, fileOffset, major); } fileOffset += length; } if (fileOffset < bytes.length) yield err({ kind: "truncated", fileOffset }); } -const readReference = (record: Buffer, offset: number, width: 8 | 16): bigint => - record.readBigUInt64LE(offset) | - (width === 16 ? record.readBigUInt64LE(offset + 8) << 64n : 0n); +const readReference = ( + record: BinaryReader, + offset: number, + width: 8 | 16, +): bigint => + record.word(offset, 64) | + (width === 16 ? record.word(offset + 8, 64) << 64n : 0n); const readNamedRecord = ( - record: Buffer, + record: BinaryReader, fileOffset: number, majorVersion: 2 | 3, ): Result => { const referenceBytes = majorVersion === 2 ? 8 : 16; const body = COMMON_HEADER_BYTES + referenceBytes * 2; const headerBytes = body + 36; - if (record.length < headerBytes) - return err({ kind: "malformed", fileOffset }); - const nameBytes = record.readUInt16LE(body + 32); - const nameOffset = record.readUInt16LE(body + 34); + const recordBytes = record.bytes.length; + if (recordBytes < headerBytes) return err({ kind: "malformed", fileOffset }); + const nameBytes = record.u16(body + 32); + const nameOffset = record.u16(body + 34); if ( nameOffset < headerBytes || nameOffset % 2 !== 0 || nameBytes % 2 !== 0 || - nameOffset + nameBytes > record.length + nameOffset + nameBytes > recordBytes ) return err({ kind: "malformed", fileOffset }); @@ -124,31 +128,34 @@ const readNamedRecord = ( majorVersion, referenceBytes, fileOffset, - recordBytes: record.length, + recordBytes, fileReference: readReference(record, 8, referenceBytes), parentReference: readReference(record, 8 + referenceBytes, referenceBytes), - usn: record.readBigInt64LE(body), - timestampFileTime: record.readBigUInt64LE(body + 8), - reasonFlags: record.readUInt32LE(body + 16), - sourceFlags: record.readUInt32LE(body + 20), - securityId: record.readUInt32LE(body + 24), - fileAttributes: record.readUInt32LE(body + 28), - name: record.toString("utf16le", nameOffset, nameOffset + nameBytes), + usn: BigInt.asIntN(64, record.word(body, 64)), + timestampFileTime: record.word(body + 8, 64), + reasonFlags: record.u32(body + 16), + sourceFlags: record.u32(body + 20), + securityId: record.u32(body + 24), + fileAttributes: record.u32(body + 28), + name: record + .range(nameOffset, nameBytes, "USN filename") + .toString("utf16le"), }); }; const readExtentRecord = ( - record: Buffer, + record: BinaryReader, fileOffset: number, ): Result => { - if (record.length < 64) return err({ kind: "malformed", fileOffset }); - const extentCount = record.readUInt16LE(60), - extentSize = record.readUInt16LE(62); + const recordBytes = record.bytes.length; + if (recordBytes < 64) return err({ kind: "malformed", fileOffset }); + const extentCount = record.u16(60), + extentSize = record.u16(62); // Validate the complete declared array before allocating the fixed-size sample. if ( extentSize < 16 || extentSize % 8 !== 0 || - extentCount > Math.floor((record.length - 64) / extentSize) + extentCount > Math.floor((recordBytes - 64) / extentSize) ) return err({ kind: "malformed", fileOffset }); const extentSample: { fileOffset: number; offset: bigint; length: bigint }[] = @@ -161,25 +168,25 @@ const readExtentRecord = ( const cursor = 64 + index * extentSize; extentSample.push({ fileOffset: fileOffset + cursor, - offset: record.readBigInt64LE(cursor), - length: record.readBigInt64LE(cursor + 8), + offset: BigInt.asIntN(64, record.word(cursor, 64)), + length: BigInt.asIntN(64, record.word(cursor + 8, 64)), }); } return ok({ majorVersion: 4, referenceBytes: 16, fileOffset, - recordBytes: record.length, + recordBytes, fileReference: readReference(record, 8, 16), parentReference: readReference(record, 24, 16), - usn: record.readBigInt64LE(40), - reasonFlags: record.readUInt32LE(48), - sourceFlags: record.readUInt32LE(52), + usn: BigInt.asIntN(64, record.word(40, 64)), + reasonFlags: record.u32(48), + sourceFlags: record.u32(52), timestampFileTime: null, securityId: null, fileAttributes: null, name: null, - remainingExtents: record.readUInt32LE(56), + remainingExtents: record.u32(56), extentCount, extentSize, omittedExtents: extentCount - extentSample.length,