refactor(ntfs): decode attributes within their declared values (#12211)

* refactor(ntfs): decode attributes within their declared values

* refactor(ntfs): bound USN journal record reads (#12216)

refactor(ntfs): bound USN record fields with BinaryReader
This commit is contained in:
morluto authored and GitHub committed 2026-11-19 05:43:39 +00:00
1 parent 4d10ca018f
commit 1aab66f12c
5 files changed
+209 -159

No files matched your search

@@ -61,8 +61,6 @@
"src/domain/python/pythonBytecodeIdentity.ts": 1,
"src/domain/riscvIsaDeclaration.ts": 1,
"src/domain/unityfsFormat.ts": 5,
"src/domain/windows/ntfsMftAttributes.ts": 38,
"src/domain/windows/ntfsUsnJournal.ts": 23,
"src/dotnet/ManagedConstantValue.ts": 12,
"src/dotnet/ManagedMemberInstructionDecoder.ts": 30,
"src/dotnet/ManagedMethodBodyReader.ts": 5,
@@ -96,6 +96,27 @@ it("rejects a FILE_NAME attribute with an undefined namespace", () => {
});
});
it("retains a prior FILE_NAME when the next name would read into attribute padding", () => {
const value = Buffer.alloc(68);
value[64] = 1;
value.write("x", 66, "utf16le");
const valid = resident(0x30, value);
value[64] = 2;
const malformed = resident(0x30, value);
expect(scan(valid, malformed)).toMatchObject({
attributes: [{ type: 0x30 }],
fileNames: [{ name: "x", fileOffset: 4120 }],
attributeCoverage: "incomplete",
fileNameCoverage: "incomplete",
attributeIssue: {
kind: "malformed",
fileOffset: 4192,
message: "FILE_NAME string exceeds its value",
},
});
});
it("retains fragmented and sparse mapping pairs and lossless nonresident sizes without reading a volume", () => {
const attribute = nonresident();
const original = Buffer.from(attribute);
+135 -111
View File
@@ -1,4 +1,5 @@
import { z } from "zod";
import { BinaryReader } from "../binaryContainer.js";
import { err, ok, type Result } from "../result.js";
export type MftFileReference = z.infer<typeof ntfsFileReferenceSchema>;
@@ -159,34 +160,62 @@ export interface MftAttributeScan {
readonly attributeCoverage: "complete" | "incomplete";
}
const readTimes = (value: Buffer) => ({
createdFileTime: value.readBigUInt64LE(0).toString(),
modifiedFileTime: value.readBigUInt64LE(8).toString(),
mftChangedFileTime: value.readBigUInt64LE(16).toString(),
accessedFileTime: value.readBigUInt64LE(24).toString(),
const attributeReader = (bytes: Buffer): BinaryReader =>
new BinaryReader(bytes, true, "ntfs-mft");
const readTimes = (value: BinaryReader, offset: number) => ({
createdFileTime: value.word(offset, 64).toString(),
modifiedFileTime: value.word(offset + 8, 64).toString(),
mftChangedFileTime: value.word(offset + 16, 64).toString(),
accessedFileTime: value.word(offset + 24, 64).toString(),
});
const readStandardInformation = (
value: Buffer,
value: BinaryReader,
): z.infer<typeof standardInformationSchema> | undefined => {
if (value.length < 48 || (value.length > 48 && value.length < 72))
return undefined;
const size = value.bytes.length;
const extended = size >= 72;
if (size !== 48 && !extended) return undefined;
return {
...readTimes(value),
fileAttributes: value.readUInt32LE(32),
maximumVersions: value.readUInt32LE(36),
version: value.readUInt32LE(40),
classId: value.readUInt32LE(44),
ownerId: value.length >= 72 ? value.readUInt32LE(48) : null,
securityId: value.length >= 72 ? value.readUInt32LE(52) : null,
quotaCharged:
value.length >= 72 ? value.readBigUInt64LE(56).toString() : null,
usn: value.length >= 72 ? value.readBigInt64LE(64).toString() : null,
...readTimes(value, 0),
fileAttributes: value.u32(32),
maximumVersions: value.u32(36),
version: value.u32(40),
classId: value.u32(44),
ownerId: extended ? value.u32(48) : null,
securityId: extended ? value.u32(52) : null,
quotaCharged: extended ? value.word(56, 64).toString() : null,
usn: extended ? BigInt.asIntN(64, value.word(64, 64)).toString() : null,
};
};
const readFileName = (
value: BinaryReader,
fileOffset: number,
): Result<MftFileName, string> => {
const { bytes } = value;
if (bytes.length < 66) return err("Truncated FILE_NAME value");
const nameBytes = bytes[64]! * 2;
if (66 + nameBytes > bytes.length)
return err("FILE_NAME string exceeds its value");
const namespace = bytes[65]!;
if (namespace > 3)
return err("FILE_NAME namespace is outside the NTFS range");
return ok({
fileOffset,
parent: decodeMftFileReference(value.word(0, 64)),
...readTimes(value, 8),
namespace,
name: bytes.toString("utf16le", 66, 66 + nameBytes),
allocatedBytes: value.word(40, 64).toString(),
dataBytes: value.word(48, 64).toString(),
fileAttributes: value.u32(56),
reparseOrEa: value.u32(60),
});
};
const readList = (
value: Buffer,
value: BinaryReader,
fileOffset: number,
): Result<
z.infer<typeof listEntrySchema>[],
@@ -202,16 +231,17 @@ const readList = (
message,
partialEntries: entries,
});
for (let cursor = 0; cursor < value.length;) {
if (value.length - cursor < 26)
const { bytes } = value;
for (let cursor = 0; cursor < bytes.length;) {
if (bytes.length - cursor < 26)
return malformed(cursor, "Truncated attribute-list entry");
const length = value.readUInt16LE(cursor + 4);
const nameBytes = value[cursor + 6]! * 2;
const nameOffset = value[cursor + 7]!;
const length = value.u16(cursor + 4);
const nameBytes = bytes[cursor + 6]! * 2;
const nameOffset = bytes[cursor + 7]!;
if (
length < 26 ||
length % 8 !== 0 ||
length > value.length - cursor ||
length > bytes.length - cursor ||
(nameBytes > 0 &&
(nameOffset < 26 ||
nameOffset % 2 !== 0 ||
@@ -221,18 +251,18 @@ const readList = (
entries.push({
fileOffset: fileOffset + cursor,
recordBytes: length,
type: value.readUInt32LE(cursor),
type: value.u32(cursor),
name:
nameBytes === 0
? ""
: value.toString(
: bytes.toString(
"utf16le",
cursor + nameOffset,
cursor + nameOffset + nameBytes,
),
lowestVcn: value.readBigUInt64LE(cursor + 8).toString(),
reference: decodeMftFileReference(value.readBigUInt64LE(cursor + 16)),
attributeId: value.readUInt16LE(cursor + 24),
lowestVcn: value.word(cursor + 8, 64).toString(),
reference: decodeMftFileReference(value.word(cursor + 16, 64)),
attributeId: value.u16(cursor + 24),
});
cursor += length;
}
@@ -287,18 +317,25 @@ const readRuns = (bytes: Buffer, expectedClusters: bigint) => {
return undefined;
};
type DecodedAttribute = {
readonly attribute: MftAttribute;
readonly fileName: MftFileName | null;
};
const readAttribute = (
attribute: Buffer,
reader: BinaryReader,
type: number,
fileOffset: number,
): Result<MftAttribute, AttributeIssue> => {
): Result<DecodedAttribute, AttributeIssue> => {
const malformed = (message: string): Result<never, AttributeIssue> =>
err({ kind: "malformed", fileOffset, message });
const attribute = reader.bytes;
const form = attribute[8];
const headerBytes = form === 0 ? 24 : 64;
if (form === undefined || form > 1 || attribute.length < headerBytes)
return malformed("Invalid attribute form or header extent");
const nameBytes = attribute[9]! * 2;
const nameOffset = attribute.readUInt16LE(10);
const nameOffset = reader.u16(10);
if (
nameBytes > 0 &&
(nameOffset < headerBytes ||
@@ -310,26 +347,38 @@ const readAttribute = (
const common = {
fileOffset,
recordBytes: attribute.length,
type: attribute.readUInt32LE(0),
id: attribute.readUInt16LE(14),
flags: attribute.readUInt16LE(12),
type,
id: reader.u16(14),
flags: reader.u16(12),
name:
nameBytes === 0 ? "" : attribute.toString("utf16le", nameOffset, nameEnd),
};
if (form === 1 && (common.type === 0x10 || common.type === 0x30))
return malformed("Standard information and FILE_NAME must be resident");
if (form === 0) {
const valueBytes = attribute.readUInt32LE(16),
valueOffset = attribute.readUInt16LE(20);
const valueBytes = reader.u32(16),
valueOffset = reader.u16(20);
if (valueOffset < nameEnd || valueBytes > attribute.length - valueOffset)
return malformed(
"Resident value overlaps its header/name or exceeds the attribute",
);
const value = attribute.subarray(valueOffset, valueOffset + valueBytes);
const value = reader.range(
valueOffset,
valueBytes,
"resident attribute value",
);
const valueFileOffset = fileOffset + valueOffset;
const standardInformation =
common.type === 0x10 ? readStandardInformation(value) : null;
type === 0x10 ? readStandardInformation(attributeReader(value)) : null;
if (standardInformation === undefined)
return malformed("Invalid standard-information value");
const fileName =
type === 0x30
? readFileName(attributeReader(value), valueFileOffset)
: null;
if (fileName !== null && !fileName.ok) return malformed(fileName.error);
const list =
common.type === 0x20 ? readList(value, fileOffset + valueOffset) : null;
type === 0x20 ? readList(attributeReader(value), valueFileOffset) : null;
const listEntries =
list === null ? null : list.ok ? list.value : list.error.partialEntries;
const listIssue =
@@ -340,24 +389,25 @@ const readAttribute = (
message: list.error.message,
}
: null;
if (standardInformation === undefined)
return malformed("Invalid standard-information value");
return ok({
...common,
storage: {
kind: "resident",
valueFileOffset: fileOffset + valueOffset,
valueBytes,
standardInformation,
listEntries,
listIssue,
attribute: {
...common,
storage: {
kind: "resident",
valueFileOffset,
valueBytes,
standardInformation,
listEntries,
listIssue,
},
},
fileName: fileName?.value ?? null,
});
}
const lowestVcn = attribute.readBigUInt64LE(16),
highestVcn = attribute.readBigUInt64LE(24);
const lowestVcn = reader.word(16, 64),
highestVcn = reader.word(24, 64);
const extended = (common.flags & 0x8001) !== 0 && lowestVcn === 0n;
const runsOffset = attribute.readUInt16LE(32);
const runsOffset = reader.u16(32);
if (
runsOffset < Math.max(nameEnd, extended ? 72 : 64) ||
runsOffset >= attribute.length ||
@@ -373,30 +423,33 @@ const readAttribute = (
);
if (runs === undefined)
return malformed("Invalid mapping pairs or incomplete VCN coverage");
const dataBytes = attribute.readBigUInt64LE(48),
initializedBytes = attribute.readBigUInt64LE(56);
const dataBytes = reader.word(48, 64),
initializedBytes = reader.word(56, 64);
if (lowestVcn === 0n && initializedBytes > dataBytes)
return malformed("Initialized size exceeds the declared data size");
return ok({
...common,
storage: {
kind: "nonresident",
lowestVcn: lowestVcn.toString(),
highestVcn: highestVcn.toString(),
compressionUnit: attribute.readUInt16LE(34),
allocatedBytes: attribute.readBigUInt64LE(40).toString(),
dataBytes: dataBytes.toString(),
initializedBytes: initializedBytes.toString(),
physicalBytes: extended ? attribute.readBigUInt64LE(64).toString() : null,
mappingPairsFileOffset: fileOffset + runsOffset,
mappingPairsHex: attribute.toString(
"hex",
runsOffset,
runsOffset + runs.bytes,
),
runCount: runs.runCount,
sparseRunCount: runs.sparseRunCount,
attribute: {
...common,
storage: {
kind: "nonresident",
lowestVcn: lowestVcn.toString(),
highestVcn: highestVcn.toString(),
compressionUnit: reader.u16(34),
allocatedBytes: reader.word(40, 64).toString(),
dataBytes: dataBytes.toString(),
initializedBytes: initializedBytes.toString(),
physicalBytes: extended ? reader.word(64, 64).toString() : null,
mappingPairsFileOffset: fileOffset + runsOffset,
mappingPairsHex: attribute.toString(
"hex",
runsOffset,
runsOffset + runs.bytes,
),
runCount: runs.runCount,
sparseRunCount: runs.sparseRunCount,
},
},
fileName: null,
});
};
@@ -409,6 +462,7 @@ export const readMftAttributes = (
): MftAttributeScan => {
const fileNames: MftFileName[] = [],
attributes: MftAttribute[] = [];
const reader = attributeReader(record.subarray(0, usedBytes));
let complete = false,
hasList = false;
let attributeIssue: AttributeIssue | null = null;
@@ -420,7 +474,8 @@ export const readMftAttributes = (
};
};
for (let cursor = first; cursor + 4 <= usedBytes;) {
if (record.readUInt32LE(cursor) === 0xffffffff) {
const type = reader.u32(cursor);
if (type === 0xffffffff) {
complete = true;
break;
}
@@ -428,58 +483,27 @@ export const readMftAttributes = (
malformed(cursor, "Truncated attribute header");
break;
}
const length = record.readUInt32LE(cursor + 4);
const length = reader.u32(cursor + 4);
if (length < 24 || length % 8 !== 0 || length > usedBytes - cursor) {
malformed(cursor, "Invalid attribute length");
break;
}
const decoded = readAttribute(
record.subarray(cursor, cursor + length),
attributeReader(reader.range(cursor, length, "attribute record")),
type,
fileOffset + cursor,
);
if (!decoded.ok) {
attributeIssue ??= decoded.error;
break;
}
const attribute = decoded.value;
const { attribute, fileName } = decoded.value;
if (
attribute.storage.kind === "resident" &&
attribute.storage.listIssue !== null
)
attributeIssue ??= attribute.storage.listIssue;
if (attribute.type === 0x30 && attribute.storage.kind === "resident") {
const valueOffset = attribute.storage.valueFileOffset - fileOffset;
const valueBytes = attribute.storage.valueBytes;
if (valueBytes < 66) {
malformed(cursor, "Truncated FILE_NAME value");
break;
}
const nameBytes = record[valueOffset + 64]! * 2;
if (66 + nameBytes > valueBytes) {
malformed(cursor, "FILE_NAME string exceeds its value");
break;
}
const namespace = record[valueOffset + 65]!;
if (namespace > 3) {
malformed(cursor, "FILE_NAME namespace is outside the NTFS range");
break;
}
fileNames.push({
fileOffset: fileOffset + valueOffset,
parent: decodeMftFileReference(record.readBigUInt64LE(valueOffset)),
...readTimes(record.subarray(valueOffset + 8)),
namespace,
name: record.toString(
"utf16le",
valueOffset + 66,
valueOffset + 66 + nameBytes,
),
allocatedBytes: record.readBigUInt64LE(valueOffset + 40).toString(),
dataBytes: record.readBigUInt64LE(valueOffset + 48).toString(),
fileAttributes: record.readUInt32LE(valueOffset + 56),
reparseOrEa: record.readUInt32LE(valueOffset + 60),
});
}
if (fileName !== null) fileNames.push(fileName);
hasList ||= attribute.type === 0x20;
attributes.push(attribute);
cursor += length;
+3 -3
View File
@@ -68,14 +68,14 @@ it("stops at a truncated record", () => {
]);
});
it("decodes V3 opaque 128-bit identifiers without narrowing to MFT references", () => {
it("preserves V3 opaque 128-bit identifiers and signed USN words", () => {
const bytes = Buffer.alloc(88);
bytes.writeUInt32LE(bytes.length);
bytes.writeUInt16LE(3, 4);
bytes.writeBigUInt64LE(0x0002_0000_0000_0007n, 8);
bytes.writeBigUInt64LE(0xfedcba9876543210n, 16);
bytes.writeBigUInt64LE(9007199254740993n, 24);
bytes.writeBigInt64LE(9007199254740993n, 40);
bytes.writeBigInt64LE(-9007199254740993n, 40);
bytes.writeBigUInt64LE(133000000000000001n, 48);
bytes.writeUInt16LE(6, 72);
bytes.writeUInt16LE(76, 74);
@@ -89,7 +89,7 @@ it("decodes V3 opaque 128-bit identifiers without narrowing to MFT references",
name: "new",
fileReference: 0xfedcba98765432100002000000000007n,
parentReference: 9007199254740993n,
usn: 9007199254740993n,
usn: -9007199254740993n,
timestampFileTime: 133000000000000001n,
},
},
+50 -43
View File
@@ -1,3 +1,4 @@
import { BinaryReader } from "../binaryContainer.js";
import { err, ok, type Result } from "../result.js";
const COMMON_HEADER_BYTES = 8;
@@ -48,16 +49,17 @@ export function* readUsnJournal(
bytes: Buffer,
startFileOffset = 0,
): Generator<Result<UsnRecord, UsnRecordIssue>> {
const journal = new BinaryReader(bytes, true, "ntfs-usn");
let fileOffset = startFileOffset;
while (fileOffset + COMMON_HEADER_BYTES <= bytes.length) {
const length = bytes.readUInt32LE(fileOffset);
if (length === 0 && bytes.readUInt32LE(fileOffset + 4) === 0) {
const length = journal.u32(fileOffset);
if (length === 0 && journal.u32(fileOffset + 4) === 0) {
const start = fileOffset;
do {
fileOffset += COMMON_HEADER_BYTES;
} while (
fileOffset + COMMON_HEADER_BYTES <= bytes.length &&
bytes.readBigUInt64LE(fileOffset) === 0n
journal.word(fileOffset, 64) === 0n
);
yield err({
kind: "padding",
@@ -75,48 +77,50 @@ export function* readUsnJournal(
return;
}
const major = bytes.readUInt16LE(fileOffset + 4);
const minor = bytes.readUInt16LE(fileOffset + 6);
const major = journal.u16(fileOffset + 4);
const minor = journal.u16(fileOffset + 6);
if ((major !== 2 && major !== 3 && major !== 4) || minor !== 0) {
yield err({ kind: "unsupported", fileOffset, major, minor });
} else if (major === 4) {
yield readExtentRecord(
bytes.subarray(fileOffset, fileOffset + length),
fileOffset,
);
} else {
yield readNamedRecord(
bytes.subarray(fileOffset, fileOffset + length),
fileOffset,
major,
const record = new BinaryReader(
journal.range(fileOffset, length, "USN record"),
journal.little,
journal.format,
);
yield major === 4
? readExtentRecord(record, fileOffset)
: readNamedRecord(record, fileOffset, major);
}
fileOffset += length;
}
if (fileOffset < bytes.length) yield err({ kind: "truncated", fileOffset });
}
const readReference = (record: Buffer, offset: number, width: 8 | 16): bigint =>
record.readBigUInt64LE(offset) |
(width === 16 ? record.readBigUInt64LE(offset + 8) << 64n : 0n);
const readReference = (
record: BinaryReader,
offset: number,
width: 8 | 16,
): bigint =>
record.word(offset, 64) |
(width === 16 ? record.word(offset + 8, 64) << 64n : 0n);
const readNamedRecord = (
record: Buffer,
record: BinaryReader,
fileOffset: number,
majorVersion: 2 | 3,
): Result<UsnRecord, UsnRecordIssue> => {
const referenceBytes = majorVersion === 2 ? 8 : 16;
const body = COMMON_HEADER_BYTES + referenceBytes * 2;
const headerBytes = body + 36;
if (record.length < headerBytes)
return err({ kind: "malformed", fileOffset });
const nameBytes = record.readUInt16LE(body + 32);
const nameOffset = record.readUInt16LE(body + 34);
const recordBytes = record.bytes.length;
if (recordBytes < headerBytes) return err({ kind: "malformed", fileOffset });
const nameBytes = record.u16(body + 32);
const nameOffset = record.u16(body + 34);
if (
nameOffset < headerBytes ||
nameOffset % 2 !== 0 ||
nameBytes % 2 !== 0 ||
nameOffset + nameBytes > record.length
nameOffset + nameBytes > recordBytes
)
return err({ kind: "malformed", fileOffset });
@@ -124,31 +128,34 @@ const readNamedRecord = (
majorVersion,
referenceBytes,
fileOffset,
recordBytes: record.length,
recordBytes,
fileReference: readReference(record, 8, referenceBytes),
parentReference: readReference(record, 8 + referenceBytes, referenceBytes),
usn: record.readBigInt64LE(body),
timestampFileTime: record.readBigUInt64LE(body + 8),
reasonFlags: record.readUInt32LE(body + 16),
sourceFlags: record.readUInt32LE(body + 20),
securityId: record.readUInt32LE(body + 24),
fileAttributes: record.readUInt32LE(body + 28),
name: record.toString("utf16le", nameOffset, nameOffset + nameBytes),
usn: BigInt.asIntN(64, record.word(body, 64)),
timestampFileTime: record.word(body + 8, 64),
reasonFlags: record.u32(body + 16),
sourceFlags: record.u32(body + 20),
securityId: record.u32(body + 24),
fileAttributes: record.u32(body + 28),
name: record
.range(nameOffset, nameBytes, "USN filename")
.toString("utf16le"),
});
};
const readExtentRecord = (
record: Buffer,
record: BinaryReader,
fileOffset: number,
): Result<UsnRecord, UsnRecordIssue> => {
if (record.length < 64) return err({ kind: "malformed", fileOffset });
const extentCount = record.readUInt16LE(60),
extentSize = record.readUInt16LE(62);
const recordBytes = record.bytes.length;
if (recordBytes < 64) return err({ kind: "malformed", fileOffset });
const extentCount = record.u16(60),
extentSize = record.u16(62);
// Validate the complete declared array before allocating the fixed-size sample.
if (
extentSize < 16 ||
extentSize % 8 !== 0 ||
extentCount > Math.floor((record.length - 64) / extentSize)
extentCount > Math.floor((recordBytes - 64) / extentSize)
)
return err({ kind: "malformed", fileOffset });
const extentSample: { fileOffset: number; offset: bigint; length: bigint }[] =
@@ -161,25 +168,25 @@ const readExtentRecord = (
const cursor = 64 + index * extentSize;
extentSample.push({
fileOffset: fileOffset + cursor,
offset: record.readBigInt64LE(cursor),
length: record.readBigInt64LE(cursor + 8),
offset: BigInt.asIntN(64, record.word(cursor, 64)),
length: BigInt.asIntN(64, record.word(cursor + 8, 64)),
});
}
return ok({
majorVersion: 4,
referenceBytes: 16,
fileOffset,
recordBytes: record.length,
recordBytes,
fileReference: readReference(record, 8, 16),
parentReference: readReference(record, 24, 16),
usn: record.readBigInt64LE(40),
reasonFlags: record.readUInt32LE(48),
sourceFlags: record.readUInt32LE(52),
usn: BigInt.asIntN(64, record.word(40, 64)),
reasonFlags: record.u32(48),
sourceFlags: record.u32(52),
timestampFileTime: null,
securityId: null,
fileAttributes: null,
name: null,
remainingExtents: record.readUInt32LE(56),
remainingExtents: record.u32(56),
extentCount,
extentSize,
omittedExtents: extentCount - extentSample.length,