fix(environment): preserve literal values and recorded comparison chronology

This commit is contained in:
morluto committed 2026-10-24 02:29:08 +08:00
1 parent e327baa7b8
commit 23ab2c4069
11 files changed
+266 -65

No files matched your search

+8 -4
View File
@@ -771,8 +771,12 @@ recording replayed in its place and never from a recording whose body the probe
two collections can reuse a live id, and an equal pair stays in that collection. A live id in a later request's path, query or body is translated before the
environment answers, a path parameter by the collection the live path names (a nested collection keeps its parent values, each decoded once, and a literal route segment stays), except a query value that pages a list, such as `limit`
or `page`. Each call is checked against the served tool's input schema first, as the MCP server checks it. After mapping, two clock values compare by format only (ISO-8601 strings, and epoch numbers under
`at`, `time`, `date`, `timestamp`, `created`, `updated` or `modified` field names, including snake-case and camel-case forms), and two different values each side generated on its own compare equal: both UUIDs, or one length of
eight or more with the same character classes and a digit. The report counts each kind of normalised value.
`at`, `time`, `date`, `timestamp`, `created`, `updated` or `modified` field names, including snake-case and camel-case forms).
Numeric array elements have no time-named field and remain literal measurements.
Different string values compare as independently generated only at entity fields
whose executed served write plan declares a unique token or UUID/token identity
generator. UUID-like or token-like spelling alone does not establish generation;
other business strings compare literally. The report counts each kind of normalised value.
ISO clock offsets are accepted only within RFC 3339's `00`–`23` hour and `00`–`59` minute ranges.
Timestamp ordering retains submillisecond precision when the source records it.
@@ -809,8 +813,8 @@ body the probe did not record compares by status only and is counted as `unobser
carries one (204, 205, 304, `HEAD`). A browser probe captured without `capture.network.response_body` is refused, as
is one that makes POST, PUT or PATCH calls without `capture.network.request_body`.
Agreement covers only the calls the probes make, after the mapping the answers allow. Values compared by form, clocks
and generated values, can still differ in content, and an equal answer after a write shows the state matched for that
Agreement covers only the calls the probes make, after the mapping the answers allow. Clocks compared by format
and fields normalized under explicit generation rules can still differ in content, and an equal answer after a write shows the state matched for that
read, not that the write had no other effect.
## Compatibility verification
@@ -74,8 +74,11 @@ credentials and resets it between sequences.
another environment value (a reference keeps raw equality, as the probe's mapping cannot tell which collection it
names); an object's members compare under the environment's mapped keys only when it is keyed by learned ids as
learning reads one, and any other object compares field by field; two clock values (ISO-8601 strings, or epoch numbers under
time-named fields) compare by format only; two different generated values (both UUIDs, or both of one length and the
same character classes with a digit) that neither side maps compare equal and are counted. A body over the replay
time-named fields) compare by format only; numeric array elements remain literal.
Different strings normalize as generated only at direct entity fields where
the executed served write plan proves a unique token or UUID/token identity
generator, and neither side participates in an ID mapping. A UUID or token
shape alone is insufficient evidence. A body over the replay
limit, text or binary, compares by status only, as does a live body the probe did not record, unless the answer has
none by definition (204, 205, 304, or an answer to `HEAD`). Both are counted.
- A step agrees when the statuses are equal and the normalised bodies are. Differences are categorised: `status`,
@@ -472,6 +472,27 @@ const compareRows = (
};
describe("ids the comparison used to misread", () => {
it.each([
["", "", true],
['""', "", false],
["null", "", true],
] as const)(
"compares a recorded missing-item body representation %j against %j",
(recorded, live, equal) => {
const probe = compareRows(
[
["GET", "/api/books", [FIRST_BOOK]],
["GET", "/api/books/1", FIRST_BOOK],
["GET", "/api/books/9", recorded, 404],
],
[
["GET", "/api/books", [{ ...FIRST_BOOK, id: 101 }]],
["GET", "/api/books/999", live, 404],
],
);
expect(probe.results[1]?.equal).toBe(equal);
},
);
it("reads a twice-encoded path id once, so book%2531 is not book1", () => {
const probe = compareRows(
[
@@ -767,6 +788,41 @@ describe("what a probe did not observe", () => {
expect(report.value.summary).toMatchObject({ steps: 2, equal: 2 });
});
it.each([
["0002", "0001"],
["002", "001"],
])(
"orders probe requests by complete fractional timestamps: %s after %s",
(readFraction, writeFraction) => {
const report = run(
analysisViewCaptureEvidence(
environmentManifestCaptureFixture([
{
...exchange(0, "GET", "http://app.example/api/books/2", 404),
startedDateTime: `2026-10-20T09:00:00.${readFraction}Z`,
response: {
status: 404,
content: {
mimeType: "application/json",
text: JSON.stringify(BOOK_NOT_FOUND),
},
},
},
{
...exchange(0, "DELETE", "http://app.example/api/books/2", 204),
startedDateTime: `2026-10-20T09:00:00.${writeFraction}Z`,
},
]),
),
);
if (!report.ok) throw report.error;
expect(report.value.summary).toMatchObject({ steps: 2, equal: 2 });
expect(
report.value.probes[0]?.results.map(({ ordinal }) => ordinal),
).toEqual([1, 0]);
},
);
it("refuses a browser probe that makes writes without recorded request bodies", () => {
const report = run(
scenarioCaptureEvidence(
@@ -157,8 +157,11 @@ const entityOf = (
return { scope, at, identity };
};
const fromState = (answer: ReplayResult): boolean =>
(answer as Partial<StateAnswer>).state?.source === "state";
const fromState = (answer: ReplayResult): boolean => {
const state = (answer as Partial<StateAnswer>).state;
// Missing-item selection comes from state, but its response is a recording.
return state?.source === "state" && state.view !== "missing";
};
/** Identity facts come from decoded JSON or text scalars, never binary presentations or truncated bodies. */
const identityValue = (body: ReplayedBody): JsonValue | undefined => {
@@ -359,7 +362,21 @@ const compareStep = (
entityAt: entity.at,
identity: entity.identity,
});
const compared = compareAnswers(live, env, run.map, entity);
const plan = run.store?.state.writes.get(call.tool.name);
const generatedFields = new Set(
fromState(answered.value) && plan?.served === true
? plan.rules.flatMap((rule) =>
rule.rule === "unique" ||
(rule.rule === "identity" && rule.generator !== "sequence")
? [rule.field]
: [],
)
: [],
);
const compared = compareAnswers(live, env, run.map, {
...entity,
generatedFields,
});
for (const key of NORMALISED_KEYS)
run.normalised[key] += compared.normalised[key];
const differences = [...compared.differences];
@@ -17,7 +17,7 @@ const compare = (
);
describe("compareAnswers", () => {
it("agrees on mapped ids, clocks of one format and generated values, and counts each", () => {
it("normalizes mapped ids and clocks while comparing unproven generated-looking values literally", () => {
const map = new IdMap();
map.learn(101, 1);
const result = compare(
@@ -35,16 +35,40 @@ describe("compareAnswers", () => {
},
map,
);
expect(result.differences).toEqual([]);
expect(
result.differences.map(({ category, pointer }) => [category, pointer]),
).toEqual([
["value", "/token"],
["value", "/uuid"],
]);
expect(result.normalised).toEqual({
ids: 1,
clocks: 1,
generated: 2,
generated: 0,
large_bodies: 0,
unobserved_bodies: 0,
});
});
it("normalizes string generation only at fields identified by an executed write plan", () => {
const result = compareAnswers(
{
status: 201,
body: { kind: "json", value: { token: "live-token", sku: "product2" } },
},
{
status: 201,
body: { kind: "json", value: { token: "env-token", sku: "product1" } },
},
new IdMap(),
{ generatedFields: new Set(["token"]) },
);
expect(result.differences).toMatchObject([
{ category: "value", pointer: "/sku" },
]);
expect(result.normalised.generated).toBe(1);
});
it("categorises status, missing and extra fields, values and clock formats", () => {
const result = compare(
{ name: "A", gone: 1, at: "2026-10-20T10:00:00.123+00:00" },
@@ -152,7 +176,7 @@ describe("compareAnswers", () => {
).toEqual([]);
});
it("reports a reordered set of scalars as order alone and reads epoch array items as clocks", () => {
it("reports scalar reordering and keeps unnamed epoch-shaped array measurements literal", () => {
expect(
compare(
{ access: ["a", "b", "c"] },
@@ -162,7 +186,7 @@ describe("compareAnswers", () => {
expect(
compare({ stats: [[1792450800, 3]] }, { stats: [[1792285200, 3]] })
.differences,
).toEqual([]);
).toMatchObject([{ category: "value", pointer: "/stats/0/0" }]);
});
});
@@ -1,6 +1,5 @@
import type { JsonObject } from "../../domain/analysisView/environmentStateRules.js";
import {
isUuid,
readClock,
type ClockFormat,
} from "../../domain/analysisView/environmentStateWriteValues.js";
@@ -74,43 +73,21 @@ const formatKey = (format: ClockFormat): string =>
? `epoch-${format.unit}`
: `iso${format.separator}${String(format.fraction)}${format.zone === "" ? "" : format.zone === "Z" ? "Z" : "offset"}`;
/** The format of a clock value: an ISO-8601 string, or an epoch number under a time-named field or as an array item. */
/** ISO text has its own clock syntax; numeric epochs require a time-named field. */
const clockFormat = (
value: JsonValue,
key: string | undefined,
): string | undefined => {
if (
typeof value === "number" &&
key !== undefined &&
!TIME_FIELD.test(key) &&
!TIME_FIELD_CAMEL.test(key)
(key === undefined ||
(!TIME_FIELD.test(key) && !TIME_FIELD_CAMEL.test(key)))
)
return undefined;
const clock = readClock(value);
return clock === undefined ? undefined : formatKey(clock.format);
};
const characterClasses = (text: string): string =>
[
...new Set(
text
.replaceAll(/\d/gu, "0")
.replaceAll(/[a-z]/gu, "a")
.replaceAll(/[A-Z]/gu, "A"),
),
]
.sort()
.join("");
/** Two values each side generated on its own: both UUIDs, or one length of eight or more with the same character classes and a digit. */
const generatedPair = (live: string, env: string): boolean =>
(isUuid(live) && isUuid(env)) ||
(live.length === env.length &&
live.length >= 8 &&
/\d/u.test(live) &&
/\d/u.test(env) &&
characterClasses(live) === characterClasses(env));
class Differ {
readonly differences: Difference[] = [];
readonly normalised: Normalised = {
@@ -128,6 +105,7 @@ class Differ {
readonly scope: string | undefined;
readonly at: string;
readonly identity: boolean;
readonly generatedFields: ReadonlySet<string>;
},
) {}
@@ -211,9 +189,11 @@ class Differ {
if (
typeof live === "string" &&
typeof env === "string" &&
at.own === true &&
key !== undefined &&
this.entity.generatedFields.has(key) &&
!this.map.hasLive(live) &&
!this.map.hasEnv(env) &&
generatedPair(live, env)
!this.map.hasEnv(env)
) {
this.normalised.generated += 1;
return;
@@ -375,12 +355,15 @@ export const compareAnswers = (
readonly scope?: string;
readonly at?: string;
readonly identity?: boolean;
/** Direct entity fields whose executed write plan proves independent generation. */
readonly generatedFields?: ReadonlySet<string>;
} = {},
): { readonly differences: Difference[]; readonly normalised: Normalised } => {
const differ = new Differ(map, {
scope: entity.scope,
at: entity.at ?? "",
identity: entity.identity === true,
generatedFields: entity.generatedFields ?? new Set(),
});
if (live.status !== env.status)
differ.add({
@@ -27,6 +27,41 @@ const compareRows = (
};
describe("environment comparison identifier semantics", () => {
it("reports changed business tokens and numeric arrays through the application comparison boundary", () => {
const probe = compareRows(
[
[
"GET",
"/api/measurements",
{ sku: "product1", samples: [1700000000] },
],
],
[
[
"GET",
"/api/measurements",
{ sku: "product2", samples: [1800000000] },
],
],
);
expect(probe.results[0]?.equal).toBe(false);
expect(
probe.results[0]?.differences.map(({ category, pointer }) => [
category,
pointer,
]),
).toEqual([
["value", "/sku"],
["value", "/samples/0"],
]);
expect(probe.normalised.generated).toBe(0);
expect(probe.normalised.clocks).toBe(0);
const same = compareRows(
[["GET", "/api/measurements", { samples: [1700000000] }]],
[["GET", "/api/measurements", { samples: [1700000000] }]],
);
expect(same.results[0]?.equal).toBe(true);
});
it("compares numeric format identifiers literally while normalizing actual clock fields", () => {
const fourcc = (value: string) => Buffer.from(value).readUInt32LE(0);
const probe = compareRows(
@@ -1,4 +1,8 @@
import { z } from "zod";
import {
compareClockInstants,
readClock,
} from "../../domain/analysisView/environmentStateWriteValues.js";
import { isJsonLike } from "../../domain/analysisView/EnvironmentManifestAccumulator.js";
import {
primaryOrigin,
@@ -46,10 +50,11 @@ const postDataSchema = z.object({
const methodSchema = z.object({ request: z.object({ method: z.string() }) });
const startedMs = (record: WebNetworkCaptureRecord): number | undefined => {
const startedClock = (
record: WebNetworkCaptureRecord,
): ReturnType<typeof readClock> => {
const started = startedSchema.safeParse(record.reported);
const ms = started.success ? Date.parse(started.data.startedDateTime) : NaN;
return Number.isFinite(ms) ? ms : undefined;
return started.success ? readClock(started.data.startedDateTime) : undefined;
};
/** Whether a probe makes calls that carry a request body: a POST, PUT or PATCH. */
@@ -118,13 +123,17 @@ export const probeRequests = (
const unobserved: UnobservedRecord[] = [];
// Without a start time for every record the probe keeps its recorded order, as state extraction does.
const starts = new Map(
records.map((record) => [record.ordinal, startedMs(record)]),
records.map((record) => [record.ordinal, startedClock(record)]),
);
const timed = [...starts.values()].every((ms) => ms !== undefined);
const ordered = [...records].sort(
(left, right) =>
(timed ? starts.get(left.ordinal)! - starts.get(right.ordinal)! : 0) ||
left.ordinal - right.ordinal,
(timed
? compareClockInstants(
starts.get(left.ordinal)!,
starts.get(right.ordinal)!,
)
: 0) || left.ordinal - right.ordinal,
);
const lines = new Map(
records.map((record) => [
@@ -1,4 +1,8 @@
import { z } from "zod";
import {
compareClockInstants,
readClock,
} from "./environmentStateWriteValues.js";
import type { AnalysisError } from "../analysisErrorBase.js";
import { parseExactJson } from "../exactJsonNumber.js";
import { MAX_JSON_DEPTH, type JsonValue } from "../jsonValue.js";
@@ -111,6 +115,7 @@ interface RecordEntry {
readonly origin: string | null;
readonly contentType: string;
readonly timed: boolean;
readonly subMillisecond: number;
readonly record: WebNetworkCaptureRecord;
}
@@ -155,16 +160,21 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
const line = readRecordedRequestLine(record.reported, true, this.scratch);
if (line === null) return this.overBudget();
const timing = timingSchema.safeParse(record.reported);
const start = timing.success
? Date.parse(timing.data.startedDateTime)
: Number.NaN;
const timed = timing.success && Number.isFinite(start);
const clock = timing.success
? readClock(timing.data.startedDateTime)
: undefined;
const start = clock?.ms ?? record.ordinal;
const timed = timing.success && clock !== undefined;
if (line.kind === "excluded")
return line.reason === "url-redacted"
? this.#redactedWrite(
record,
timing.success && timed
? { start, end: start + timing.data.time }
? {
start,
end: start + timing.data.time,
subMillisecond: clock.subMillisecond,
}
: undefined,
)
: ok(undefined);
@@ -195,6 +205,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
origin: line.origin,
contentType: line.rawMime,
timed,
subMillisecond: clock?.subMillisecond ?? 0,
record,
});
return ok(undefined);
@@ -206,7 +217,13 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
/** Keep a successful write whose URL was redacted, with an empty path whose root relates to every type. */
#redactedWrite(
record: WebNetworkCaptureRecord,
timing: { readonly start: number; readonly end: number } | undefined,
timing:
| {
readonly start: number;
readonly end: number;
readonly subMillisecond: number;
}
| undefined,
): Result<void, AnalysisError> {
const parsed = redactedWriteSchema.safeParse(record.reported);
if (!parsed.success) return ok(undefined);
@@ -228,6 +245,7 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
origin: null,
contentType: "",
timed: timing !== undefined,
subMillisecond: timing?.subMillisecond ?? 0,
record,
});
return ok(undefined);
@@ -307,12 +325,15 @@ export class EnvironmentStateAccumulator extends EnvironmentManifestAccumulator
? "recorded-start-time"
: "record-order";
const records = entries
.map((entry) => entry.facts)
.sort((left, right) =>
timeline === "record-order"
? left.ordinal - right.ordinal
: left.start - right.start || left.ordinal - right.ordinal,
? left.facts.ordinal - right.facts.ordinal
: compareClockInstants(
{ ms: left.facts.start, subMillisecond: left.subMillisecond },
{ ms: right.facts.start, subMillisecond: right.subMillisecond },
) || left.facts.ordinal - right.facts.ordinal,
)
.map((entry) => entry.facts)
.map((facts) =>
timeline === "record-order"
? { ...facts, start: facts.ordinal, end: facts.ordinal }
@@ -53,6 +53,9 @@ it.each<[string, number, ClockFormat, number?]>([
);
it.each([
"0001-02-29T00:00:00Z",
"0099-02-29T00:00:00Z",
"0100-02-29T00:00:00Z",
"2026-02-30T00:00:00Z",
"2026-02-29T00:00:00Z",
"2026-13-01T00:00:00Z",
@@ -72,6 +75,30 @@ it.each([
expect(readClock(text)).toBeUndefined();
});
it.each(["0000", "0001", "0099", "0100"])(
"retains first-century ISO year %s without Date.UTC's adjustment",
(year) => {
const value = `${year}-01-01T00:00:00Z`;
expect(readClock(value)).toBeDefined();
expect(respellClock(value, iso(" ", 3, "Z"))).toBe(
`${year}-01-01 00:00:00.000Z`,
);
},
);
it.each(["𐐀42𐐁", "𝟙42𝟚", "é42é", "A42B"])(
"keeps token42 embedded in complete Unicode letters or digits: %s",
(text) => {
expect(delimitedOffsets(text, "42")).toEqual([]);
expect(replaceDelimited(text, "42", "43")).toBe(text);
},
);
it("preserves UTF-16 offsets while accepting emoji as a delimiter", () => {
expect(delimitedOffsets("😀42😀", "42")).toEqual([2]);
expect(replaceDelimited("😀42😀", "42", "43")).toBe("😀43😀");
});
it.each<[JsonValue | undefined, ReturnType<typeof readClock>]>([
[
WHOLE_SECOND / 1000,
@@ -18,6 +18,12 @@ interface ClockReading {
readonly format: ClockFormat;
}
/** Compare recorded instants without rounding away their submillisecond digits. */
export const compareClockInstants = (
left: Pick<ClockReading, "ms" | "subMillisecond">,
right: Pick<ClockReading, "ms" | "subMillisecond">,
): number => left.ms - right.ms || left.subMillisecond - right.subMillisecond;
const ISO_INSTANT =
/^(\d{4})-(\d{2})-(\d{2})([T ])(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:?\d{2})?$/u;
const MINUTE_MS = 60_000;
@@ -56,23 +62,24 @@ const readIsoClock = (value: string): ClockReading | undefined => {
const zone = match[9] ?? "";
const offsetMinutes = zoneMinutes(zone);
if (offsetMinutes === undefined) return undefined;
const local = Date.UTC(
year as number,
(month as number) - 1,
day as number,
// setUTCFullYear avoids Date.UTC's special 1900 adjustment for years 0–99.
const rolled = new Date(0);
rolled.setUTCFullYear(year as number, (month as number) - 1, day as number);
rolled.setUTCHours(
hour as number,
minute as number,
second as number,
Number(`${fraction}000`.slice(0, 3)),
);
// Date.UTC rolls invalid fields over; the round trip refuses them.
const rolled = new Date(local);
const local = rolled.getTime();
// Date setters roll invalid fields over; the round trip refuses them.
if (
rolled.getUTCFullYear() !== year ||
rolled.getUTCMonth() !== (month as number) - 1 ||
rolled.getUTCDate() !== day ||
rolled.getUTCHours() !== hour ||
rolled.getUTCMinutes() !== minute
rolled.getUTCMinutes() !== minute ||
rolled.getUTCSeconds() !== second
)
return undefined;
return {
@@ -256,6 +263,20 @@ export const slugOf = (text: string): string =>
const isAlphanumeric = (character: string | undefined): boolean =>
character !== undefined && /[\p{L}\p{N}]/u.test(character);
/** Source offsets stay UTF-16, while delimiter classification reads complete code points. */
const previousCharacter = (text: string, at: number): string | undefined => {
if (at === 0) return undefined;
const last = text.charCodeAt(at - 1);
return last >= 0xdc00 && last <= 0xdfff && at > 1
? text.slice(at - 2, at)
: text[at - 1];
};
const nextCharacter = (text: string, at: number): string | undefined => {
const point = text.codePointAt(at);
return point === undefined ? undefined : String.fromCodePoint(point);
};
/** Start offsets of `token` in `text` where neither neighbour is a letter or digit. */
export const delimitedOffsets = (text: string, token: string): number[] => {
const offsets: number[] = [];
@@ -263,7 +284,8 @@ export const delimitedOffsets = (text: string, token: string): number[] => {
let at = text.indexOf(token);
while (at >= 0) {
const delimited =
!isAlphanumeric(text[at - 1]) && !isAlphanumeric(text[at + token.length]);
!isAlphanumeric(previousCharacter(text, at)) &&
!isAlphanumeric(nextCharacter(text, at + token.length));
if (delimited) offsets.push(at);
// A rejected match may overlap a delimited one, so the search moves on by one character.
at = text.indexOf(token, at + (delimited ? token.length : 1));