diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 8ac2f6f9f4..cae068f6d7 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -189,7 +189,8 @@ export const inspectAnalysisViewValidated = ( ) : undefined; const stateAccumulator = - input.view.kind === "environment-state" + input.view.kind === "environment-state" || + input.view.kind === "environment-coverage" ? new EnvironmentStateAccumulator(undefined, input.view.template_rules) : undefined; const accumulator = @@ -377,8 +378,9 @@ const createAnalysisViewEvidence = ( rawResult: null, confidence: // Grouping is whole-capture, so any templated group or parameter makes the result inferred; - // entity types and identity in an extracted state always are. + // entity types and identity in an extracted state, and the coverage summarizing it, always are. result.kind === "environment-state" || + result.kind === "environment-coverage" || ((result.kind === "api-surface" || result.kind === "environment-manifest") && result.templated_groups > 0) diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index c58952cc1e..1a661d2fb5 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -220,6 +220,11 @@ import { environmentStateViewRequestSchema, projectEnvironmentStateView, } from "./environmentStateView.js"; +import { + environmentCoverageFields, + environmentCoverageViewRequestSchema, + projectEnvironmentCoverageView, +} from "./environmentCoverageView.js"; import { pcapViewRequestSchema, pcapViewPageSchema, @@ -357,6 +362,7 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ apiSurfaceViewSchema, environmentManifestViewRequestSchema, environmentStateViewRequestSchema, + environmentCoverageViewRequestSchema, captureViewRequestSchema, protobufViewRequestSchema, qlogViewRequestSchema, @@ -820,6 +826,16 @@ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ ]), view: environmentStateViewRequestSchema, }), + z.strictObject({ + ...viewResultBase, + ...environmentCoverageFields, + kind: z.literal("environment-coverage"), + parent_operation: z.enum([ + "inspect_web_network_capture", + "capture_browser_scenario", + ]), + view: environmentCoverageViewRequestSchema, + }), z.discriminatedUnion("parent_operation", [ nativeExportsSummaryViewResultSchema, byteEntropySummaryViewResultSchema, @@ -1090,7 +1106,7 @@ const unsupportedParent = ( INSPECT_ANALYSIS_VIEW_OPERATION, parentArtifactPath(parent), parent.operation === "capture_browser_scenario" - ? `Parent Evidence operation "capture_browser_scenario" supports only the environment-manifest and environment-state views.` + ? `Parent Evidence operation "capture_browser_scenario" supports only the environment-manifest, environment-state and environment-coverage views.` : `Parent Evidence operation ${JSON.stringify(parent.operation)} does not support analysis views. Use completed Evidence from a supported parent operation with a compatible view. For a summary receipt, use its normalized_result.parent_evidence_id rather than the receipt's own evidence_id.`, ); @@ -1151,9 +1167,16 @@ export interface AnalysisViewAccumulators { readonly state?: EnvironmentStateAccumulator | undefined; } +/** Views that group a capture's records into tools. */ +export type GroupingViewKind = + | "api-surface" + | "environment-manifest" + | "environment-state" + | "environment-coverage"; + type GroupingViewRequest = Extract< AnalysisViewRequest, - { kind: "api-surface" | "environment-manifest" | "environment-state" } + { kind: GroupingViewKind } >; const isGroupingView = ( @@ -1161,7 +1184,8 @@ const isGroupingView = ( ): view is GroupingViewRequest => view.kind === "api-surface" || view.kind === "environment-manifest" || - view.kind === "environment-state"; + view.kind === "environment-state" || + view.kind === "environment-coverage"; /** Views that group a capture's records, each under the budget its caller created. */ const projectGroupingView = ( @@ -1177,6 +1201,8 @@ const projectGroupingView = ( ); if (view.kind === "environment-manifest") return projectEnvironmentManifestView(parent, view, accumulators.manifest); + if (view.kind === "environment-coverage") + return projectEnvironmentCoverageView(parent, view, accumulators.state); return projectEnvironmentStateView(parent, view, accumulators.state); }; diff --git a/src/domain/analysisView/binaryLayoutView.ts b/src/domain/analysisView/binaryLayoutView.ts index b441524ddd..09128066e6 100644 --- a/src/domain/analysisView/binaryLayoutView.ts +++ b/src/domain/analysisView/binaryLayoutView.ts @@ -9,6 +9,7 @@ import type { AnalysisViewParent, AnalysisViewRequest, UnsignedAnalysisView, + GroupingViewKind, } from "./analysisView.js"; import { boundedViewMessages, @@ -157,10 +158,7 @@ const layoutSummary = (layout: BinaryLayout) => ({ export const projectBinaryLayoutView = ( parent: AnalysisViewParent, layout: BinaryLayout, - view: Exclude< - AnalysisViewRequest, - { kind: "api-surface" | "environment-manifest" | "environment-state" } - >, + view: Exclude, ): Result => { const shared = parentFields(parent, layout); if (view.kind === "native") diff --git a/src/domain/analysisView/environmentCoverageView.ts b/src/domain/analysisView/environmentCoverageView.ts new file mode 100644 index 0000000000..bbbf71ca4a --- /dev/null +++ b/src/domain/analysisView/environmentCoverageView.ts @@ -0,0 +1,181 @@ +import { z } from "zod"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { ok, type Result } from "../result.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { EnvironmentStateAccumulator } from "./EnvironmentStateAccumulator.js"; +import { templateRulesSchema } from "./environmentManifestView.js"; +import type { StateView } from "./environmentStateRules.js"; +import { + environmentStateFields, + projectEnvironmentStateView, +} from "./environmentStateView.js"; +import type { + StateWritePlan, + WriteEffect, + WriteReason, +} from "./environmentStateWriteRules.js"; + +export const environmentCoverageViewRequestSchema = z.strictObject({ + kind: z.literal("environment-coverage"), + template_rules: templateRulesSchema.describe( + "Opt-in template rules as for environment-manifest, each listed at most once. Use the rules of the manifest the environment serves so tool names match.", + ), +}); + +/** What to record next for each reason a write tool is withheld; advice only, it never changes what is served. */ +export const WITHHELD_WRITE_HINTS = { + "no-state-target": + "The write's path names no collection or item the capture reads. Read the collection or item it changes.", + "no-call": + "No recorded call of this tool was modelled, for example because every call failed. Record a successful call.", + "body-unusable": "The request body is not a usable JSON object.", + "no-observation": + "Nothing showed the written entity. Read it back by id right after the write.", + "unexplained-change": + "The write changed fields it did not send, in a way no rule explains. A second, different call of the same tool can show the pattern.", + contradicted: + "A later read disagreed with the modelled effect. Check the read at contradicted_by.", + "side-effect": + "The write changed something besides its own entity, which the environment does not model.", + unconfirmed: + "No later read showed the effect. Read the entity and its list after the write.", + "incomplete-entities": + "The write is confirmed, but the views that show its entity cannot render it in full. Read the written entity through every view that lists it.", +} as const satisfies Record; + +const writeReasons = Object.keys(WITHHELD_WRITE_HINTS) as [ + WriteReason, + ...WriteReason[], +]; +const effectSchema = z.enum(["create", "update", "delete", "document", "none"]); +const ordinalSchema = z.number().int().nonnegative(); + +/** Which tools an environment answers from its state, which replay their recordings, and why each write is withheld. */ +export const environmentCoverageSchema = z.strictObject({ + reads: z.strictObject({ + state: z + .number() + .int() + .nonnegative() + .describe("Read tools answered from the state."), + recorded: z + .number() + .int() + .nonnegative() + .describe("Read tools that replay their recordings."), + recorded_reasons: z + .record(z.string(), z.number().int().positive()) + .describe( + "Replayed read tools per reason, as the environment-state view names the reasons.", + ), + }), + writes: z.strictObject({ + served: z.array(z.strictObject({ tool: z.string(), effect: effectSchema })), + withheld: z.array( + z.strictObject({ + tool: z.string(), + effect: effectSchema, + reason: z.enum(writeReasons), + hint: z + .string() + .describe( + "What to record next; advice that never changes what is served.", + ), + exemplar_ordinal: ordinalSchema + .nullable() + .describe( + "The first successful recorded call the plan was built from.", + ), + contradicted_by: ordinalSchema + .nullable() + .describe("The read that disagreed with the modelled effect."), + }), + ), + }), +}); + +export type EnvironmentCoverage = z.infer; + +export const environmentCoverageFields = { + origin: environmentStateFields.origin, + name: environmentStateFields.name, + ...environmentCoverageSchema.shape, + unknowns: environmentStateFields.unknowns, +}; + +/** Summarize a state's read views and write plans as served, replayed and withheld tools. */ +export const summarizeEnvironmentCoverage = (state: { + readonly views: Iterable; + readonly writes: Iterable; +}): EnvironmentCoverage => { + let answered = 0; + let recorded = 0; + const reasons: Record = {}; + for (const view of state.views) + if (view.shape === "recorded") { + recorded++; + reasons[view.reason] = (reasons[view.reason] ?? 0) + 1; + } else answered++; + const served: { tool: string; effect: WriteEffect }[] = []; + const withheld: EnvironmentCoverage["writes"]["withheld"] = []; + for (const plan of state.writes) + if (plan.served || plan.reason === null) + served.push({ tool: plan.tool, effect: plan.effect }); + else + withheld.push({ + tool: plan.tool, + effect: plan.effect, + reason: plan.reason, + hint: WITHHELD_WRITE_HINTS[plan.reason], + exemplar_ordinal: plan.exemplar_ordinal, + contradicted_by: plan.contradicted_by, + }); + return { + reads: { + state: answered, + recorded, + recorded_reasons: Object.fromEntries( + Object.entries(reasons).sort(([left], [right]) => + left < right ? -1 : left > right ? 1 : 0, + ), + ), + }, + writes: { served, withheld }, + }; +}; + +/** Project the state an environment would serve and summarize which tools it answers from that state. */ +export const projectEnvironmentCoverageView = ( + parent: AnalysisViewParent, + view: Extract, + accumulator = new EnvironmentStateAccumulator(undefined, view.template_rules), +): Result => { + const projected = projectEnvironmentStateView( + parent, + { kind: "environment-state", template_rules: view.template_rules }, + accumulator, + ); + if (!projected.ok) return projected; + const state = projected.value; + return ok({ + kind: "environment-coverage", + view, + parent_evidence_id: state.parent_evidence_id, + parent_operation: state.parent_operation, + parent_digest: state.parent_digest, + artifact: state.artifact, + coverage: state.coverage, + limitations: [ + ...state.limitations, + "Coverage summarizes the environment-state view the environment derives at startup; it does not run the environment. Hints are advice about what to record next.", + ], + unknowns: state.unknowns, + origin: state.origin, + name: state.name, + ...summarizeEnvironmentCoverage(state), + }); +}; diff --git a/src/domain/analysisView/environmentStateView.ts b/src/domain/analysisView/environmentStateView.ts index 7cc8b1f80e..fbb83c9982 100644 --- a/src/domain/analysisView/environmentStateView.ts +++ b/src/domain/analysisView/environmentStateView.ts @@ -392,7 +392,10 @@ export const projectEnvironmentStateView = ( parent: AnalysisViewParent, view: Extract, accumulator = new EnvironmentStateAccumulator(undefined, view.template_rules), -): Result => { +): Result< + Extract, + AnalysisError +> => { const grouped = groupCaptureRecords(parent, view.kind, view.template_rules, { accumulator, noun: "state", diff --git a/src/domain/analysisView/javascriptApplicationView.ts b/src/domain/analysisView/javascriptApplicationView.ts index 912c93c9ae..eb38371aff 100644 --- a/src/domain/analysisView/javascriptApplicationView.ts +++ b/src/domain/analysisView/javascriptApplicationView.ts @@ -23,6 +23,7 @@ import type { AnalysisViewParent, AnalysisViewRequest, UnsignedAnalysisView, + GroupingViewKind, } from "./analysisView.js"; const OPERATION = "inspect_analysis_view"; @@ -340,10 +341,7 @@ const selectModule = ( export const projectJavaScriptApplicationView = ( parent: AnalysisViewParent, analysis: JavaScriptApplicationAnalysisResult, - view: Exclude< - AnalysisViewRequest, - { kind: "api-surface" | "environment-manifest" | "environment-state" } - >, + view: Exclude, ): Result => { if ( view.kind !== "summary" && diff --git a/src/domain/analysisView/webNetworkCaptureView.ts b/src/domain/analysisView/webNetworkCaptureView.ts index 0439ab3efe..d236af38ab 100644 --- a/src/domain/analysisView/webNetworkCaptureView.ts +++ b/src/domain/analysisView/webNetworkCaptureView.ts @@ -458,7 +458,7 @@ export const projectWebNetworkCaptureView = ( path: ["view"], reason: "invalid_value", message: - "Historical captures support summary, capture record pages, qlog fact pages and admitted api-surface, environment-manifest or environment-state views.", + "Historical captures support summary, capture record pages, qlog fact pages and admitted api-surface, environment-manifest, environment-state or environment-coverage views.", }, ]), );