diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index f91f003e42..df639a6732 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -5,9 +5,11 @@ import java.io.BufferedReader; import java.io.BufferedWriter; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.SocketTimeoutException; import java.net.StandardProtocolFamily; import java.net.UnixDomainSocketAddress; import java.nio.channels.Channels; @@ -81,6 +83,7 @@ import ghidra.program.model.listing.Function; import ghidra.program.model.listing.FunctionIterator; import ghidra.program.model.listing.Instruction; import ghidra.program.model.listing.InstructionIterator; +import ghidra.program.model.listing.Listing; import ghidra.program.model.listing.Variable; import ghidra.program.model.mem.MemoryBlock; import ghidra.program.model.mem.MemoryBlockSourceInfo; @@ -297,7 +300,7 @@ public final class ReaGhidraBridge extends HeadlessScript { Files.move(pendingPath, endpointPath); } } - acceptClient(server, descriptor); + acceptAuthenticatedLoopbackClient(server, descriptor); } finally { Files.deleteIfExists(pendingPath); @@ -309,6 +312,50 @@ public final class ReaGhidraBridge extends HeadlessScript { return System.getProperty("os.name", "").startsWith("Windows"); } + private static String readLoopbackHandshake( + SocketChannel client, int maxBytes, long timeoutMillis) throws IOException { + long deadline = System.nanoTime() + timeoutMillis * 1_000_000L; + InputStream input = client.socket().getInputStream(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + while (true) { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) throw new SocketTimeoutException("Bridge authentication timed out"); + client.socket().setSoTimeout((int) Math.max(1, + Math.min(Integer.MAX_VALUE, (remaining + 999_999L) / 1_000_000L))); + int value = input.read(); + if (value == -1) return null; + if (value == '\n') return bytes.toString(StandardCharsets.UTF_8); + if (bytes.size() == maxBytes) throw new IOException("Bridge authentication frame is too large"); + bytes.write(value); + } + } + + private void acceptAuthenticatedLoopbackClient( + ServerSocketChannel server, SessionDescriptor descriptor) throws Exception { + while (true) { + try (SocketChannel client = server.accept()) { + Request first; + try { + String line = readLoopbackHandshake(client, 64 * 1024, 1_000); + if (line == null) continue; + first = parseRequest(line, descriptor.token); + } + catch (IOException | RuntimeException rejected) { + // A rejected unauthenticated peer cannot consume the bridge session. + continue; + } + client.socket().setSoTimeout(0); + try (BufferedReader reader = new BufferedReader( + Channels.newReader(client, StandardCharsets.UTF_8)); + BufferedWriter writer = new BufferedWriter( + Channels.newWriter(client, StandardCharsets.UTF_8))) { + serveClient(descriptor, reader, writer, first); + } + return; + } + } + } + private void acceptClient( ServerSocketChannel server, SessionDescriptor descriptor) throws Exception { @@ -319,26 +366,28 @@ public final class ReaGhidraBridge extends HeadlessScript { BufferedWriter writer = new BufferedWriter( Channels.newWriter(client, StandardCharsets.UTF_8) )) { - serveClient(descriptor, reader, writer); + serveClient(descriptor, reader, writer, null); } } private void serveClient( SessionDescriptor descriptor, BufferedReader reader, - BufferedWriter writer) throws Exception { + BufferedWriter writer, + Request first) throws Exception { while (true) { - String line = reader.readLine(); - if (line == null) { - return; - } - Request request; - try { - request = parseRequest(line, descriptor.token); - } - catch (RuntimeException exception) { - writeFailure(writer, 1, "invalid_request", "Bridge request is invalid"); - return; + Request request = first; + first = null; + if (request == null) { + String line = reader.readLine(); + if (line == null) return; + try { + request = parseRequest(line, descriptor.token); + } + catch (RuntimeException exception) { + writeFailure(writer, 1, "invalid_request", "Bridge request is invalid"); + return; + } } try { if (request.method.equals("shutdown")) { @@ -405,7 +454,9 @@ public final class ReaGhidraBridge extends HeadlessScript { case "xrefs" -> xrefs(request.params); case "analyze_function" -> { FunctionAnalysisObservation observation = analyzeFunctionObservation(request.params); - yield functionObservation(observation.dossier(), observation.limitations()); + JsonObject result = functionObservation(observation.dossier(), observation.limitations()); + result.add("relationship_coverage", observation.relationshipCoverage()); + yield result; } case "set_address_name", "set_addresses_names" -> { if (!descriptor.transport.equals("unix-socket")) @@ -884,6 +935,7 @@ public final class ReaGhidraBridge extends HeadlessScript { JsonObject result = functionObservation( value == null ? JsonNull.INSTANCE : GSON.toJsonTree(value), noReturnCallLimitations( + function, currentProgram.getListing().getInstructions(function.getBody(), true) ) ); @@ -975,7 +1027,7 @@ public final class ReaGhidraBridge extends HeadlessScript { InstructionScan scan = scanInstructions(function, Integer.MAX_VALUE); return functionObservation( GSON.toJsonTree(renderAssembly(scan.instructions)), - noReturnCallLimitations(scan.instructions) + noReturnCallLimitations(function, scan.instructions) ); } @@ -994,7 +1046,7 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "The fast path does not invoke the decompiler or scan whole-program names and strings." ); - List boundaryLimitations = noReturnCallLimitations(scan.instructions); + List boundaryLimitations = noReturnCallLimitations(function, scan.instructions); for (String limitation : boundaryLimitations) limitations.add(limitation); JsonObject result = new JsonObject(); result.add("procedure", procedureIdentity(function)); @@ -1016,12 +1068,19 @@ public final class ReaGhidraBridge extends HeadlessScript { for (Function item : ordered) { result.add(canonicalAddress(item.getEntryPoint())); } - return functionObservation( - result, - noReturnCallLimitations( + if (callers) { + JsonObject observation = functionObservation(result, noReturnCallLimitations(function, currentProgram.getListing().getInstructions(function.getBody(), true) - ) + )); + observation.add("relationship_coverage", callRelationshipCoverage(null, + unrepresentedIncomingCalls(function, observed))); + monitor.checkCancelled(); + return observation; + } + CallBoundaryObservation calls = callBoundaryObservation( + function, currentProgram.getListing().getInstructions(function.getBody(), true), observed, false, null ); + return callRelationshipObservation(result, calls); } private JsonObject procedureInfo(JsonObject params) throws Exception { @@ -1041,6 +1100,7 @@ public final class ReaGhidraBridge extends HeadlessScript { return functionObservation( result, noReturnCallLimitations( + function, currentProgram.getListing().getInstructions(function.getBody(), true) ) ); @@ -1071,18 +1131,10 @@ public final class ReaGhidraBridge extends HeadlessScript { result.addProperty("direction", direction); result.add("references", edges); result.addProperty("reference_kinds_available", true); - JsonArray unresolvedCalls = new JsonArray(); - if (direction.equals("outgoing")) for (Instruction instruction : scanInstructions(function, Integer.MAX_VALUE).instructions) { - if (!instruction.getFlowType().isCall()) continue; - boolean resolved = false; - for (Reference reference : instruction.getReferencesFrom()) if (reference.getReferenceType().isCall()) resolved = true; - if (!resolved) { - JsonObject site = new JsonObject(); - site.addProperty("address", canonicalAddress(instruction.getAddress())); - site.addProperty("reason", "Instruction decoder identifies a call but ReferenceManager supplies no call destination"); - unresolvedCalls.add(site); - } - } + JsonArray unresolvedCalls = direction.equals("outgoing") + ? callBoundaryObservation( + function, currentProgram.getListing().getInstructions(function.getBody(), true), null, true, references + ).unresolvedCalls() : new JsonArray(); result.add("unresolved_calls", unresolvedCalls); return result; } @@ -1490,16 +1542,28 @@ public final class ReaGhidraBridge extends HeadlessScript { InstructionScan scan = scanInstructions(function, Integer.MAX_VALUE); DecompileResults decompilation = decompile(function); String pseudocode = decompiledText(decompilation); - List incomingReferences = collectReferences(function, "incoming") - .stream() + JsonObject availability = new JsonObject(); + availability.add("pseudocode", facetAvailability( + pseudocode != null, + function.isExternal() ? "Ghidra does not decompile external functions." + : function.getBody().isEmpty() ? "Ghidra has no function body to decompile." + : "Ghidra completed decompilation without returning pseudocode." + )); + if (pseudocode == null) { + pseudocode = ""; + } + List bodyIncoming = collectReferences(function, "incoming"); + List incomingReferences = bodyIncoming.stream() .filter(reference -> !function.getBody().contains(reference.getFromAddress())) .toList(); List outgoingReferences = collectReferences(function, "outgoing"); JsonArray incoming = referenceEdges(incomingReferences); JsonArray outgoing = referenceEdges(outgoingReferences); - JsonArray comments = comments(scan.instructions); - JsonArray callers = procedureIdentities(function.getCallingFunctions(monitor)); - JsonArray callees = procedureIdentities(function.getCalledFunctions(monitor)); + JsonArray comments = comments(function); + Set callerFunctions = function.getCallingFunctions(monitor); + JsonArray callers = procedureIdentities(callerFunctions); + Set calleeFunctions = function.getCalledFunctions(monitor); + JsonArray callees = procedureIdentities(calleeFunctions); JsonArray referencedStrings = referencedStrings(outgoingReferences); JsonArray referencedNames = referencedNames(outgoingReferences); JsonArray blocks = basicBlockValues(function); @@ -1511,7 +1575,7 @@ public final class ReaGhidraBridge extends HeadlessScript { procedure.add("locals", functionLocals(function)); JsonObject result = new JsonObject(); result.add("procedure", procedure); - result.addProperty("pseudocode", pseudocode == null ? "" : pseudocode); + result.addProperty("pseudocode", pseudocode); result.add("assembly", assembly); result.add("comments", comments); result.add("callers", callers); @@ -1520,20 +1584,6 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("outgoing_references", outgoing); result.add("referenced_strings", referencedStrings); result.add("referenced_names", referencedNames); - JsonObject facets = new JsonObject(); - JsonObject pseudocodeAvailability = new JsonObject(); - pseudocodeAvailability.addProperty("available", pseudocode != null); - if (pseudocode == null) { - pseudocodeAvailability.addProperty("reason", "Ghidra did not supply pseudocode."); - } - facets.add("pseudocode", pseudocodeAvailability); - for (String facet : List.of("assembly", "comments", "callers", "callees", - "references", "strings_names", "cfg")) { - JsonObject available = new JsonObject(); - available.addProperty("available", true); - facets.add(facet, available); - } - result.add("facet_availability", facets); boolean swift = swiftProgram(); boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; @@ -1547,6 +1597,15 @@ public final class ReaGhidraBridge extends HeadlessScript { nativeApiBoundary(function, decompilation) ); result.add("native_value_flow", nativeValueFlow(decompilation)); + // Successful full queries distinguish collected empty sets from missing facts. + // Availability does not claim that Ghidra resolved every runtime relationship. + for (String facet : List.of("comments", "callers", "callees", "references", "strings_names", "cfg")) { + availability.add(facet, facetAvailability(true, null)); + } + availability.add("assembly", facetAvailability( + !scan.truncated(), "The function instruction scan was truncated." + )); + result.add("facet_availability", availability); JsonArray limitations = new JsonArray(); limitations.add( "Unresolved computed or indirect flows without target addresses are not represented as reference edges." @@ -1562,7 +1621,7 @@ public final class ReaGhidraBridge extends HeadlessScript { ); if (swift && aarch64) { limitations.add( - "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported." + "Swift String literals are decoded from constant register pairs at calls, returns and paired stores. Literals stored one word at a time, chosen by conditional selects, or built at runtime are not reported; candidate pairs at unresolved jumps are retained as unknown omissions." ); if (swiftLiterals.omissions().size() > 0) { limitations.add( @@ -1574,8 +1633,10 @@ public final class ReaGhidraBridge extends HeadlessScript { "Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture." ); } - List boundaryLimitations = noReturnCallLimitations(scan.instructions); - for (String limitation : boundaryLimitations) { + CallBoundaryObservation calls = callBoundaryObservation( + function, scan.instructions, calleeFunctions, true, outgoingReferences); + result.add("unresolved_calls", calls.unresolvedCalls()); + for (String limitation : calls.limitations()) { limitations.add(limitation); } if (decompilation != null && decompilation.getErrorMessage() != null && @@ -1583,7 +1644,14 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add("Ghidra decompiler diagnostic: " + decompilation.getErrorMessage()); } result.add("limitations", limitations); - return new FunctionAnalysisObservation(result, boundaryLimitations); + monitor.checkCancelled(); + int missingCallers = 0; + for (Reference reference : bodyIncoming) { + monitor.checkCancelled(); + if (unrepresentedCaller(reference, callerFunctions)) missingCallers++; + } + return new FunctionAnalysisObservation(result, calls.limitations(), + callRelationshipCoverage(calls, missingCallers)); } private JsonObject nativeValueFlow(DecompileResults decompilation) throws Exception { @@ -3287,54 +3355,276 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } - private List noReturnCallLimitations(Iterable instructions) + private static JsonObject facetAvailability(boolean available, String reason) { + JsonObject result = new JsonObject(); + result.addProperty("available", available); + if (!available) result.addProperty("reason", reason); + return result; + } + + private JsonObject callRelationshipObservation(JsonElement value, CallBoundaryObservation calls) + throws Exception { + monitor.checkCancelled(); + JsonObject result = functionObservation(value, calls.limitations()); + result.add("relationship_coverage", callRelationshipCoverage(calls, null)); + return result; + } + + private static JsonObject callRelationshipCoverage( + CallBoundaryObservation calls, Integer missingCallers) { + JsonObject coverage = completeResolvedCallCoverage(); + Integer omitted = calls == null ? null : calls.unrepresentedCalleeCalls(); + JsonArray directions = new JsonArray(); + if (missingCallers != null && missingCallers > 0) directions.add("callers"); + String callerLimitation = missingCallers != null && missingCallers > 0 + ? "Ghidra's caller inventory omits sources of " + missingCallers + + " call reference(s) into this function's body." : null; + if ((calls != null && omitted == null) || (calls == null && missingCallers == null)) { + coverage.addProperty("status", "unknown"); + String reason = calls == null ? "No call inventory was examined." + : "The callee inventory was not examined alongside decoded call targets."; + coverage.addProperty("reason", reason + (callerLimitation == null ? "" : " " + callerLimitation)); + if (calls == null && missingCallers == null) directions.add("callers"); + directions.add("callees"); + coverage.add("limited_directions", directions); + return coverage; + } + JsonArray limitations = new JsonArray(); + if (callerLimitation != null) limitations.add(callerLimitation); + if (omitted != null && omitted > 0) { + limitations.add("Ghidra's callee inventory omits targets of " + omitted + + " call reference(s) or decoded call(s) without destinations."); + directions.add("callees"); + } + if (!limitations.isEmpty()) { + coverage.addProperty("status", "partial"); + coverage.add("limitations", limitations); + coverage.add("limited_directions", directions); + } + return coverage; + } + + private static JsonObject completeResolvedCallCoverage() { + JsonObject coverage = new JsonObject(); + coverage.addProperty("status", "complete"); + JsonArray diagnostics = new JsonArray(); + diagnostics.add( + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals." + ); + coverage.add("diagnostics", diagnostics); + return coverage; + } + + private boolean unrepresentedCaller(Reference reference, Set callers) { + if (!reference.getReferenceType().isCall()) return false; + Function caller = containingFunction(reference.getFromAddress()); + return caller == null || !callers.contains(caller); + } + + private int unrepresentedIncomingCalls(Function function, Set callers) + throws Exception { + int[] omitted = { 0 }; + visitReferences(function, "incoming", reference -> { + if (unrepresentedCaller(reference, callers)) omitted[0]++; + }); + return omitted[0]; + } + + private List noReturnCallLimitations( + Function function, Iterable instructions) throws Exception { List limitations = new ArrayList<>(); + Instruction beforePrevious = null; + Instruction previous = null; for (Instruction instruction : instructions) { monitor.checkCancelled(); - if (!instruction.getFlowType().isCall() || !instruction.getFlowType().isTerminal()) { - continue; + String limitation = terminalCallLimitation(instruction); + if (limitation != null) limitations.add(limitation); + if (beforePrevious != null && previous != null) { + String seh = borlandSehHandlerLimitation( + function, beforePrevious, previous, instruction); + if (seh != null) limitations.add(seh); } - Address[] destinations = instruction.getFlows(); - StringBuilder limitation = new StringBuilder(); - limitation.append("Ghidra Listing reports a terminal call at ") - .append(canonicalAddress(instruction.getAddress())); - if (destinations.length == 0) { - limitation.append(" with no resolved callee; its no-return basis is unknown."); - } - else { - limitation.append(" to "); - for (int index = 0; index < destinations.length; index++) { - if (index > 0) limitation.append(", "); - Address destination = destinations[index]; - limitation.append(canonicalAddress(destination)); - Function target = currentProgram.getFunctionManager().getFunctionAt(destination); - if (target == null) { - limitation.append(" (callee identity unknown)"); - } - else { - limitation.append(" ").append(procedureName(target)); - limitation.append(" (external=").append(target.isExternal()); - limitation.append(", thunk=").append(target.isThunk()); - limitation.append(", hasNoReturn=").append(target.hasNoReturn()).append(")"); - } - } - limitation.append("; fallthrough is excluded by Ghidra's flow model."); - Function target = destinations.length == 1 - ? currentProgram.getFunctionManager().getFunctionAt(destinations[0]) - : null; - if (target != null && target.hasNoReturn()) { - limitation.append(" This records Ghidra's FunctionManager flag, not an independent verification that the callee cannot return."); - } - else { - limitation.append(" The reason for the terminal flow is not established by this observation."); - } - } - limitations.add(limitation.toString()); + beforePrevious = previous; + previous = instruction; } return limitations; } + private CallBoundaryObservation callBoundaryObservation( + Function function, Iterable instructions, Set returnedCallees, + boolean includeUnresolvedSites, List observedReferences) + throws Exception { + List limitations = new ArrayList<>(); + CallBoundaryReferences calls = new CallBoundaryReferences(returnedCallees, includeUnresolvedSites); + Instruction beforePrevious = null; + Instruction previous = null; + if (observedReferences == null) visitReferences(function, "outgoing", calls::observe); + else for (Reference reference : observedReferences) { + monitor.checkCancelled(); + calls.observe(reference); + } + for (Instruction instruction : instructions) { + monitor.checkCancelled(); + String limitation = terminalCallLimitation(instruction); + if (limitation != null) limitations.add(limitation); + if (beforePrevious != null && previous != null) { + String seh = borlandSehHandlerLimitation( + function, beforePrevious, previous, instruction); + if (seh != null) limitations.add(seh); + } + beforePrevious = previous; + previous = instruction; + if (!instruction.getFlowType().isCall()) continue; + boolean destination = false; + for (Reference reference : instruction.getReferencesFrom()) { + if (!reference.getReferenceType().isCall()) continue; + destination = true; + break; + } + if (!destination) { + calls.unrepresented++; + calls.unresolved(instruction.getAddress(), + "Instruction decoder identifies a call but ReferenceManager supplies no call destination"); + } + } + return new CallBoundaryObservation(List.copyOf(limitations), + returnedCallees == null ? null : calls.unrepresented, calls.unresolvedCalls()); + } + + private String borlandSehHandlerLimitation( + Function function, Instruction handlerPush, Instruction framePush, + Instruction frameLink) { + if (!immediatelyPrecedes(handlerPush, framePush) || + !immediatelyPrecedes(framePush, frameLink) || + !handlerPush.getMnemonicString().equalsIgnoreCase("PUSH") || + handlerPush.getNumOperands() != 1 || + !framePush.getMnemonicString().equalsIgnoreCase("PUSH") || + framePush.getNumOperands() != 1 || + !frameLink.getMnemonicString().equalsIgnoreCase("MOV") || + frameLink.getNumOperands() != 2 || + !sameFsMemoryOperand(framePush, frameLink) || + !hasRegisterOperand(frameLink, 1, "ESP")) + return null; + Object[] handlerOperand = handlerPush.getOpObjects(0); + if (handlerOperand.length != 1 || !(handlerOperand[0] instanceof Scalar scalar) || + scalar.bitLength() != 32) + return null; + Address handler = currentProgram.getAddressFactory().getDefaultAddressSpace() + .getAddress(scalar.getUnsignedValue()); + if (function.getBody().contains(handler)) return null; + return "Ghidra Listing shows a Borland/Delphi-style SEH frame registration at " + + canonicalAddress(frameLink.getAddress()) + " (push " + canonicalAddress(handler) + + " followed by fs-segment stack chaining); handler " + canonicalAddress(handler) + + " lies outside this procedure's body, so callees, strings, and cross-references may omit the handler region."; + } + + private static boolean immediatelyPrecedes(Instruction first, Instruction second) { + Address fallThrough = first.getFallThrough(); + return fallThrough != null && fallThrough.equals(second.getAddress()); + } + + private static boolean sameFsMemoryOperand(Instruction first, Instruction second) { + String firstOperand = normalizedFsMemoryOperand(first); + String secondOperand = normalizedFsMemoryOperand(second); + return firstOperand != null && firstOperand.equalsIgnoreCase(secondOperand); + } + + private static String normalizedFsMemoryOperand(Instruction instruction) { + String representation = instruction.getDefaultOperandRepresentation(0); + representation = representation.replaceAll("(?i)\\b(?:byte|word|dword|qword)\\s+ptr\\s+", "") + .replaceAll("\\s+", ""); + return representation.regionMatches(true, 0, "fs:", 0, 3) && + representation.indexOf('[') >= 0 ? representation : null; + } + + private static boolean hasRegisterOperand( + Instruction instruction, int operand, String name) { + for (Object object : instruction.getOpObjects(operand)) { + if (object instanceof Register register && register.getName().equalsIgnoreCase(name)) + return true; + } + return false; + } + + private final class CallBoundaryReferences { + private final Set returnedCallees; + private final TreeMap unresolved; + private int unrepresented; + + CallBoundaryReferences(Set returnedCallees, boolean includeUnresolvedSites) { + this.returnedCallees = returnedCallees; + unresolved = includeUnresolvedSites ? new TreeMap<>() : null; + } + + void observe(Reference reference) { + if (!reference.getReferenceType().isCall()) return; + // Reference identity follows the same containing-function owner as referenceEdge. + Function target = containingFunction(reference.getToAddress()); + if (returnedCallees != null && (target == null || !returnedCallees.contains(target))) + unrepresented++; + if (target == null) unresolved(reference.getFromAddress(), + "A call destination has no known containing Ghidra procedure"); + } + + void unresolved(Address address, String reason) { + if (unresolved != null) unresolved.put(address, reason); + } + + JsonArray unresolvedCalls() { + if (unresolved == null) return null; + JsonArray result = new JsonArray(); + for (var item : unresolved.entrySet()) { + JsonObject site = new JsonObject(); + site.addProperty("address", canonicalAddress(item.getKey())); + site.addProperty("reason", item.getValue()); + result.add(site); + } + return result; + } + } + + private String terminalCallLimitation(Instruction instruction) { + if (!instruction.getFlowType().isCall() || !instruction.getFlowType().isTerminal()) + return null; + Address[] destinations = instruction.getFlows(); + StringBuilder limitation = new StringBuilder(); + limitation.append("Ghidra Listing reports a terminal call at ") + .append(canonicalAddress(instruction.getAddress())); + if (destinations.length == 0) { + limitation.append(" with no resolved callee; its no-return basis is unknown."); + } + else { + limitation.append(" to "); + for (int index = 0; index < destinations.length; index++) { + if (index > 0) limitation.append(", "); + Address destination = destinations[index]; + limitation.append(canonicalAddress(destination)); + Function target = currentProgram.getFunctionManager().getFunctionAt(destination); + if (target == null) { + limitation.append(" (callee identity unknown)"); + } + else { + limitation.append(" ").append(procedureName(target)); + limitation.append(" (external=").append(target.isExternal()); + limitation.append(", thunk=").append(target.isThunk()); + limitation.append(", hasNoReturn=").append(target.hasNoReturn()).append(")"); + } + } + limitation.append("; fallthrough is excluded by Ghidra's flow model."); + Function target = destinations.length == 1 + ? currentProgram.getFunctionManager().getFunctionAt(destinations[0]) + : null; + if (target != null && target.hasNoReturn()) { + limitation.append(" This records Ghidra's FunctionManager flag, not an independent verification that the callee cannot return."); + } + else { + limitation.append(" The reason for the terminal flow is not established by this observation."); + } + } + return limitation.toString(); + } + private static String renderAssembly(List instructions) { return String.join("\n", renderAssemblyLines(instructions)); } @@ -3358,15 +3648,27 @@ public final class ReaGhidraBridge extends HeadlessScript { private List collectReferences( Function function, String direction) throws Exception { + TreeMap observed = new TreeMap<>(); + visitReferences(function, direction, reference -> addReference(observed, reference)); + List result = new ArrayList<>(observed.values()); + result.sort(REFERENCE_ORDER); + return result; + } + + @FunctionalInterface + private interface ReferenceVisitor { + void accept(Reference reference) throws Exception; + } + + private void visitReferences(Function function, String direction, ReferenceVisitor visitor) + throws Exception { // References can target instruction interiors or originate in embedded // data. Use actual body ownership, including an external's exact entry. - AddressSet scope = new AddressSet(function.getBody()); - scope.add(function.getEntryPoint()); + AddressSet scope = functionObservationScope(function); boolean outgoing = direction.equals("outgoing"); AddressIterator addresses = outgoing ? currentProgram.getReferenceManager().getReferenceSourceIterator(scope, true) : currentProgram.getReferenceManager().getReferenceDestinationIterator(scope, true); - TreeMap observed = new TreeMap<>(); while (addresses.hasNext()) { monitor.checkCancelled(); Address address = addresses.next(); @@ -3374,7 +3676,7 @@ public final class ReaGhidraBridge extends HeadlessScript { for (Reference reference : currentProgram.getReferenceManager() .getReferencesFrom(address)) { monitor.checkCancelled(); - addReference(observed, reference); + if (!reference.isEntryPointReference()) visitor.accept(reference); } } else { @@ -3382,19 +3684,14 @@ public final class ReaGhidraBridge extends HeadlessScript { .getReferencesTo(address); while (iterator.hasNext()) { monitor.checkCancelled(); - addReference(observed, iterator.next()); + Reference reference = iterator.next(); + if (!reference.isEntryPointReference()) visitor.accept(reference); } } } - List result = new ArrayList<>(observed.values()); - result.sort(REFERENCE_ORDER); - return result; } private static void addReference(TreeMap observed, Reference reference) { - if (reference.isEntryPointReference()) { - return; - } String key = canonicalAddress(reference.getFromAddress()) + "\u0000" + canonicalAddress(reference.getToAddress()) + "\u0000" + reference.getReferenceType().getName() + "\u0000" + @@ -3403,6 +3700,12 @@ public final class ReaGhidraBridge extends HeadlessScript { observed.putIfAbsent(key, reference); } + private static AddressSet functionObservationScope(Function function) { + AddressSet scope = new AddressSet(function.getBody()); + scope.add(function.getEntryPoint()); + return scope; + } + private JsonArray referenceEdges(List references) { JsonArray result = new JsonArray(); for (Reference reference : references) { @@ -3556,23 +3859,28 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } - private static JsonArray comments(List instructions) { + private JsonArray comments(Function function) throws Exception { + Listing listing = currentProgram.getListing(); + AddressSet scope = functionObservationScope(function); + AddressIterator addresses = listing.getCommentAddressIterator(scope, true); TreeMap observed = new TreeMap<>(); - for (Instruction instruction : instructions) { + while (addresses.hasNext()) { + monitor.checkCancelled(); + Address address = addresses.next(); for (CommentType type : CommentType.values()) { - String text = instruction.getComment(type); + String text = listing.getComment(type, address); if (text == null || text.isEmpty()) { continue; } String kind = type == CommentType.EOL ? "inline" : "comment"; JsonObject item = new JsonObject(); - item.addProperty("address", canonicalAddress(instruction.getAddress())); + item.addProperty("address", canonicalAddress(address)); item.addProperty("kind", kind); item.addProperty("text", text); - String key = canonicalAddress(instruction.getAddress()) + "\u0000" + + String key = canonicalAddress(address) + "\u0000" + kind + "\u0000" + type.name() + "\u0000" + text; observed.put(key, item); - } + } } JsonArray result = new JsonArray(); observed.values().forEach(result::add); @@ -3643,13 +3951,20 @@ public final class ReaGhidraBridge extends HeadlessScript { * The canonical empty string is a small literal with a zero count. */ private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception { + return swiftStringLiterals(function, new SwiftLiteralBudget()); + } + + // An explicit budget makes capacity omissions deterministic in producer tests. + private SwiftStringLiteralDecode swiftStringLiterals( + Function function, + SwiftLiteralBudget budget + ) throws Exception { Register[] registers = new Register[31]; for (int index = 0; index < registers.length; index++) { registers[index] = currentProgram.getRegister("x" + index); } TreeMap observed = new TreeMap<>(); TreeMap omitted = new TreeMap<>(); - SwiftLiteralBudget budget = new SwiftLiteralBudget(); Map selectedValues = new HashMap<>(); SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { @@ -3657,6 +3972,9 @@ public final class ReaGhidraBridge extends HeadlessScript { public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) { // Returning true ends only the current flow, so every queued flow checks the shared stop. if (budget.stopped()) return true; + FlowType flow = instruction.getFlowType(); + Address[] flowTargets = instruction.getFlows(); + boolean unresolvedJump = flow.isJump() && !flow.isConditional() && flowTargets.length == 0; for (Register[] pair : swiftStringPairs(function, instruction, registers)) { Long word = registerConstant(context, pair[0]); Long object = registerConstant(context, pair[1]); @@ -3706,12 +4024,14 @@ public final class ReaGhidraBridge extends HeadlessScript { JsonObject item = overCapacity ? swiftStringOmission(layout, word, object) : swiftStringItem(value, layout, word, object); + if (unresolvedJump && !overCapacity) + item = swiftStringOmission(layout, word, object, "unresolved_jump"); item.addProperty("source_address", canonicalAddress(instruction.getAddress())); JsonArray pairRegisters = new JsonArray(); pairRegisters.add(pair[0].getName()); pairRegisters.add(pair[1].getName()); item.add("registers", pairRegisters); - (overCapacity ? omitted : observed).put(key, item); + (overCapacity || unresolvedJump ? omitted : observed).put(key, item); if (overCapacity) { budget.stop(); return true; @@ -3747,11 +4067,12 @@ public final class ReaGhidraBridge extends HeadlessScript { if (operation.getOpcode() == PcodeOp.RETURN) returns = true; } // A jump is a proven tail call only when it has resolved targets and all leave the - // body. A targetless computed jump can still dispatch inside the function. + // body. For unresolved targetless jumps, candidate literals remain unknown omissions. Address[] targets = instruction.getFlows(); boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 && Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target)); - if (!flow.isCall() && !returns && !tailCall) return pairs; + boolean unresolvedJump = flow.isJump() && !flow.isConditional() && targets.length == 0; + if (!flow.isCall() && !returns && !tailCall && !unresolvedJump) return pairs; // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. int lastFirstRegister = returns ? 0 : 6; for (int index = 0; index <= lastFirstRegister; index++) { @@ -3894,6 +4215,11 @@ public final class ReaGhidraBridge extends HeadlessScript { remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); } + SwiftLiteralBudget(long remaining) { + if (remaining < 0) throw new IllegalArgumentException("Swift literal budget must be nonnegative"); + this.remaining = remaining; + } + boolean takeState() { if (remaining < BYTES_PER_RECORD) return false; remaining -= BYTES_PER_RECORD; @@ -3932,6 +4258,15 @@ public final class ReaGhidraBridge extends HeadlessScript { } private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + return swiftStringOmission(layout, word, object, "capacity"); + } + + private static JsonObject swiftStringOmission( + SwiftLiteralLayout layout, + long word, + long object, + String reason + ) { JsonObject item = new JsonObject(); item.addProperty("representation", layout.representation()); if (layout.address() == null) { @@ -3940,7 +4275,7 @@ public final class ReaGhidraBridge extends HeadlessScript { item.addProperty("address", canonicalAddress(layout.address())); } item.addProperty("byte_length", layout.byteLength()); - item.addProperty("reason", "capacity"); + item.addProperty("reason", reason); item.add("words", swiftStringWords(word, object)); return item; } @@ -4508,7 +4843,8 @@ public final class ReaGhidraBridge extends HeadlessScript { private record InventoryItem(Address address, String value, JsonObject facts) {} private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} - private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} + private record FunctionAnalysisObservation(JsonObject dossier, List limitations, JsonObject relationshipCoverage) {} + private record CallBoundaryObservation(List limitations, Integer unrepresentedCalleeCalls, JsonArray unresolvedCalls) {} private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} private record SwiftLiteralLayout( String representation, diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 4188fe0da6..0425cc0d91 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -736,7 +736,12 @@ def _analyze_function(document, params): def _dispatch(method, params): """Dispatch only the closed operation set implemented by REA's public tools.""" if method == "health": - return {"name": "REA Hopper bridge", "version": "1.0.0", "run_id": REA_RUN_ID} + return { + "name": "REA Hopper bridge", + "version": "1.0.0", + "run_id": REA_RUN_ID, + "process_id": os.getpid(), + } if method in ("shutdown", "shutdown_document"): document = _session_document() if document is None: diff --git a/docs/adb-device-analysis.md b/docs/adb-device-analysis.md index 12fd77cd25..8c94df25f7 100644 --- a/docs/adb-device-analysis.md +++ b/docs/adb-device-analysis.md @@ -73,32 +73,32 @@ existing analysis families directly, for example ## Operations -| Tool | Kind | What it does | -| ----------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `inspect_adb_client` | read | Resolved binary, version, install path; never contacts the server | -| `list_adb_devices` | read | Serial, state, transport, model metadata, inferred emulator/physical kind with basis | -| `inspect_adb_device` | read | Fixed `getprop` whitelist: build, SDK, patch, fingerprint, product, ABIs, debuggable, `ro.kernel.qemu` | -| `inspect_adb_display` | read | `wm size` and `wm density` as declared | -| `list_adb_features` | read | `pm list features` (hardware/software features, hex GL versions) | -| `list_adb_services` | read | `service list` binder services and interfaces | -| `list_adb_processes` | read | `ps -A`: user, pid, ppid, rss, state, name | -| `inspect_adb_window` | read | `dumpsys window` focus declarations (what is on screen) | -| `read_adb_setting` | read | One `settings get` value; device-side null is reported as such | -| `read_adb_logcat` | read | Bounded `logcat -d -t` dump for one buffer, optional pid filter | -| `list_adb_packages` | read | `pm list packages -f` with scope; base APK device paths | -| `resolve_adb_packages` | read | Case-insensitive package-name substring search with exact-match flag | -| `inspect_adb_package` | read | `dumpsys package` projection: versions, install times, installer, flags, permission count | -| `list_adb_directory` | read | NUL-framed immediate children, including hidden entries: kinds, permissions, owner, group, sizes, symlink targets | -| `pull_adb_package` | read | Complete split APK set from `pm path`, each file digested | -| `pull_adb_file` | read | One absolute device path pulled and digested | -| `capture_adb_screen` | read | One `screencap -p` frame (exec-out, no device-side file) with dimensions | -| `collect_adb_bugreport` | read | Device-generated bugreport zip, digested | -| `push_adb_file` | **mutates device** | One local file to a device path (e.g. pushing tools into `/data/local/tmp`); refuses existing targets without explicit overwrite; verifies via the device's own sha256sum | -| `install_adb_package` | **mutates device** | Local APK via `adb install [-r]`; preserves the device's Success/Failure verdict | -| `uninstall_adb_package` | **mutates device** | Removes a package and its data for user 0 | -| `start_adb_app` | **mutates device** | Resolves the package's launcher activity (`cmd package resolve-activity --brief`) and starts it (`am start -n`) | -| `start_adb_activity` | **mutates device** | Starts an intent (`am start -a` with optional URI, component, and up to 16 typed extras); preserves the activity manager's verdict | -| `stop_adb_app` | **mutates device** | Force-stops every process of a package, ending a dynamic-analysis run | +| Tool | Kind | What it does | +| ----------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `inspect_adb_client` | read | Resolved binary, version, install path; never contacts the server | +| `list_adb_devices` | read | Serial, state, transport, model metadata, inferred emulator/physical kind with basis | +| `inspect_adb_device` | read | Fixed `getprop` whitelist: build, SDK, patch, fingerprint, product, ABIs, debuggable, `ro.kernel.qemu` | +| `inspect_adb_display` | read | `wm size` and `wm density` as declared | +| `list_adb_features` | read | `pm list features` (hardware/software features, hex GL versions) | +| `list_adb_services` | read | `service list` binder services and interfaces | +| `list_adb_processes` | read | `ps -A`: user, pid, ppid, rss, state, name | +| `inspect_adb_window` | read | `dumpsys window` focus declarations (what is on screen) | +| `read_adb_setting` | read | One `settings get` value; device-side null is reported as such | +| `read_adb_logcat` | read | Bounded `logcat -d -t` dump for one buffer, optional pid filter | +| `list_adb_packages` | read | `pm list packages -f` with scope; base APK device paths | +| `resolve_adb_packages` | read | Case-insensitive package-name substring search with exact-match flag | +| `inspect_adb_package` | read | `dumpsys package` projection: versions, install times, installer, flags, permission count | +| `list_adb_directory` | read | NUL-framed immediate children, including hidden entries: kinds, permissions, owner, group, sizes, symlink targets | +| `pull_adb_package` | read | Complete split APK set from `pm path`, each file digested | +| `pull_adb_file` | read | One absolute device path pulled and digested | +| `capture_adb_screen` | read | One `screencap -p` frame (exec-out, no device-side file) with dimensions | +| `collect_adb_bugreport` | read | Device-generated bugreport zip, digested | +| `push_adb_file` | **mutates device** | One local file to a device path (e.g. pushing tools into `/data/local/tmp`); checks for existing targets and refuses them without explicit overwrite (the check is not atomic); compares digests when device `sha256sum` is available | +| `install_adb_package` | **mutates device** | Local APK via `adb install [-r]`; preserves the device's Success/Failure verdict | +| `uninstall_adb_package` | **mutates device** | Removes a package and its data for user 0 | +| `start_adb_app` | **mutates device** | Resolves the package's launcher activity (`cmd package resolve-activity --brief`) and starts it (`am start -n`) | +| `start_adb_activity` | **mutates device** | Starts an intent (`am start -a` with optional URI, component, and up to 16 typed extras); preserves the activity manager's verdict | +| `stop_adb_app` | **mutates device** | Force-stops every process of a package, ending a dynamic-analysis run | ## Interpret the result @@ -118,6 +118,16 @@ existing analysis families directly, for example while POSIX hosts retain valid POSIX filenames such as names with spaces or colons. A package pull checks reported names against the destination filesystem before pulling, and refuses names that would share a local path. +- **Package path refusals are diagnosable.** Refusals from `pull_adb_package` + return `unreadable_output` with `retryable: false`, keep the `pm path` output + in `details.captured_output`, and name the refusal in `details.reason`. + `unsafe_device_path` lists refused entries in + `details.diagnostics.rejected_paths` as `{device_path, reason}`; + `duplicate_local_destination` reports the shared `file_name` and both + `device_paths`; `local_destination_unavailable` reports the `device_path`, + `file_name`, and filesystem `error_code`. Diagnostics retain raw device + strings, while the caller-facing message escapes invisible and bidirectional + characters so they cannot hide text or reorder the path. - **Splits are part of the answer.** `pm path` enumerates the base APK and every installed split; pulls cover the whole set. System packages whose APKs keep other names report the `unknown` role rather than guessing. @@ -127,8 +137,10 @@ existing analysis families directly, for example partial coverage instead of dropping rows. - **Mutations are explicit.** `push_adb_file`, `install_adb_package`, and `uninstall_adb_package` declare `mutatesTarget` and never run implicitly: - pushes refuse existing device files unless `overwrite` is set and verify - the transfer with the device's own `sha256sum`; installs preserve the + pushes check for existing device files before transfer and refuse them + unless `overwrite` is set. The check is not atomic, so a different process + can create the path before `adb push` writes it. When available, the device's + `sha256sum` verifies the transfer; installs preserve the device's `Failure [...]` reason verbatim. - **Producer identity stays attached.** Every result embeds the producing adb client (`binary_path`, `path_source`, `version`, `installed_path`). diff --git a/docs/ghidra-mips.md b/docs/ghidra-mips.md index f010b4f4ca..397741a6ad 100644 --- a/docs/ghidra-mips.md +++ b/docs/ghidra-mips.md @@ -52,13 +52,15 @@ section names or the record being inside the initial 4 KiB header probe. Program and section declarations must agree; duplicate, conflicting, truncated or out-of-file records are rejected. Stripped section tables and section-only records are supported. Extended table numbering is explicitly unsupported. -For carved ELFs whose section table extends beyond the captured file, REA -examines every complete captured section header and records the unavailable -headers as a limitation. A valid program-header ABI record can still establish -the supported declaration. Available section declarations must agree with it; -their duplicate, conflicting or unreadable records remain errors. A missing -program-header record does not make ABI declarations in uncaptured sections -absent or supply a default ABI. +GNU `SHT_GNU_ATTRIBUTES` sections are refused by numeric section type until REA +parses and reconciles their declarations; section names and payload sizes do not +control detection or allocation. For carved ELFs whose section table extends +beyond the captured file, REA examines every complete captured section header +and records the unavailable headers as a limitation. The provider refuses an +incomplete ABI declaration inspection even when a program-header ABI record is +valid, because an unavailable section header could carry an unsupported or +conflicting declaration. A missing program-header record does not make ABI +declarations in uncaptured sections absent or supply a default ABI. The field layout follows [`Elf_MIPS_ABIFlags_v0`](https://github.com/bminor/glibc/blob/master/elf/elf.h). diff --git a/docs/ghidra-provider.md b/docs/ghidra-provider.md index fd23dcd073..2f70b48bc7 100644 --- a/docs/ghidra-provider.md +++ b/docs/ghidra-provider.md @@ -154,12 +154,15 @@ An inferred no-return call can terminate that body before subsequent bytes. Pseudocode, assembly, procedure info, instruction windows, callee and caller reads, and dossiers report observed terminal call sites in Evidence limitations, including resolved callee names, addresses, and FunctionManager flags when -available. Instruction windows -and dossiers also retain these warnings in their result limitations. An -unresolved callee or unestablished terminal-flow basis remains explicit. These -observations qualify the analysis; they do not prove that a no-return flag is -incorrect or recover excluded code. Analysis profiles version this observation -contract so older snapshots cannot replay unqualified results. +available. On x86-32, the same views also report the exact Borland/Delphi-style +FS exception-registration pattern when its handler address falls outside the +function body. This is a static registration observation; it does not establish +handler execution or change the body. Instruction windows and dossiers also +retain these warnings in their result limitations. An unresolved callee or +unestablished terminal-flow basis remains explicit. These observations qualify +the analysis; they do not prove that a no-return flag is incorrect or recover +excluded code. Analysis profiles version this observation contract so older +snapshots cannot replay unqualified results. On non-Windows hosts, the ephemeral import repair clears no-return flags on the known returning external ABI names `malloc`, `__tls_get_addr`, `memcpy`, `memset`, @@ -189,7 +192,7 @@ host/provider acceptance run with only the host compiler. `npm run verify:ghidra:windows` checks source-owned native x86-64 PE applications; `-- --x86` selects x86 and `-- --dll` selects DLLs, and the flags can be combined. -These lanes check all 25 read-only operations, target/snapshot/import SHA-256 linkage, +These lanes check all 26 read-only operations, target/snapshot/import SHA-256 linkage, and project, endpoint, process, and runtime cleanup. The independent native lane checks DACLs, handle admission, cancellation, and Job Object lifecycle. `npm run verify:ghidra:windows:package` packs and installs REA into an isolated diff --git a/docs/go-binaries.md b/docs/go-binaries.md index 155780984b..134a8c74d4 100644 --- a/docs/go-binaries.md +++ b/docs/go-binaries.md @@ -58,6 +58,12 @@ original stable file snapshot, including pointer-backed strings. It does not scan arbitrary byte strings as compiler metadata or interpret a fat Mach-O container as one selected architecture. +For ELF images without a named build-info section, discovery checks each +nonempty writable, non-executable load segment in program-header order. This +includes sectionless PIE images with metadata in a later data segment. Named +sections retain priority, and alignment, file mapping and record validation +remain required. + Historical moduleless Go binaries can store their empty module-string header in declared zero-initialized virtual memory rather than file-backed bytes. The reader recognizes this only when the complete optional module header lies in that memory @@ -78,9 +84,13 @@ host permission denial, changed artifacts and cancellation retain distinct errors. The selected file is never modified; this inspection creates no temporary files, starts no subprocesses and accesses no network endpoint. -Resource guards bound the selected file to 256 MiB, aggregate embedded strings +Resource guards bound the selected file and cumulative header searches to 256 MiB +each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and the complete normalized metadata -report to 8 MiB. Oversized results fail without partial success. MCP also applies +report to 8 MiB. Identical search spans with the same virtual alignment are +searched once; partially overlapping spans consume the cumulative search budget +and fail with a CPU resource constraint when it is exhausted. Oversized results +fail without partial success. MCP also applies its [response transport budget](mcp-contracts.md), which counts both text and structured content; a report within the metadata limit can still require export from retained Evidence. These limits are not operating-system memory or CPU diff --git a/docs/ida-provider.md b/docs/ida-provider.md index 012f75bfb5..fca4d0bb56 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -137,11 +137,11 @@ For guided registration, set `REA_IDA_MCP_CONFIG` before `rea setup --client { + let excerpt = ""; + let count = 0; + for (const character of devicePath) { + if (count === 160) break; + excerpt += character; + count += 1; + } + const quoted = JSON.stringify(excerpt).replace( + /[\p{C}\p{Z}\p{Default_Ignorable_Code_Point}\u2800]/gu, + (character) => + character === " " + ? character + : `\\u{${character.codePointAt(0)!.toString(16).toUpperCase()}}`, + ); + return excerpt.length === devicePath.length ? quoted : `${quoted} (excerpt)`; +}; + /** One APK of an installed package exactly as `pm path` reports it. */ export interface AdbPackagePathEntry { readonly device_path: string; diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index 3d15325fd6..9c126785c5 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -240,11 +240,12 @@ const shellStub = ( ): ((root: string) => string) => { const cases = Object.entries(dispatch) .map(([key, output]) => { - const [first, second] = key.split(":"); - const guard = - second === undefined - ? `[ "$4" = "${first}" ]` - : `[ "$4" = "${first}" ] && [ "$5" = "${second}" ]`; + const [first, second, third] = key.split(":"); + const guard = [ + `[ "$4" = "${first}" ]`, + ...(second === undefined ? [] : [`[ "$5" = "${second}" ]`]), + ...(third === undefined ? [] : [`[ "$6" = "${third}" ]`]), + ].join(" && "); return ` if ${guard}; then printf '%b' '${output.replaceAll("\\", "\\\\").replaceAll("\0", "\\000").replace(/'/gu, "'\\''")}'; exit 0; fi`; }) .join("\n"); @@ -379,6 +380,30 @@ exit 1`, expect(result.error.capturedOutput?.stdout).toContain( pmOutput.split("\n")[0], ); + const projected = projectAnalysisError(result.error); + const duplicate = pmOutput.includes("\n"); + expect(projected).toMatchObject({ + code: "unreadable_output", + message: expect.stringContaining("pm path for com.example"), + remediation: { action: expect.stringContaining("details.diagnostics") }, + details: { + reason: duplicate ? "duplicate_local_destination" : "unsafe_device_path", + diagnostics: { + package_name: "com.example", + ...(duplicate + ? { device_paths: ["/data/app/x/base.apk", "/data/app/y/base.apk"] } + : { + rejected_paths: [ + { + device_path: "/data/app/x/", + reason: "empty_file_name", + }, + ], + }), + }, + captured_output: { stdout: expect.stringContaining("package:") }, + }, + }); await expect(readFile(join(root, "pull-ran"))).rejects.toThrow(); await expect(lstat(output)).rejects.toMatchObject({ code: "ENOENT" }); }); @@ -410,7 +435,15 @@ it.each([ if (aliases) { expect(result.ok).toBe(false); if (result.ok) return; - expect(result.error.message).toContain("unique local destinations"); + expect(projectAnalysisError(result.error)).toMatchObject({ + message: expect.stringContaining("unique local destination"), + details: { + reason: "local_destination_unavailable", + diagnostics: { + device_path: `/data/app/y/${secondName}`, + }, + }, + }); await expect(readFile(pullLog)).rejects.toThrow(); await expect(lstat(packageDirectory)).rejects.toThrow(); return; @@ -1015,6 +1048,60 @@ it("inspects the display and window focus", async () => { ); }); +it("reads Android 16 focus declarations from displays when windows omits them", async () => { + const { binary } = await writeAdbStub( + shellStub({ + "dumpsys:window:windows": "Window state has no focus fields\\n", + "dumpsys:window:displays": + "mCurrentFocus=Window{abc u0 com.example/.Main}\nmFocusedApp=ActivityRecord{def u0 com.example/.Main t1}\n", + }), + ); + const result = await evidenceResult(providerOver(binary), { + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(result).toMatchObject({ + focused_window: "Window{abc u0 com.example/.Main}", + focused_app: "ActivityRecord{def u0 com.example/.Main t1}", + observed_lines: [ + "Window{abc u0 com.example/.Main}", + "ActivityRecord{def u0 com.example/.Main t1}", + ], + }); + const execution = await providerOver(binary).execute({ + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(execution.ok).toBe(true); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContainEqual( + expect.stringContaining("dumpsys window displays"), + ); +}); + +it("describes missing window focus as unknown after both sections omit declarations", async () => { + const { binary } = await writeAdbStub( + shellStub({ + "dumpsys:window:windows": "no focus\\n", + "dumpsys:window:displays": "no focus\\n", + }), + ); + const result = await evidenceResult(providerOver(binary), { + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(result).toMatchObject({ focused_window: null, focused_app: null }); + const execution = await providerOver(binary).execute({ + operation: "inspect_adb_window", + input: { serial: "emulator-5554" }, + }); + expect(execution.ok).toBe(true); + if (!execution.ok) throw execution.error; + expect(execution.value.limitations).toContainEqual( + expect.stringContaining("focus is unknown"), + ); +}); + it("reads one settings value and reports device-side nulls", async () => { const { binary } = await writeAdbStub(shellStub({ "settings:get": "1\n" })); const enabled = await evidenceResult(providerOver(binary), { diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index efe3eb3fa0..b7bc545ea4 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -56,6 +56,7 @@ import { type AdbClientIdentity, } from "./AdbConfiguration.js"; import { + displayDevicePath, parseAdbDevicesOutput, parseGetpropOutput, parsePackageListOutput, @@ -417,8 +418,12 @@ const removeOwnedOutputDirectory = async ( const reservePackageDestinations = async ( operation: string, + packageName: string, packageDirectory: string, - entries: readonly { readonly file_name: string }[], + entries: readonly { + readonly file_name: string; + readonly device_path: string; + }[], capturedOutput: { readonly stdout: string; readonly stderr: string; @@ -427,6 +432,8 @@ const reservePackageDestinations = async ( ): Promise => { const reservations: { readonly path: string; + readonly file_name: string; + readonly device_path: string; readonly identity: OutputDirectoryIdentity; }[] = []; for (const entry of entries) { @@ -438,6 +445,8 @@ const reservePackageDestinations = async ( throw new Error(`Could not reserve local filename ${entry.file_name}`); reservations.push({ path, + file_name: entry.file_name, + device_path: entry.device_path, identity: { dev: created.dev, ino: created.ino, @@ -445,10 +454,28 @@ const reservePackageDestinations = async ( }, }); } catch (cause: unknown) { + const errorCode = + cause instanceof Error && + "code" in cause && + typeof cause.code === "string" + ? cause.code + : null; return new AnalysisOutputError( operation, - `Package Manager paths cannot be written to unique local destinations at ${JSON.stringify(entry.file_name)}; refusing before pulling any files`, - { cause, capturedOutput }, + "local_destination_unavailable", + { + cause, + capturedOutput, + userMessage: `pm path for ${packageName} reported ${displayDevicePath(entry.device_path)}, whose file name ${displayDevicePath(entry.file_name)} cannot be written to a unique local destination (${errorCode ?? "unknown error"}). No APK was pulled.`, + remediationAction: + "Review details.diagnostics and details.captured_output for the refused destination. Correct the output filesystem conflict before retrying; REA does not rename APKs.", + diagnostics: { + package_name: packageName, + device_path: entry.device_path, + file_name: entry.file_name, + error_code: errorCode, + }, + }, ); } } @@ -467,10 +494,28 @@ const reservePackageDestinations = async ( ); await rmdir(reservation.path); } catch (cause: unknown) { + const errorCode = + cause instanceof Error && + "code" in cause && + typeof cause.code === "string" + ? cause.code + : null; return new AnalysisOutputError( operation, - `Could not release a local filename reservation before pulling: ${reservation.path}`, - { cause, capturedOutput }, + "local_destination_unavailable", + { + cause, + capturedOutput, + userMessage: `Could not release the local filename reservation for ${displayDevicePath(reservation.device_path)} (${errorCode ?? "unknown error"}); no APK was pulled.`, + remediationAction: + "Review details.diagnostics and details.captured_output for the refused destination. Correct the output filesystem conflict before retrying; REA does not rename APKs.", + diagnostics: { + package_name: packageName, + device_path: reservation.device_path, + file_name: reservation.file_name, + error_code: errorCode, + }, + }, ); } } @@ -1095,14 +1140,27 @@ export class AdbProvider { ); } case "inspect_adb_window": { - const ran = await shell(["dumpsys", "window", "windows"], { + let section = "windows"; + const ran = await shell(["dumpsys", "window", section], { timeoutMs: SHELL_TIMEOUT_MS, maxOutputBytes: DUMPSYS_WINDOW_MAX_OUTPUT_BYTES, }); if (!ran.ok) return ran; - const { focusedWindow, focusedApp } = parseWindowFocusOutput( + let { focusedWindow, focusedApp } = parseWindowFocusOutput( ran.value.stdout, ); + if (focusedWindow === null && focusedApp === null) { + section = "displays"; + const fallback = await shell(["dumpsys", "window", section], { + timeoutMs: SHELL_TIMEOUT_MS, + maxOutputBytes: DUMPSYS_WINDOW_MAX_OUTPUT_BYTES, + }); + if (!fallback.ok) return fallback; + ({ focusedWindow, focusedApp } = parseWindowFocusOutput( + fallback.value.stdout, + )); + } + const declared = focusedWindow !== null || focusedApp !== null; return ok( createAnalysisExecution( { @@ -1119,6 +1177,9 @@ export class AdbProvider { { limitations: [ "Focus values are the window manager's declarations at read time; no interaction occurred", + declared + ? `Focus declarations were read from \`dumpsys window ${section}\`` + : "Neither `dumpsys window windows` nor `dumpsys window displays` declared mCurrentFocus or mFocusedApp; focus is unknown, not absent", ], }, ), @@ -1269,14 +1330,27 @@ export class AdbProvider { stderr: paths.value.stderr, truncated: false, }; - if (parsed.rejectedPaths.length > 0) + if (parsed.rejectedPaths.length > 0) { + const first = parsed.rejectedPaths[0]!; + const otherCount = parsed.rejectedPaths.length - 1; return err( - new AnalysisOutputError( - operation, - `Package Manager reported paths that cannot be represented as host files: ${parsed.rejectedPaths.map((entry) => `${JSON.stringify(entry.device_path)} (${entry.reason})`).join(", ")}`, - { capturedOutput }, - ), + new AnalysisOutputError(operation, "unsafe_device_path", { + capturedOutput, + userMessage: `pm path for ${packageName} reported an APK path that cannot be written as a file on this host: ${displayDevicePath(first.device_path)} (${first.reason})${otherCount > 0 ? ` and ${String(otherCount)} other path(s)` : ""}. No APK was pulled.`, + remediationAction: + "Review details.diagnostics for the refused device paths and details.captured_output for the retained pm path output. REA does not rename device-reported APK paths; correct the device path policy before retrying.", + diagnostics: { + package_name: packageName, + rejected_paths: parsed.rejectedPaths.map( + ({ device_path, reason }) => ({ + device_path, + reason, + }), + ), + }, + }), ); + } if (parsed.paths.length === 0 && parsed.unparsedLines.length === 0) return err( new AnalysisUnsupportedTargetError( @@ -1296,11 +1370,19 @@ export class AdbProvider { for (const entry of parsed.paths) { if (destinationNames.has(entry.file_name)) return err( - new AnalysisOutputError( - operation, - `Package Manager reported multiple paths with the same local destination ${JSON.stringify(entry.file_name)}; refusing before pulling any files`, - { capturedOutput }, - ), + new AnalysisOutputError(operation, "duplicate_local_destination", { + capturedOutput, + userMessage: `pm path for ${packageName} reported multiple APK paths with the same local file name ${displayDevicePath(entry.file_name)}; no APK was pulled.`, + remediationAction: + "Review details.diagnostics and details.captured_output for the conflicting device paths. REA does not rename device-reported paths; correct Package Manager's path response before retrying.", + diagnostics: { + package_name: packageName, + file_name: entry.file_name, + device_paths: parsed.paths + .filter((candidate) => candidate.file_name === entry.file_name) + .map((candidate) => candidate.device_path), + }, + }), ); destinationNames.add(entry.file_name); } @@ -1395,6 +1477,7 @@ export class AdbProvider { }; const destinationError = await reservePackageDestinations( operation, + packageName, packageDirectory, parsed.paths, capturedOutput, diff --git a/src/adb/AdbShellOutput.ts b/src/adb/AdbShellOutput.ts index 6dac4cc0c5..70a1584a22 100644 --- a/src/adb/AdbShellOutput.ts +++ b/src/adb/AdbShellOutput.ts @@ -172,7 +172,7 @@ export const parseWmDensityOutput = ( return { physical, override }; }; -/** Parse the focus declarations from `dumpsys window windows`. */ +/** Parse focus declarations from `dumpsys window windows` or `displays`. */ export const parseWindowFocusOutput = ( stdout: string, ): { diff --git a/src/android/JadxConfiguration.ts b/src/android/JadxConfiguration.ts index b44d1f28bf..e07e2c2116 100644 --- a/src/android/JadxConfiguration.ts +++ b/src/android/JadxConfiguration.ts @@ -1,6 +1,7 @@ import { constants } from "node:fs"; import { access, realpath, stat } from "node:fs/promises"; import { delimiter, isAbsolute, join, resolve } from "node:path"; +import { inspectWindowsNativeReadiness } from "../process/WindowsAuthority.js"; import { systemWindowsNativeAuthority } from "../windows/WindowsNativeLoader.js"; import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import type { AndroidOperation } from "../domain/android/androidAnalysis.js"; @@ -11,7 +12,15 @@ import { execFileOutput, execFileOutputFailure, } from "../process/ExecFileOutput.js"; -import { JADX_JAR_CONFIGURATION_REMEDIATION } from "./JadxRelease.js"; +import type { DoctorProviderCheck } from "../application/Doctor.js"; +import { + JADX_JAR_CONFIGURATION_REMEDIATION, + JADX_JAVA_PROBE_REMEDIATION, + JADX_JAVA_RUNTIME_REMEDIATION, + JADX_JVM_OPTIONS_REMEDIATION, + JADX_UNSUPPORTED_HOST_REMEDIATION, + JADX_WINDOWS_NATIVE_REMEDIATION, +} from "./JadxRelease.js"; const JAVA_MODULE_LISTING_TIMEOUT_MS = 10_000; const JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES = 1024 * 1024; @@ -30,16 +39,110 @@ type ConfigurationFailureCode = | "not_configured" | "executable_missing" | "runtime_missing" + | "runtime_unavailable" | "unsupported_host" | "unsupported_version" | "version_unresolved" | "open_options_invalid"; +/** + * Failure kinds tagged at the JADX configuration boundary. Consumers (the + * doctor adapter) project the boundary's remediation and classification for + * a kind; they must not reinterpret the phase/setting evidence to derive + * them. + */ +export type JadxFailureKind = + | "windows_native_artifact" + | "windows_native_runtime" + | "java_runtime" + | "java_home" + | "jvm_options" + | "jar" + | "java_probe" + | "unsupported_host"; + +/** Runtime mirror of JadxFailureKind for validating the loose diagnostics tag. */ +export const JADX_FAILURE_KINDS: readonly JadxFailureKind[] = [ + "windows_native_artifact", + "windows_native_runtime", + "java_runtime", + "java_home", + "jvm_options", + "jar", + "java_probe", + "unsupported_host", +]; + +export interface JadxFailureOutcome { + readonly remediation: string; + readonly classification: DoctorProviderCheck["classification"]; +} + +/** + * Project the boundary-owned remediation and classification for a failure + * kind. A null kind (missing or corrupt tag) keeps JAR guidance, matching + * the previous fallthrough for unclassified failures. + */ +export const projectJadxFailureOutcome = ( + kind: JadxFailureKind | null, + configured: boolean, +): JadxFailureOutcome => { + const drift = configured ? "config_drift" : "missing_analysis_engine"; + switch (kind) { + case "windows_native_artifact": + return { + remediation: JADX_WINDOWS_NATIVE_REMEDIATION, + classification: "missing_dependency", + }; + case "windows_native_runtime": + return { + remediation: JADX_WINDOWS_NATIVE_REMEDIATION, + classification: "runtime_failure", + }; + case "java_runtime": + return { + remediation: JADX_JAVA_RUNTIME_REMEDIATION, + classification: "missing_dependency", + }; + case "java_home": + return { + remediation: JADX_JAVA_RUNTIME_REMEDIATION, + classification: drift, + }; + case "jvm_options": + return { + remediation: JADX_JVM_OPTIONS_REMEDIATION, + classification: drift, + }; + case "jar": + return { + remediation: JADX_JAR_CONFIGURATION_REMEDIATION, + classification: drift, + }; + case "java_probe": + return { + remediation: JADX_JAVA_PROBE_REMEDIATION, + classification: drift, + }; + case "unsupported_host": + return { + remediation: JADX_UNSUPPORTED_HOST_REMEDIATION, + classification: "unsupported_host", + }; + case null: + return { + remediation: JADX_JAR_CONFIGURATION_REMEDIATION, + classification: drift, + }; + } +}; + class JadxConfigurationFailure extends Error { constructor( readonly code: ConfigurationFailureCode, message: string, readonly diagnostics: Readonly>, + readonly kind: JadxFailureKind, options?: ErrorOptions, ) { super(message, options); @@ -75,7 +178,7 @@ export const inspectJadxAvailability = async ( status: "unavailable", code: cause.code, reason: cause.message, - diagnostics: cause.diagnostics, + diagnostics: { ...cause.diagnostics, failure_kind: cause.kind }, }; throw cause; } @@ -191,6 +294,7 @@ const requireEngineArchive = async ( archive_error: reason, maximum_metadata_read_bytes: JAR_METADATA_READ_MAX_BYTES, }, + "jar", cause, ); } finally { @@ -200,6 +304,7 @@ const requireEngineArchive = async ( "unsupported_version", `REA_JADX_MCP_JAR ${jar} does not contain the required metadata bridge classes: ${[...missing].join(", ")}. ${JADX_JAR_CONFIGURATION_REMEDIATION}`, { jar_path: jar, phase: "jar-inspection", missing_classes: [...missing] }, + "jar", ); }; @@ -207,25 +312,52 @@ const requireSupportedHost = (): void => { if (process.platform === "linux" || process.platform === "darwin") return; if (process.platform === "win32" && process.arch === "x64") { const loaded = systemWindowsNativeAuthority(); - if (loaded.available && loaded.authority.inspection.protocolStdin === true) - return; + if (loaded.available) { + if (loaded.authority.inspection.protocolStdin === true) return; + throw configurationFailure( + "runtime_missing", + "The bundled Windows native artifact lacks owned protocol stdin; use a matching current Windows x64 REA package.", + { + platform: process.platform, + architecture: process.arch, + phase: "process-ownership", + owned_protocol_stdin: false, + }, + "windows_native_artifact", + ); + } + const readiness = inspectWindowsNativeReadiness(process.platform, loaded); throw configurationFailure( - "runtime_missing", - loaded.available - ? "The bundled Windows native artifact lacks owned protocol stdin; use a matching current Windows x64 REA package." - : loaded.reason, + readiness.available ? "runtime_unavailable" : readiness.code, + readiness.available + ? "Windows native readiness disagreed with the native loader result." + : readiness.reason, { platform: process.platform, architecture: process.arch, phase: "process-ownership", owned_protocol_stdin: false, + ...(readiness.available + ? {} + : { + failure_stage: readiness.failureStage, + remediation: readiness.remediation, + }), }, + readiness.available + ? "windows_native_runtime" + : readiness.code === "unsupported_host" + ? "unsupported_host" + : readiness.code === "runtime_missing" + ? "windows_native_artifact" + : "windows_native_runtime", ); } throw configurationFailure( "unsupported_host", `Android JADX subprocess ownership is unsupported on ${process.platform}/${process.arch}; use Linux, macOS, or Windows x64 with its bundled native controls.`, { platform: process.platform }, + "unsupported_host", ); }; @@ -238,6 +370,7 @@ const resolveJar = async ( "not_configured", JADX_JAR_CONFIGURATION_REMEDIATION, { jar_configured: false }, + "jar", ); try { await access(jar, constants.R_OK); @@ -246,6 +379,7 @@ const resolveJar = async ( "executable_missing", `REA_JADX_MCP_JAR is not a regular file: ${jar}`, { jar_path: jar }, + "jar", ); return await realpath(jar); } catch (cause) { @@ -254,6 +388,7 @@ const resolveJar = async ( "executable_missing", `Cannot read REA_JADX_MCP_JAR ${jar}: ${cause instanceof Error ? cause.message : String(cause)}`, { jar_path: jar }, + "jar", cause, ); } @@ -293,6 +428,7 @@ const resolveJava = async ( "runtime_missing", `Cannot inspect selected Java executable ${candidate}: ${errorText(cause)}`, { java_executable: candidate }, + "java_runtime", cause, ); } @@ -301,6 +437,7 @@ const resolveJava = async ( "runtime_missing", "Cannot find java.exe on the selected Windows PATH. Select a full JDK with JAVA_HOME or PATH, then verify it with java --list-modules. REA does not install Java.", { java_executable: executable }, + "java_runtime", ); } if (isAbsolute(environment.JAVA_HOME)) @@ -309,6 +446,7 @@ const resolveJava = async ( "open_options_invalid", "JAVA_HOME must select an existing JDK by absolute path; omit it to use java on PATH.", { java_home: environment.JAVA_HOME }, + "java_home", ); }; @@ -324,6 +462,7 @@ const requireJavaHomeExecutable = async ( "runtime_missing", `JAVA_HOME java is not a regular file: ${java}`, { java_executable: java }, + "java_runtime", ); } catch (cause) { if (cause instanceof JadxConfigurationFailure) throw cause; @@ -331,6 +470,7 @@ const requireJavaHomeExecutable = async ( "runtime_missing", `Cannot execute JAVA_HOME java ${java}: ${cause instanceof Error ? cause.message : String(cause)}`, { java_executable: java }, + "java_runtime", cause, ); } @@ -351,6 +491,7 @@ const resolveJvmArguments = ( "open_options_invalid", `${name} must be a positive safe integer; received ${value}.`, { setting: name, value }, + "jvm_options", ); if ( name === "REA_JADX_ACTIVE_PROCESSOR_COUNT" && @@ -360,6 +501,7 @@ const resolveJvmArguments = ( "open_options_invalid", `${name} exceeds the JVM's signed 32-bit processor-count range.`, { setting: name, value }, + "jvm_options", ); result.push(`${prefix}${value}${suffix}`); } @@ -405,6 +547,7 @@ const inspectJavaRuntime = async ( failure: "output_limit", output_limit_bytes: JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES, }, + "java_probe", cause, ); if (output.killed) @@ -416,12 +559,14 @@ const inspectJavaRuntime = async ( failure: "probe_terminated", deadline_ms: JAVA_MODULE_LISTING_TIMEOUT_MS, }, + "java_probe", cause, ); throw configurationFailure( "runtime_missing", `Cannot inspect selected Java runtime ${java}: ${output.stderr.trim() || errorText(cause)}. Select a full JDK including jdk.compiler via JAVA_HOME or PATH; verify it with java --list-modules.`, diagnostics, + "java_runtime", cause, ); } @@ -438,6 +583,7 @@ const inspectJavaRuntime = async ( stderr, output_truncated: false, }, + "java_runtime", ); const major = /^java\.base@(\d+)(?:[.+_-]|$)/mu.exec(stdout)?.[1]; if (major === undefined) @@ -453,6 +599,7 @@ const inspectJavaRuntime = async ( stderr, output_truncated: false, }, + "java_probe", ); if (Number(major) < 17) throw configurationFailure( @@ -467,6 +614,7 @@ const inspectJavaRuntime = async ( stderr, output_truncated: false, }, + "java_runtime", ); }; @@ -474,12 +622,14 @@ const configurationFailure = ( code: ConfigurationFailureCode, message: string, diagnostics: Readonly>, + kind: JadxFailureKind, cause?: unknown, ): JadxConfigurationFailure => new JadxConfigurationFailure( code, message, diagnostics, + kind, cause === undefined ? undefined : { cause }, ); diff --git a/src/android/JadxRelease.ts b/src/android/JadxRelease.ts index 8c7d16840b..447886b523 100644 --- a/src/android/JadxRelease.ts +++ b/src/android/JadxRelease.ts @@ -15,6 +15,26 @@ export const JADX_BRIDGE_IDENTITY = Object.freeze({ /** Remediation when `REA_JADX_MCP_JAR` is missing or not an absolute path. */ export const JADX_JAR_CONFIGURATION_REMEDIATION = `Set REA_JADX_MCP_JAR to the absolute path of a caller-supplied jadx-headless-mcp ${JADX_RELEASE.version} JAR. REA does not download or install it.`; +/** Remediation when the selected Java runtime cannot satisfy the JADX bridge. */ +export const JADX_JAVA_RUNTIME_REMEDIATION = + "Select a full JDK including jdk.compiler via JAVA_HOME or PATH, then verify it with java --list-modules. REA does not install Java."; + +/** Remediation when a JADX JVM resource override is malformed. */ +export const JADX_JVM_OPTIONS_REMEDIATION = + "Set REA_JADX_HEAP_MIB to a positive integer (heap size in MiB) and REA_JADX_ACTIVE_PROCESSOR_COUNT to a positive integer within the JVM's signed 32-bit range."; + +/** Remediation when the host cannot run the Android JADX subprocess. */ +export const JADX_UNSUPPORTED_HOST_REMEDIATION = + "Use Linux, macOS, or Windows x64 with its bundled native controls for Android JADX analysis."; + +/** Remediation when the Windows native backend cannot take process ownership. */ +export const JADX_WINDOWS_NATIVE_REMEDIATION = + "Use a matching current Windows x64 REA package with its bundled native controls. The native failure above identifies the failed constraint; Java installation or registration changes cannot enable a missing native backend."; + +/** Remediation when the Java module-listing probe cannot establish the runtime. */ +export const JADX_JAVA_PROBE_REMEDIATION = + "Verify the selected Java launcher with java --list-modules: it must start promptly and emit a concise module listing including a parseable java.base version. REA does not install Java."; + /** Producing engine identity used by Evidence and the generated product catalog. */ export const JADX_PROVIDER_IDENTITY = Object.freeze({ id: "jadx", diff --git a/src/apktool/ApktoolProvider.test.ts b/src/apktool/ApktoolProvider.test.ts index b6ee249e54..5ac36876ab 100644 --- a/src/apktool/ApktoolProvider.test.ts +++ b/src/apktool/ApktoolProvider.test.ts @@ -100,10 +100,12 @@ const decodeStub = ( readonly manifestFile?: string; readonly stringsFile?: string; readonly stringsXml?: string; + readonly extraValuesDirectories?: readonly string[]; } = {}, ): ((root: string) => string) => { const withStrings = options.withStrings ?? true; const withGerman = options.withGerman ?? true; + const extraValuesDirectories = options.extraValuesDirectories ?? []; const stringsXml = options.stringsXml ?? REAL_STRINGS; const ymlWrite = options.ymlFile === undefined @@ -117,6 +119,12 @@ const decodeStub = ( options.stringsFile === undefined ? `printf '%s\\n' '${stringsXml.replace(/'/gu, "'\\''")}' > "$5/res/values/strings.xml"` : `cp '${options.stringsFile}' "$5/res/values/strings.xml"`; + const qualifierStrings = extraValuesDirectories + .map( + (name) => + `printf '%s\\n' 'Wrong' > "$5/res/values-${name}/strings.xml"`, + ) + .join("\\n "); return () => `if [ "$1" = "--version" ]; then ${VERSION_BLOCK}; exit 0; fi if [ "$1" = "d" ]; then ${ @@ -124,9 +132,10 @@ if [ "$1" = "d" ]; then ? `printf 'Exception in thread "main" brut.androlib.AndrolibException: Could not decode the APK\\n' >&2; exit 1;` : `${ymlWrite} ${manifestWrite} - mkdir -p "$5/res/values"${withGerman ? ' "$5/res/values-de"' : ""} + mkdir -p "$5/res/values"${withGerman ? ' "$5/res/values-de"' : ""}${extraValuesDirectories.map((name) => ` "$5/res/values-${name}"`).join("")} ${withStrings ? stringsWrite : ""} ${withGerman ? `printf '%s\\n' '${DE_STRINGS.replace(/'/gu, "'\\''")}' > "$5/res/values-de/strings.xml"` : ""} + ${qualifierStrings} printf 'I: Using Apktool on stub\\n' exit 0` } @@ -254,6 +263,30 @@ it("preserves empty resource strings and reports markup omissions as partial", a expect(result.coverage).toBe("partial"); }); +it("ignores Android resource qualifiers when discovering locales", async () => { + const { command, root } = await writeApktoolStub( + decodeStub({ extraValuesDirectories: ["car", "night", "v21"] }), + ); + const apk = await apkFixture(root); + const result = await evidenceResult(providerOver(command), { + operation: "decode_android_resources", + input: { path: apk.path, include_strings: true }, + }); + expect(result.locales).toEqual(["de"]); +}); + +it.each(["car", "night", "v21"])( + "rejects %s as a requested locale", + (locale) => { + expect( + apktoolRequestSchema.safeParse({ + operation: "decode_android_resources", + input: { path: "/tmp/probe.apk", locale }, + }).success, + ).toBe(false); + }, +); + it("projects one locale's strings on request", async () => { const { command, root } = await writeApktoolStub(decodeStub()); const apk = await apkFixture(root); diff --git a/src/apktool/ApktoolProvider.ts b/src/apktool/ApktoolProvider.ts index c58ea1f491..eec08fe433 100644 --- a/src/apktool/ApktoolProvider.ts +++ b/src/apktool/ApktoolProvider.ts @@ -25,7 +25,10 @@ import { createAnalysisExecution, type AnalysisExecution, } from "../application/AnalysisProvider.js"; -import type { ApktoolRequest } from "../domain/apktool/apktoolResourceAnalysis.js"; +import { + androidLocaleQualifierSchema, + type ApktoolRequest, +} from "../domain/apktool/apktoolResourceAnalysis.js"; import { AnalysisCapabilityUnavailableError, AnalysisCancelledError, @@ -291,6 +294,7 @@ const countDecodedFiles = async ( const match = /^values-(.+)$/u.exec(entry.name); if ( match !== null && + androidLocaleQualifierSchema.safeParse(match[1]).success && (await access(join(path, "strings.xml")).then( () => true, () => false, diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 376036fa13..d1d41800ce 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -275,6 +275,12 @@ export type ProviderTargetSupport = readonly diagnostics: Readonly>; }; +/** One owner-observed availability/capability snapshot for provider discovery. */ +export interface AnalysisProviderReadinessSnapshot { + readonly availability: ProviderAvailability; + readonly capabilities: readonly CapabilityDescriptor[]; +} + /** Deep provider candidate discoverable before one target-bound route is chosen. */ export interface AnalysisProviderCandidate extends AnalysisProvider { resolveAnalysisProfile( @@ -282,5 +288,6 @@ export interface AnalysisProviderCandidate extends AnalysisProvider { options?: AnalysisProfileResolutionOptions, ): Promise>; inspectAvailability(): ProviderAvailability; + inspectReadinessSnapshot?(): AnalysisProviderReadinessSnapshot; inspectTargetSupport(target: BinaryTarget): ProviderTargetSupport; } diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 94b9ac659b..4f1a217f0c 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -98,6 +98,21 @@ const entry = ( }; describe("capability failure reasons", () => { + it("keeps an unavailable Android runtime distinct from a missing provider", () => { + const inventory = buildCapabilityInventory(status(), { + ...enabledPolicy, + androidAnalysisAvailability: { + status: "unavailable", + code: "runtime_unavailable", + reason: "The JADX native self-test failed.", + diagnostics: {}, + }, + }); + expect( + inventory.find(({ name }) => name === "inspect_android_package"), + ).toMatchObject({ available: false, reason: "provider_unavailable" }); + }); + it("admits physical universal Mach-O targets for native Mach-O operations", () => { expect( entry( @@ -501,7 +516,6 @@ it("reports host denial before target mismatch but keeps composed workflow prere describe("binary_overview capability requirements", () => { it("admits a provider with current_document and other mandatory capabilities", () => { const capabilities = [ - "list_segments", "current_document", "list_procedures", "list_strings", @@ -520,7 +534,6 @@ describe("binary_overview capability requirements", () => { it("admits a provider with list_documents and other mandatory capabilities", () => { const capabilities = [ - "list_segments", "list_documents", "list_procedures", "list_strings", @@ -556,7 +569,7 @@ describe("binary_overview capability requirements", () => { }); }); - it("reports missing segment enumeration independently when segments are missing", () => { + it("admits a provider without optional segment enumeration", () => { const capabilities = [ "current_document", "list_procedures", @@ -568,12 +581,31 @@ describe("binary_overview capability requirements", () => { status({ open: true, kind: "executable", capabilities }), ), ).toMatchObject({ - available: false, - reason: "provider_missing", - remediation: "Configure a provider for required operation list_segments.", + available: true, + reason: "available", + remediation: null, }); }); + it("keeps binary overview available when a declared segment facet is unavailable", () => { + const capabilities = [ + { + operation: "list_segments", + available: false, + reason: "This upstream profile does not expose segment enumeration.", + }, + { operation: "list_documents", available: true, reason: null }, + { operation: "list_procedures", available: true, reason: null }, + { operation: "list_strings", available: true, reason: null }, + ]; + expect( + entry( + "binary_overview", + status({ open: true, kind: "executable", capabilities }), + ), + ).toMatchObject({ available: true, reason: "available" }); + }); + it("reports provider_unavailable when declared document selection capability is unavailable", () => { const capabilities = [ { operation: "list_segments", available: true, reason: null }, diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 87734fbd1b..029b233577 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -115,7 +115,6 @@ const ENHANCED_REQUIREMENTS: Readonly< analyze_swift_types: ["list_procedures"], find_xrefs_to_name: ["list_names", "xrefs"], binary_overview: [ - "list_segments", ["current_document", "list_documents"], "list_procedures", "list_strings", @@ -462,7 +461,10 @@ const workflowAvailabilityFor = ({ reason: policy.androidAnalysisAvailability?.code === "unsupported_host" ? "unsupported_host" - : "provider_missing", + : policy.androidAnalysisAvailability?.code === + "runtime_unavailable" + ? "provider_unavailable" + : "provider_missing", remediation: policy.androidAnalysisAvailability?.reason ?? "Set REA_JADX_MCP_JAR to a caller-supplied jadx-headless-mcp 0.7.1 JAR and provide a full JDK on Linux, macOS, or Windows x64 with its matching bundled native controls.", diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 207186cccc..7bd340d3bf 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -525,33 +525,47 @@ export class EnhancedTools { this.#call("list_strings", {}, signal), ], ); - if (!segmentsResult.ok) return segmentsResult; + if ( + !segmentsResult.ok && + !(segmentsResult.error instanceof AnalysisCapabilityUnavailableError) + ) + return segmentsResult; if (!proceduresResult.ok) return proceduresResult; if (!stringsResult.ok) return stringsResult; - const segments = parseSegments(segmentsResult.value); - if (!segments.ok) return segments; + const segments = segmentsResult.ok + ? parseSegments(segmentsResult.value) + : undefined; + if (segments !== undefined && !segments.ok) return segments; const procedureCount = parseListCount(proceduresResult.value, "procedures"); if (!procedureCount.ok) return procedureCount; const stringCount = parseListCount(stringsResult.value, "strings"); if (!stringCount.ok) return stringCount; - const imageBase = parseSegmentImageBase(segments.value); + const observedSegments = segments?.ok ? segments.value : undefined; + const imageBase = + observedSegments === undefined + ? undefined + : parseSegmentImageBase(observedSegments); return ok({ document: document.value, ...(imageBase === undefined ? {} : { image_base: imageBase }), - segments: segments.value.map( - ({ name, start, end, image_base, address_space }) => ({ - name, - start, - end, - length: addressDistance(start, end), - ...(image_base === undefined ? {} : { image_base }), - ...(address_space === undefined ? {} : { address_space }), - }), - ), - segment_count: segments.value.length, + ...(observedSegments === undefined + ? {} + : { + segments: observedSegments.map( + ({ name, start, end, image_base, address_space }) => ({ + name, + start, + end, + length: addressDistance(start, end), + ...(image_base === undefined ? {} : { image_base }), + ...(address_space === undefined ? {} : { address_space }), + }), + ), + segment_count: observedSegments.length, + }), procedure_count: procedureCount.value, string_count: stringCount.value, }); diff --git a/src/application/MipsElfMetadata.ts b/src/application/MipsElfMetadata.ts index 8fd9d0e894..38b2bccb8d 100644 --- a/src/application/MipsElfMetadata.ts +++ b/src/application/MipsElfMetadata.ts @@ -10,6 +10,8 @@ import { type MipsElfRelocationSection, } from "./MipsElfSharedLoad.js"; +const SHT_GNU_ATTRIBUTES = 0x6ffffff5; + /** Observed ABI and load declarations, retaining unavailable source coverage. */ export interface MipsElfInspection { readonly abiFlags: MipsAbiFlags | null; @@ -151,6 +153,10 @@ export const readMipsElfMetadata = async ( checkCancelled(); const position = i * table.entrySize; const type = u32(contents.value, position + table.typeOffset); + if (table.optional && type === SHT_GNU_ATTRIBUTES) + return err( + "MIPS ELF declares GNU ABI attributes that REA does not inspect; refusing provider selection until the declared ABI is parsed and reconciled", + ); if (metadata.type === 3 && table.optional && (type === 4 || type === 9)) relocationSections.push({ type, diff --git a/src/application/WindowsCapabilities.ts b/src/application/WindowsCapabilities.ts index dfd092c69c..61b96c40f8 100644 --- a/src/application/WindowsCapabilities.ts +++ b/src/application/WindowsCapabilities.ts @@ -70,24 +70,26 @@ export const probeWindowsCapabilities = async ( }; }; -const systemDependencies = (): WindowsCapabilityDependencies => ({ - platform: process.platform, - architecture: process.arch, - probeSymlinkCreation, - probeNoFollowOpen: () => - systemNoFollowOpenCapability(process.platform, constants.O_NOFOLLOW), - probePrivateAcl: () => - windowsNativeCapabilities(process.platform).private_runtime_dacl, - nativeCapabilities: () => windowsNativeCapabilities(process.platform), - probeUnixDomainSocket: () => - process.platform === "win32" - ? { - available: false, - reason: "Node path-based IPC uses named pipes on Windows", - } - : { available: true, reason: null }, - probePty: probeProcessCaptureCapability, -}); +const systemDependencies = (): WindowsCapabilityDependencies => { + const security = windowsNativeCapabilities(process.platform); + return { + platform: process.platform, + architecture: process.arch, + probeSymlinkCreation, + probeNoFollowOpen: () => + systemNoFollowOpenCapability(process.platform, constants.O_NOFOLLOW), + probePrivateAcl: () => security.private_runtime_dacl, + nativeCapabilities: () => security, + probeUnixDomainSocket: () => + process.platform === "win32" + ? { + available: false, + reason: "Node path-based IPC uses named pipes on Windows", + } + : { available: true, reason: null }, + probePty: probeProcessCaptureCapability, + }; +}; /** Report pathname no-follow support without treating it as Windows authority. */ export const systemNoFollowOpenCapability = ( diff --git a/src/application/binary/AnalysisProviderRegistry.test.ts b/src/application/binary/AnalysisProviderRegistry.test.ts index fde2f67189..60d3ab62aa 100644 --- a/src/application/binary/AnalysisProviderRegistry.test.ts +++ b/src/application/binary/AnalysisProviderRegistry.test.ts @@ -13,6 +13,7 @@ import { import { createAnalysisProfile } from "../../domain/analysisProfile.js"; import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { ProviderSelectionError } from "../../domain/providerSelectionError.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { err, ok } from "../../domain/result.js"; @@ -63,6 +64,46 @@ it("rejects malformed provider capability declarations", () => { ).toThrow(/Invalid analysis provider ID: auto/u); }); +it("uses one owner readiness snapshot for candidate availability and capabilities", () => { + const base = candidate("alpha").provider; + let observed = 0; + let capabilityReads = 0; + const provider: AnalysisProviderCandidate = { + ...base, + inspectAvailability: () => { + throw new Error("availability must come from the owner snapshot"); + }, + capabilities: () => { + capabilityReads += 1; + return [capability(base.identity())]; + }, + inspectReadinessSnapshot: () => { + observed += 1; + return { + availability: { + status: "unavailable", + code: "runtime_unavailable", + reason: "native inspection failed", + diagnostics: { failure_stage: "self-test" }, + }, + capabilities: [capability(base.identity())], + }; + }, + }; + + const [status] = new AnalysisProviderRegistry([provider]).candidates(); + expect(observed).toBe(1); + expect(capabilityReads).toBe(1); + expect(status).toMatchObject({ + availability: { + status: "unavailable", + code: "runtime_unavailable", + diagnostics: { failure_stage: "self-test" }, + }, + capabilities: [{ operation: "address_name" }], + }); +}); + it("selects the sole available provider and honors explicit selection precedence", async () => { const alpha = candidate("alpha"); const beta = candidate("beta", { available: false }); @@ -104,7 +145,7 @@ it("selects the sole available provider and honors explicit selection precedence }); }); -describe("analysis provider registry: binding failures and cancellation", () => { +describe("analysis provider registry: selection failures", () => { it("returns actionable unknown, unavailable, and unsupported failures", async () => { const unavailable = candidate("alpha", { available: false }); const registry = new AnalysisProviderRegistry([unavailable.provider]); @@ -207,7 +248,9 @@ describe("analysis provider registry: binding failures and cancellation", () => }); } }); +}); +describe("analysis provider registry: profile and cleanup failures", () => { it("does not bind a profile with unresolved version or adapter failure", async () => { const unresolved = candidate("unresolved", { profile: "missing" }); const failed = candidate("failed", { profile: "error" }); @@ -240,6 +283,36 @@ describe("analysis provider registry: binding failures and cancellation", () => }); }); + it("aborts automatic selection when a provider reports unconfirmed cleanup", async () => { + const ghidra = candidate("ghidra"); + const alternate = candidate("hopper"); + const cleanupFailure = new ProviderCleanupError( + "ghidra", + ["/opt/ghidra/data/languages/x86.ldefs"], + { reason: "descriptor close failed" }, + { operation: "resolve_analysis_profile" }, + ); + const provider: AnalysisProviderCandidate = { + ...ghidra.provider, + resolveAnalysisProfile: () => Promise.resolve(err(cleanupFailure)), + }; + + const selected = await new AnalysisProviderRegistry([ + alternate.provider, + provider, + ]).select(DATABASE_TARGET); + + expect(selected).toMatchObject({ + ok: false, + error: { + _tag: "ProviderAdapterError", + providerId: "ghidra", + cleanupIncomplete: true, + }, + }); + expect(alternate.created).toEqual([]); + }); + it("cancels a pending profile probe even when the adapter ignores its signal", async () => { const alpha = candidate("alpha"); let observedSignal: AbortSignal | undefined; diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index 0336778811..c6a1739a31 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -306,6 +306,10 @@ export class AnalysisProviderRegistry { throw new TypeError("Cancelled candidate evaluation disappeared"); return result.value; }); + const cleanupFailure = evaluations.find( + ({ profileError }) => profileError?.cleanupIncomplete === true, + )?.profileError; + if (cleanupFailure !== undefined) return err(cleanupFailure); const statuses = evaluations.map(({ status }) => status); const usable = evaluations.filter( ( @@ -334,23 +338,29 @@ export class AnalysisProviderRegistry { const candidateStatus = ( provider: AnalysisProviderCandidate, target: BinaryTarget | undefined, -): AnalysisProviderCandidateStatus => ({ - provider: structuredClone(provider.identity()), - availability: cloneAvailability(provider.inspectAvailability()), - targetSupport: - target === undefined - ? { - status: "unknown", - code: null, - reason: "No target is open.", - diagnostics: {}, - } - : cloneTargetSupport(provider.inspectTargetSupport(target)), - selected: false, - capabilities: provider - .capabilities() - .map((capability) => structuredClone(capability)), -}); +): AnalysisProviderCandidateStatus => { + const readiness = provider.inspectReadinessSnapshot?.() ?? { + availability: provider.inspectAvailability(), + capabilities: provider.capabilities(), + }; + return { + provider: structuredClone(provider.identity()), + availability: cloneAvailability(readiness.availability), + targetSupport: + target === undefined + ? { + status: "unknown", + code: null, + reason: "No target is open.", + diagnostics: {}, + } + : cloneTargetSupport(provider.inspectTargetSupport(target)), + selected: false, + capabilities: readiness.capabilities.map((capability) => + structuredClone(capability), + ), + }; +}; const signalIsAborted = (signal: AbortSignal | undefined): boolean => signal?.aborted === true; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index da39ce53c0..24158e4a1c 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -125,7 +125,7 @@ export const resolveGhidraAnalysisProfile = async ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", - call_relationship_coverage: "resolved-call-reference-diagnostics-v1", + call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", string_inventory_evidence: "defined-data-coverage-v1", @@ -134,7 +134,8 @@ export const resolveGhidraAnalysisProfile = async ( "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", - function_boundary_observations: "ghidra-terminal-call-limitations-v2", + function_boundary_observations: + "ghidra-function-boundary-limitations-v3", no_return_repair: installation.platform === "win32" ? "disabled-windows-p0" @@ -222,7 +223,7 @@ export const resolveGhidraAnalysisProfile = async ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", - call_relationship_coverage: "resolved-call-reference-diagnostics-v1", + call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", string_inventory_evidence: "defined-data-coverage-v1", location_resolution: "explicit-address-exact-entry-symbol-first-v3", @@ -230,7 +231,8 @@ export const resolveGhidraAnalysisProfile = async ( "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", - function_boundary_observations: "ghidra-terminal-call-limitations-v2", + function_boundary_observations: + "ghidra-function-boundary-limitations-v3", no_return_repair: installation.platform === "win32" ? "disabled-windows-p0" diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index d1dee3f01a..acbd58ee68 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -58,6 +58,7 @@ import { sameRegularFileState, } from "../filesystem/RegularFile.js"; import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { redactCapturedTransportCredential } from "../process/ProviderDiagnosticRedaction.js"; export type GhidraTargetSnapshot = | { readonly kind: "captured"; readonly bytes: Buffer } @@ -119,7 +120,7 @@ export class GhidraClient { #closePromise: Promise | undefined; readonly #failure = bindGhidraSessionFailure( () => this.#diagnostics(), - (value) => this.#redactAuthentication(value), + (value, incomplete) => this.#redactAuthentication(value, incomplete), ); readonly #wire: GhidraWire; readonly #requestQueue: GhidraRequestQueue; @@ -662,25 +663,47 @@ export class GhidraClient { maxDiagnosticBytes: GHIDRA_PROCESS_DIAGNOSTIC_BYTES, onDiagnostic: (event) => this.#onProcessDiagnostic(event), }); - const connected = await this.#connect(endpoint, deadline); - if (!connected.ok) { - const failure = connected.error; - return this.#startupFailure(failure); - } - const completed = await completeGhidraStartupHandshake({ - deadline, - request: (method, params, requestOptions) => - this.#wire.request(method, params, requestOptions), - parseSessionInfo: (value) => this.#parseSessionInfo(value), - failure: this.#failure, - startupTimeoutMs: this.#options.startupTimeoutMs, - }); + const completed = await this.#connectAndHandshake(endpoint, deadline); if (!completed.ok) return this.#startupFailure(completed.error); await this.#lineage.observe(this.#launch); this.#requestQueue.reopen(); return completed; } + async #connectAndHandshake( + endpoint: GhidraEndpoint, + deadline: ProviderStartupDeadline, + ): Promise { + // Retry only a dropped loopback transport, never identity or protocol failures. + for (let attempt = 0; ; attempt += 1) { + const connected = await this.#connect(endpoint, deadline); + if (!connected.ok) return connected; + const completed = await completeGhidraStartupHandshake({ + deadline, + request: (method, params, requestOptions) => + this.#wire.request(method, params, requestOptions), + parseSessionInfo: (value) => this.#parseSessionInfo(value), + failure: this.#failure, + startupTimeoutMs: this.#options.startupTimeoutMs, + }); + if ( + completed.ok || + completed.error.kind !== "process" || + endpoint.transport !== "authenticated-loopback-tcp" || + attempt >= 2 || + deadline.signal.aborted || + this.#closing || + this.#process?.observedExit() !== undefined + ) + return completed; + this.#detachSocket?.(); + this.#detachSocket = undefined; + this.#socket?.destroy(); + this.#socket = undefined; + this.#responseBuffer.reset(); + } + } + async #connect( endpoint: GhidraEndpoint, deadline: ProviderStartupDeadline, @@ -880,9 +903,15 @@ export class GhidraClient { ); } - #redactAuthentication(value: string): string { - for (const token of this.#authenticationTokens) - value = value.replaceAll(token, "[REDACTED]"); + #redactAuthentication(value: string, incomplete = false): string { + for (const token of this.#authenticationTokens) { + const bytes = Buffer.byteLength(value); + value = redactCapturedTransportCredential( + { text: value, bytes, observedBytes: bytes + (incomplete ? 1 : 0) }, + token, + "[REDACTED]", + ); + } return value; } diff --git a/src/ghidra/GhidraDoctor.test.ts b/src/ghidra/GhidraDoctor.test.ts index 47ec9c8821..6a85045474 100644 --- a/src/ghidra/GhidraDoctor.test.ts +++ b/src/ghidra/GhidraDoctor.test.ts @@ -4,7 +4,7 @@ import { runDoctor, type DoctorHost } from "../application/Doctor.js"; import { createDoctorHostFixture } from "../application/Doctor.fixture.js"; import { providerRegistrationEnvironment } from "../application/SetupRegistrationEnvironment.js"; import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; -import * as windowsAuthority from "../process/WindowsAuthority.js"; +import * as nativeLoader from "../windows/WindowsNativeLoader.js"; import { inspectGhidraInstallation, type GhidraInstallationHost, @@ -180,13 +180,11 @@ describe("Ghidra doctor integration", () => { describe("Windows Ghidra doctor authority", () => { beforeEach(() => { - vi.spyOn(windowsAuthority, "hasWindowsNativeAuthority").mockReturnValue( - false, - ); - vi.spyOn( - windowsAuthority, - "windowsNativeAuthorityUnavailableReason", - ).mockReturnValue(WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON); + vi.spyOn(nativeLoader, "systemWindowsNativeAuthority").mockReturnValue({ + available: false, + failureStage: "artifact", + reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }); }); afterEach(() => vi.restoreAllMocks()); it("distinguishes a valid Windows installation from unavailable native authority", async () => { @@ -228,9 +226,9 @@ describe("Windows Ghidra doctor authority", () => { result.checks.find(({ name }) => name === "ghidra-native_authority"), ).toMatchObject({ ok: false, - classification: "unsupported_host", + classification: "missing_dependency", detail: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, - remediation: expect.stringContaining("cannot enable"), + remediation: expect.stringContaining("matching Windows x64 REA package"), }); }); @@ -252,7 +250,109 @@ describe("Windows Ghidra doctor authority", () => { expect(result.scope_checks).toContainEqual( expect.objectContaining({ name: "ghidra-native_authority", - classification: "unsupported_host", + classification: "missing_dependency", + detail: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }), + ); + }); +}); + +describe("Windows native doctor failure diagnostics", () => { + beforeEach(() => { + vi.spyOn(nativeLoader, "systemWindowsNativeAuthority").mockReturnValue({ + available: false, + failureStage: "artifact", + reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }); + }); + afterEach(() => vi.restoreAllMocks()); + + it.each([ + ["host", "unsupported_host", "unsupported_host", "Windows x64 host"], + [ + "artifact", + "runtime_missing", + "missing_dependency", + "bundled native addon", + ], + [ + "self-test", + "runtime_unavailable", + "runtime_failure", + "preserved native self-test error", + ], + [ + "inspection", + "runtime_unavailable", + "runtime_failure", + "every required security control", + ], + ] as const)( + "projects %s failure without blaming Ghidra", + async (failureStage, code, classification, remediation) => { + const reason = + "Windows native failure: remove_owned_directory: directory is not empty (145)"; + const load = vi.mocked(nativeLoader.systemWindowsNativeAuthority); + load.mockReturnValue({ available: false, failureStage, reason }); + const result = await runDoctor( + undefined, + { + ...doctorHost(inspection({ installDir: INSTALL, platform: "win32" })), + platform: "win32", + linuxDistribution: () => Promise.resolve(undefined), + }, + { providers: ["ghidra"] }, + ); + expect(load).toHaveBeenCalledTimes(1); + expect(result.healthy).toBe(false); + expect(result.providerInspections?.[0]).toMatchObject({ + available: false, + registrationEnvironment: { GHIDRA_INSTALL_DIR: INSTALL }, + checks: expect.arrayContaining([ + expect.objectContaining({ + name: "native_authority", + ok: false, + code, + detail: reason, + }), + ]), + }); + expect(result.scope_checks).toContainEqual( + expect.objectContaining({ + name: "ghidra-native_authority", + classification, + detail: reason, + remediation: expect.stringContaining(remediation), + }), + ); + }, + ); + + it("keeps one failure observation even when the next query would differ", () => { + const load = vi.mocked(nativeLoader.systemWindowsNativeAuthority); + load.mockReturnValueOnce({ + available: false, + failureStage: "self-test", + retryable: true, + reason: "cleanup failed (145)", + }); + const ghidra = inspection({ installDir: INSTALL, platform: "win32" }); + const failed = projectGhidraDoctorInspection(ghidra); + expect(load).toHaveBeenCalledTimes(1); + expect(failed.checks).toContainEqual( + expect.objectContaining({ + name: "native_authority", + ok: false, + code: "runtime_unavailable", + detail: "cleanup failed (145)", + }), + ); + const later = projectGhidraDoctorInspection(ghidra); + expect(load).toHaveBeenCalledTimes(2); + expect(later.checks).toContainEqual( + expect.objectContaining({ + name: "native_authority", + code: "runtime_missing", detail: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, }), ); diff --git a/src/ghidra/GhidraDoctor.ts b/src/ghidra/GhidraDoctor.ts index 5e74a91995..1590ea14b8 100644 --- a/src/ghidra/GhidraDoctor.ts +++ b/src/ghidra/GhidraDoctor.ts @@ -2,10 +2,7 @@ import type { DoctorProviderCheck, DoctorProviderInspection, } from "../application/Doctor.js"; -import { - hasWindowsNativeAuthority, - windowsNativeAuthorityUnavailableReason, -} from "../process/WindowsAuthority.js"; +import { inspectWindowsNativeReadiness } from "../process/WindowsAuthority.js"; import { inspectGhidraInstallation, type GhidraInstallationCheck, @@ -16,10 +13,11 @@ import { export const projectGhidraDoctorInspection = ( inspection: GhidraInstallationInspection, ): DoctorProviderInspection => { - const windowsAuthorityUnavailable = - inspection.status === "available" && - inspection.platform === "win32" && - !hasWindowsNativeAuthority(inspection.platform); + const readiness = + inspection.status === "available" && inspection.platform === "win32" + ? inspectWindowsNativeReadiness(inspection.platform) + : undefined; + const windowsAuthorityUnavailable = readiness?.available === false; return { id: "ghidra", configured: inspection.installDir !== null, @@ -37,19 +35,22 @@ export const projectGhidraDoctorInspection = ( ...selectedChecks(inspection).map((check) => projectCheck(check, inspection.installDir !== null), ), - ...(inspection.status === "available" && inspection.platform === "win32" + ...(readiness !== undefined ? [ { name: "native_authority", ok: !windowsAuthorityUnavailable, - code: windowsAuthorityUnavailable ? "unsupported_host" : null, - detail: windowsAuthorityUnavailable - ? windowsNativeAuthorityUnavailableReason(inspection.platform) + code: !readiness.available ? readiness.code : null, + detail: !readiness.available + ? readiness.reason : "Verified Windows native Job Object, private DACL, and reparse-safe handle controls; local NTFS only.", - remediation: windowsAuthorityUnavailable - ? "Use the matching Windows x64 REA package with its bundled native addon and a writable local NTFS temporary directory. The native failure above identifies the failed constraint; Ghidra installation or registration changes cannot enable a missing native backend." - : null, - classification: "unsupported_host" as const, + remediation: !readiness.available ? readiness.remediation : null, + classification: + !readiness.available && readiness.code === "runtime_unavailable" + ? ("runtime_failure" as const) + : !readiness.available && readiness.code === "runtime_missing" + ? ("missing_dependency" as const) + : ("unsupported_host" as const), }, ] : []), diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index e7e5d8e9df..f40cc1ca37 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -43,7 +43,152 @@ const parseFunctionValueFixture = ( return parsed.ok ? { ok: true as const, value: parsed.value.value } : parsed; }; +describe("Ghidra unresolved call locations", () => { + it("rejects a noncanonical unresolved call site rather than retaining unverifiable location evidence", () => { + expect( + parseGhidraFunctionResult("analyze_function", { + value: { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + unresolved_calls: [ + { address: "not-an-address", reason: "No call destination." }, + ], + }, + limitations: [], + relationship_coverage: { + status: "partial", + limitations: ["An unresolved call remains."], + }, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); +}); + describe("Ghidra function-analysis result values", () => { + it("preserves unresolved Swift jump omissions", () => { + const dossier = { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "small", + address: null, + byte_length: 5, + reason: "unresolved_jump", + registers: ["x0", "x1"], + words: ["0x5000000000000005", "0x0000000000000000"], + }, + ], + }; + expect( + parseGhidraFunctionResult("analyze_function", { + value: dossier, + limitations: [], + }), + ).toMatchObject({ + ok: true, + value: { + value: { + swift_string_literal_omissions: + dossier.swift_string_literal_omissions, + }, + }, + }); + }); + + it("retains unavailable pseudocode instead of treating its empty representation as observed source", () => { + const availability = { + pseudocode: { + available: false, + reason: "Ghidra does not decompile external functions.", + }, + comments: { available: true }, + }; + const parsed = parseGhidraFunctionResult("analyze_function", { + value: { + ...functionDossierSchema.parse(ghidraFunctionDossier()), + pseudocode: "", + facet_availability: availability, + }, + limitations: [], + }); + expect(parsed).toMatchObject({ + ok: true, + value: { value: { pseudocode: "", facet_availability: availability } }, + }); + }); + it.each([ + { status: "complete", diagnostics: ["Resolved static references only."] }, + { + status: "partial", + limitations: ["Only the first instruction range was examined."], + }, + { status: "unknown", reason: "Relationship inventory was unavailable." }, + { + status: "partial", + limitations: ["Missing callers."], + limited_directions: ["callers"], + }, + { + status: "unknown", + reason: "Missing callees.", + limited_directions: ["callees"], + }, + ])("preserves observed $status relationship coverage", (coverage) => { + const parsed = parseGhidraFunctionResult("analyze_function", { + value: ghidraFunctionDossier(), + limitations: [], + relationship_coverage: coverage, + }); + expect(parsed).toMatchObject({ + ok: true, + value: { relationshipCoverage: coverage }, + }); + }); + + it("preserves omitted relationship coverage for legacy observations", () => { + const parsed = parseGhidraFunctionResult("analyze_function", { + value: ghidraFunctionDossier(), + limitations: [], + }); + if (!parsed.ok) throw parsed.error; + expect(parsed.value).not.toHaveProperty("relationshipCoverage"); + }); +}); + +describe("Ghidra relationship metadata validation", () => { + it.each([ + { status: "partial", limitations: [] }, + { status: "unknown", reason: "" }, + { status: "complete", limitations: ["Truncated."] }, + { status: "partial", limitations: ["Truncated."], limited_directions: [] }, + { + status: "partial", + limitations: ["Truncated."], + limited_directions: ["callers", "callers"], + }, + { status: "unknown", reason: "Missing.", limited_directions: ["incoming"] }, + { status: "complete", limited_directions: ["callers"] }, + ])("rejects malformed relationship coverage %j", (coverage) => { + expect( + parseGhidraFunctionResult("analyze_function", { + value: ghidraFunctionDossier(), + limitations: [], + relationship_coverage: coverage, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); + + it("rejects relationship metadata on unrelated function operations", () => { + expect( + parseGhidraFunctionResult("procedure_assembly", { + value: "ret", + limitations: [], + relationship_coverage: { status: "complete" }, + }), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" } }); + }); + it("preserves file-offset resolution in Evidence locations and rejects inexact offsets", () => { const selection = { file_offset: 0x210, diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index a6e62c93cf..d038ac82e7 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -27,6 +27,10 @@ import { import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js"; import { nativeValueFlowSchema } from "../domain/native/nativeValueFlow.js"; import type { EvidenceLocation } from "../domain/evidence.js"; +import { + relationshipCoverageSchema, + type RelationshipCoverage, +} from "../domain/relationshipCoverage.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import { @@ -307,6 +311,7 @@ const ghidraFunctionDossier = functionDossierSchema ...value.comments.map(({ address }) => address), ...value.callers.map(({ address }) => address), ...value.callees.map(({ address }) => address), + ...(value.unresolved_calls ?? []).map(({ address }) => address), ...value.incoming_references.flatMap(referenceAddresses), ...value.outgoing_references.flatMap(referenceAddresses), ...value.referenced_strings.flatMap(({ address, source_address }) => [ @@ -419,6 +424,7 @@ const functionBoundaryObservationOperations: ReadonlySet["selection"]; @@ -450,13 +458,17 @@ export const parseGhidraFunctionResult = ( const envelope = functionObservationEnvelope.safeParse(value); if ( !envelope.success || + (operation !== "analyze_function" && + operation !== "procedure_callers" && + operation !== "procedure_callees" && + envelope.data.relationship_coverage !== undefined) || (operation !== "procedure_pseudo_code" && envelope.data.selection !== undefined) ) return err( new AnalysisOutputError( operation, - "Ghidra bridge omitted the function observation envelope", + "Ghidra bridge returned an invalid function observation envelope for this operation", envelope.success ? undefined : { cause: envelope.error }, ), ); @@ -469,6 +481,9 @@ export const parseGhidraFunctionResult = ( ? ok({ value: jsonValueSchema.parse(parsed.data), limitations: observation.limitations, + ...(observation.relationship_coverage === undefined + ? {} + : { relationshipCoverage: observation.relationship_coverage }), ...(observation.selection === undefined ? {} : { diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index 95c1967b79..f30f6ef7a5 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -252,7 +252,9 @@ export interface GhidraJavaObservation { readonly version: string; readonly major: number; readonly home: string; - readonly bits: number; + readonly bits: number | null; + /** Present when Java reported a bitness value that could not be parsed exactly. */ + readonly unparsedBits?: string; readonly runtime: "jdk" | "jre"; readonly vmName?: string; /** @@ -803,12 +805,23 @@ const javaCheck = ( ghidraJavaMajorAccepted(observation.major, requirement) && observation.bits === 64 && observation.runtime === "jdk"; + const bitness = + observation === undefined + ? undefined + : observation.bits === null + ? `bitness unknown${observation.unparsedBits === undefined ? "" : ` (sun.arch.data.model = ${JSON.stringify(observation.unparsedBits)})`}` + : `${String(observation.bits)}-bit`; + const knownMismatch = + observation !== undefined && + (!ghidraJavaMajorAccepted(observation.major, requirement) || + (observation.bits !== null && observation.bits !== 64) || + observation.runtime !== "jdk"); const detail = observation === undefined ? requirement.unresolved === null ? command : `${command}; ${requirement.unresolved}` - : `${observation.version}; ${String(observation.bits)}-bit; ${observation.runtime === "jdk" ? "JDK" : "runtime only"}; ${observation.home}; accepted ${range}`; + : `${observation.version}; ${bitness}; ${observation.runtime === "jdk" ? "JDK" : "runtime only"}; ${observation.home}; accepted ${range}`; return installationCheck({ name: "java", passed: supported, @@ -817,7 +830,9 @@ const javaCheck = ( ? "version_unresolved" : observation === undefined ? "runtime_missing" - : "unsupported_version", + : knownMismatch + ? "unsupported_version" + : "version_unresolved", detail, remediation: javaRemediation(requirement, configuredHome, range), }); @@ -905,11 +920,22 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ const vmName = javaProperty(output, "java.vm.name"); const osArch = javaProperty(output, "os.arch"); const osName = javaProperty(output, "os.name"); - const bits = Number.parseInt( - javaProperty(output, "sun.arch.data.model") ?? "0", - 10, - ); - if (version === null || home === null) return undefined; + const reportedBits = javaProperty(output, "sun.arch.data.model"); + const parsedBits = + reportedBits !== null && /^\d+$/.test(reportedBits) + ? Number(reportedBits) + : null; + const bits = + parsedBits !== null && Number.isSafeInteger(parsedBits) + ? parsedBits + : null; + if ( + version === null || + version.length === 0 || + home === null || + home.length === 0 + ) + return undefined; return { version, major: javaMajor(version), @@ -918,6 +944,9 @@ const systemGhidraInstallationHost = (): GhidraInstallationHost => ({ ...(osArch === null ? {} : { osArch }), ...(osName === null ? {} : { osName }), bits, + ...(reportedBits === null || bits !== null + ? {} + : { unparsedBits: reportedBits }), runtime: executablePath( (process.platform === "win32" ? win32 : posix).join( home, @@ -944,7 +973,7 @@ const executablePath = (path: string): boolean => { const javaProperty = (output: string, name: string): string | null => new RegExp( - `^\\s*${name.replaceAll(".", "\\.")}\\s*=\\s*(.+?)\\s*$`, + `^[ \\t]*${name.replaceAll(".", "\\.")}[ \\t]*=[ \\t]*([^\\r\\n]*?)[ \\t]*\\r?$`, "mu", ).exec(output)?.[1] ?? null; diff --git a/src/ghidra/GhidraJavaLaunch.ts b/src/ghidra/GhidraJavaLaunch.ts index 317c0939f0..598a152c2a 100644 --- a/src/ghidra/GhidraJavaLaunch.ts +++ b/src/ghidra/GhidraJavaLaunch.ts @@ -1,6 +1,39 @@ import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { dirname, join } from "node:path"; -import { execFileOutput } from "../process/ExecFileOutput.js"; +import { + execFileOutput, + execFileOutputFailure, + type ExecFileOutputFailure, +} from "../process/ExecFileOutput.js"; + +export const GHIDRA_LAUNCH_SUPPORT_CAPTURE_BYTES = 64 * 1024; +type LaunchSupportPhase = "-envvars" | "-vmargs"; +type LaunchSupportOutcome = Omit< + ExecFileOutputFailure, + "killed" | "outputTruncated" +> & { + readonly killed: boolean | null; + readonly outputTruncated: boolean | null; +}; + +export class GhidraLaunchSupportError extends Error { + readonly outcome: LaunchSupportOutcome | undefined; + + constructor(phase: LaunchSupportPhase, cause: unknown) { + super(`Ghidra LaunchSupport ${phase} failed`, { cause }); + this.name = "GhidraLaunchSupportError"; + const outcome = execFileOutputFailure(cause); + this.outcome = + outcome === undefined + ? undefined + : { + ...outcome, + killed: outcome.code === "ABORT_ERR" ? null : outcome.killed, + outputTruncated: + outcome.code === "ABORT_ERR" ? null : outcome.outputTruncated, + }; + } +} /** JVM option environments remove quotes, including quotes inside an option. */ const jvmEnvironmentArguments = (value: string): readonly string[] => { @@ -85,17 +118,20 @@ export const ghidraJavaLaunch = async (options: { const environment = { ...snapshotEnvironment(options.environment, options.platform), }; - const configuredEnvironment = await execFileOutput( - command, - [...supportArguments, "-envvars"], - { - env: environment, - // Wait for the helper's exit on abort so a cancelled or timed-out - // startup never leaves the probe running without an owning process. - stopSignal: "SIGTERM", - ...(options.signal === undefined ? {} : { signal: options.signal }), - }, - ); + const supportOutput = async (phase: LaunchSupportPhase) => { + try { + return await execFileOutput(command, [...supportArguments, phase], { + env: environment, + maxBuffer: GHIDRA_LAUNCH_SUPPORT_CAPTURE_BYTES, + // Wait for the helper's exit on abort so no probe outlives its owner. + stopSignal: "SIGTERM", + ...(options.signal === undefined ? {} : { signal: options.signal }), + }); + } catch (cause: unknown) { + throw new GhidraLaunchSupportError(phase, cause); + } + }; + const configuredEnvironment = await supportOutput("-envvars"); options.onStderr("-envvars", configuredEnvironment.stderr); for (const line of configuredEnvironment.stdout.split(/\r?\n/u)) { if (line === "") continue; @@ -121,17 +157,7 @@ export const ghidraJavaLaunch = async (options: { ] as const) { rejectJvmPlatformOverrides(jvmEnvironmentArguments(environment[key] ?? "")); } - const configuredArguments = await execFileOutput( - command, - [...supportArguments, "-vmargs"], - { - env: environment, - // Wait for the helper's exit on abort so a cancelled or timed-out - // startup never leaves the probe running without an owning process. - stopSignal: "SIGTERM", - ...(options.signal === undefined ? {} : { signal: options.signal }), - }, - ); + const configuredArguments = await supportOutput("-vmargs"); options.onStderr("-vmargs", configuredArguments.stderr); const vmArguments = configuredArguments.stdout .split(/\r?\n/u) diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 6373f187f3..1cde8769f4 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -1,9 +1,12 @@ -import { readdir, readFile } from "node:fs/promises"; +import { open, readdir } from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; import { join } from "node:path"; import { DOMParser, onWarningStopParsing } from "@xmldom/xmldom"; import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; import { ConfigurationError } from "../domain/configurationErrors.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; /** One language definition and the compiler specs it declares. */ export interface GhidraLanguageDefinition { @@ -16,6 +19,62 @@ const MAX_LDEFS_BYTES = 4 * 1024 * 1024; // Processor modules ship under Processors; installed extensions may add more. const LANGUAGE_ROOTS = ["Ghidra/Processors", "Ghidra/Extensions"] as const; +interface RetainedLanguageFile { + readonly path: string; + readonly owner: OwnedFileHandle; +} + +/** Language definition handles whose close is unconfirmed but retryable. */ +const pendingLanguageFileCleanup = new Set(); + +/** + * Cleanup of one or more language definition handles is unconfirmed. Any + * read failure that preceded it is kept as the primary failure. + */ +export class GhidraLanguageCatalogCleanupError extends Error { + constructor( + readonly paths: readonly string[], + readonly causes: readonly unknown[], + readonly readFailure?: unknown, + ) { + super( + `Language definition cleanup is unconfirmed: ${paths.join(", ")}`, + readFailure === undefined ? undefined : { cause: readFailure }, + ); + } +} + +/** Close one owner; retain it only while a later close can still succeed. */ +const closeLanguageFile = async ( + file: RetainedLanguageFile, +): Promise => { + try { + await file.owner.close(); + pendingLanguageFileCleanup.delete(file); + return undefined; + } catch (cause: unknown) { + // Descriptor invalidation does not prove that this owner confirmed close. + // Keep even sticky failures in the lifecycle scope so later reads surface + // the unresolved cleanup instead of silently abandoning ownership. + pendingLanguageFileCleanup.add(file); + return cause; + } +}; + +/** Retry every retained owner before reporting the failures together. */ +const retryLanguageFileCleanup = async (): Promise => { + const failed: { readonly path: string; readonly cause: unknown }[] = []; + for (const file of pendingLanguageFileCleanup) { + const cause = await closeLanguageFile(file); + if (cause !== undefined) failed.push({ path: file.path, cause }); + } + if (failed.length > 0) + throw new GhidraLanguageCatalogCleanupError( + failed.map(({ path }) => path), + failed.map(({ cause }) => cause), + ); +}; + /** Parse the `` elements of one `.ldefs` document. */ export const parseGhidraLanguageDefinitions = ( text: string, @@ -59,10 +118,40 @@ const languageDirectories = async (installDir: string): Promise => { return directories; }; +/** Read at most the catalog limit; undefined marks an oversized file. */ +const readLanguageDefinitionFile = async ( + path: string, +): Promise => { + await retryLanguageFileCleanup(); + const file = { path, owner: new OwnedFileHandle(await open(path, "r")) }; + let read: + | { readonly ok: true; readonly bytes: Buffer | undefined } + | { readonly ok: false; readonly cause: unknown }; + try { + read = { + ok: true, + bytes: await readBoundedFileBytes(file.owner.handle, MAX_LDEFS_BYTES), + }; + } catch (cause: unknown) { + read = { ok: false, cause }; + } + const cleanup = await closeLanguageFile(file); + if (cleanup !== undefined) + throw new GhidraLanguageCatalogCleanupError( + [path], + [cleanup], + read.ok ? undefined : read.cause, + ); + if (!read.ok) throw read.cause; + return read.bytes; +}; + /** Read every language definition shipped by one Ghidra installation. */ export const readGhidraLanguageCatalog = async ( installDir: string, ): Promise => { + // Surface retained cleanup even when this catalog has no definitions to read. + await retryLanguageFileCleanup(); const definitions: GhidraLanguageDefinition[] = []; for (const directory of await languageDirectories(installDir)) { let entries: string[]; @@ -73,8 +162,8 @@ export const readGhidraLanguageCatalog = async ( } for (const entry of entries.sort()) { if (!entry.endsWith(".ldefs")) continue; - const bytes = await readFile(join(directory, entry)); - if (bytes.length > MAX_LDEFS_BYTES) continue; + const bytes = await readLanguageDefinitionFile(join(directory, entry)); + if (bytes === undefined) continue; // XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature, // not document content; TextDecoder consumes it, Buffer#toString does not. definitions.push( diff --git a/src/ghidra/GhidraLaunchFailure.test.ts b/src/ghidra/GhidraLaunchFailure.test.ts index 246ec82d0f..d25cd9932c 100644 --- a/src/ghidra/GhidraLaunchFailure.test.ts +++ b/src/ghidra/GhidraLaunchFailure.test.ts @@ -56,7 +56,8 @@ const launchOutcomeOf = (error: Error): Record | undefined => { string, unknown >; - const outcome = cause["launch_outcome"] as Record; + const nested = cause["cause"] as Record | undefined; + const outcome = nested?.["command_outcome"] as Record; return outcome === undefined ? undefined : outcome; }; @@ -78,12 +79,16 @@ describe("Ghidra launch failure diagnostics", () => { launched.error instanceof GhidraLaunchError ? launched.error.capturedOutput : undefined; - expect(captured).toBeDefined(); - expect(captured?.exit_code).toBe(3); - expect(captured?.signal).toBeNull(); - expect(captured?.stdout).toBe(""); - expect(captured?.stderr).toContain("jvm probe exploded"); - expect(captured?.truncated).toBe(false); + expect(captured).toBeUndefined(); + const outcome = launchOutcomeOf(launched.error); + expect(outcome).toMatchObject({ + killed: false, + outputTruncated: false, + stdoutTruncated: false, + stderrTruncated: false, + stdout: "", + }); + expect(outcome?.["stderr"]).toContain("jvm probe exploded"); } finally { await rm(javaHome, { recursive: true, force: true }); await rm(runtimeRoot, { recursive: true, force: true }); @@ -103,10 +108,11 @@ describe("Ghidra launch failure diagnostics", () => { if (launched.ok) throw new Error("expected the launch to fail"); const outcome = launchOutcomeOf(launched.error); expect(outcome).toBeDefined(); - expect(outcome?.["exit_code"]).toBe(3); - expect(outcome?.["signal"]).toBeNull(); + expect(outcome?.["killed"]).toBe(false); + expect(outcome?.["outputTruncated"]).toBe(false); + expect(outcome?.["stderrTruncated"]).toBe(false); expect(outcome?.["stderr"]).toContain("jvm probe exploded"); - expect(outcome?.["truncated"]).toBe(false); + expect(outcome?.["outputTruncated"]).toBe(false); } finally { await rm(javaHome, { recursive: true, force: true }); await rm(runtimeRoot, { recursive: true, force: true }); @@ -126,9 +132,9 @@ describe("Ghidra launch failure diagnostics", () => { expect(launched.ok).toBe(false); if (launched.ok) throw new Error("expected the launch to fail"); const outcome = launchOutcomeOf(launched.error); - expect(outcome?.["truncated"]).toBe(true); - expect(outcome?.["exit_code"]).toBe(1); - expect(outcome?.["stderr_bytes"]).toBe(oversized.length); + expect(outcome?.["stderrTruncated"]).toBe(true); + expect(outcome?.["outputTruncated"]).toBe(false); + expect(outcome?.["stderrBytes"]).toBe(oversized.length); expect((outcome?.["stderr"] as string).length).toBeLessThanOrEqual( EXPECTED_OUTPUT_BOUND_BYTES, ); diff --git a/src/ghidra/GhidraMipsProfile.ts b/src/ghidra/GhidraMipsProfile.ts index 76a753ccb6..51ac546f27 100644 --- a/src/ghidra/GhidraMipsProfile.ts +++ b/src/ghidra/GhidraMipsProfile.ts @@ -137,6 +137,11 @@ export const ghidraMipsUnsupportedReason = ( if (load.symbolRelativeCount !== 0) return "Symbol-bearing R_MIPS_REL32 is unsupported: Ghidra normalizes MIPS16 function addresses and can lose the source function-pointer mode tag. Symbol-index-zero REL32 is verified separately."; } + if ((metadata.limitations?.length ?? 0) > 0) + return [ + "MIPS ABI declaration inspection is incomplete; unsupported attributes or conflicting declaration sources may remain unknown.", + ...(metadata.limitations ?? []), + ].join(" "); const abi = metadata.abiFlags; if (shared && abi === null && (metadata.limitations?.length ?? 0) === 0) return null; // Inspected absence leaves FP semantics unknown, never inferred. diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index c69c83bd9b..9d4892fc84 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -8,7 +8,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { describe, expect, it, onTestFinished, vi } from "vitest"; +import { afterEach, describe, expect, it, onTestFinished, vi } from "vitest"; +import * as nativeLoader from "../windows/WindowsNativeLoader.js"; +import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; import { fixtureDosLoadImage } from "./GhidraLoadImage.fixture.js"; import { fixtureRawBinaryLoadImage } from "./GhidraRawBinaryLoadImage.fixture.js"; import { @@ -30,13 +32,12 @@ import { err, ok } from "../domain/result.js"; import type { Result } from "../domain/result.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; import { silentLogger } from "../logger.js"; -import * as WindowsNativeLoader from "../windows/WindowsNativeLoader.js"; -import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ProviderStartupDeadline } from "../process/ProviderDeadline.js"; const INSTALL = "/opt/ghidra_12.1.4_PUBLIC"; +afterEach(() => vi.restoreAllMocks()); const installationHost = ( platform: NodeJS.Platform = "linux", ): GhidraInstallationHost => ({ @@ -360,18 +361,20 @@ describe("Ghidra platform support", () => { availabilityCode: "unsupported_host", }); }); +}); +describe("Ghidra Windows platform support", () => { it("keeps Windows P0 unavailable until native isolation authority exists", () => { const native = vi - .spyOn(WindowsNativeLoader, "systemWindowsNativeAuthority") + .spyOn(nativeLoader, "systemWindowsNativeAuthority") .mockReturnValue({ available: false, + failureStage: "artifact", reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, }); onTestFinished(() => native.mockRestore()); const ghidra = provider(installationHost("win32")); const nativeApplication = peTarget("x86_64"); - for (const architecture of ["x86", "x86_64"] as const) expect(ghidra.inspectTargetSupport(peTarget(architecture))).toMatchObject( { @@ -386,11 +389,15 @@ describe("Ghidra platform support", () => { ); expect(ghidra.inspectAvailability()).toMatchObject({ status: "unavailable", - code: "unsupported_host", + code: "runtime_missing", reason: expect.stringContaining( "Windows native controls are unavailable", ), diagnostics: { + windows_native_failure: { + stage: "artifact", + remediation: expect.any(String), + }, windows_security: { job_object_process_ownership: expect.objectContaining({ available: false, @@ -412,7 +419,7 @@ describe("Ghidra platform support", () => { expect.objectContaining({ operation: "list_procedures", available: false, - availabilityCode: "unsupported_host", + availabilityCode: "runtime_missing", limitations: expect.arrayContaining([ expect.stringContaining( "matching packaged Windows x64 native addon", @@ -454,6 +461,125 @@ describe("Ghidra platform support", () => { }); }); +describe("Windows Ghidra native failure diagnostics", () => { + it.each([ + ["artifact", "runtime_missing"], + ["self-test", "runtime_unavailable"], + ["inspection", "runtime_unavailable"], + ] as const)( + "keeps Windows P0 unavailable after a %s failure", + async (failureStage, code) => { + const load = vi + .spyOn(nativeLoader, "systemWindowsNativeAuthority") + .mockReturnValue({ + available: false, + failureStage, + reason: "Windows native controls are unavailable: missing manifest", + }); + const ghidra = provider(installationHost("win32")); + const nativeApplication = peTarget("x86_64"); + + for (const architecture of ["x86", "x86_64"] as const) + expect( + ghidra.inspectTargetSupport(peTarget(architecture)), + ).toMatchObject({ + status: "supported", + diagnostics: { + host_platform: "win32", + architecture, + executable_role: "application", + managed: false, + }, + }); + expect(ghidra.inspectAvailability()).toMatchObject({ + status: "unavailable", + code, + reason: expect.stringContaining( + "Windows native controls are unavailable", + ), + diagnostics: { + windows_native_failure: { + stage: failureStage, + remediation: expect.any(String), + }, + windows_security: { + job_object_process_ownership: expect.objectContaining({ + available: false, + proof: "not-proven", + }), + private_runtime_dacl: expect.objectContaining({ + available: false, + proof: "not-proven", + }), + reparse_safe_path_admission: expect.objectContaining({ + available: false, + proof: "not-proven", + }), + }, + }, + }); + expect(load).toHaveBeenCalledTimes(1); + expect(ghidra.capabilities()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "list_procedures", + available: false, + availabilityCode: code, + limitations: expect.arrayContaining([ + expect.stringContaining( + "matching packaged Windows x64 native addon", + ), + expect.stringContaining( + "private DACLs, and Job Object ownership automatically", + ), + ]), + }), + ]), + ); + expect( + ghidra.inspectTargetSupport({ + ...nativeApplication, + executableRole: "non-executable", + }), + ).toMatchObject({ + status: "unsupported", + code: "target_role_unsupported", + }); + expect( + ghidra.inspectTargetSupport({ ...nativeApplication, managed: true }), + ).toMatchObject({ + status: "unsupported", + code: "managed_target_unsupported", + }); + expect( + ghidra.inspectTargetSupport(executableTarget("elf", "x86_64")), + ).toMatchObject({ + status: "unsupported", + code: "target_format_unsupported", + }); + expect( + ghidra.inspectTargetSupport(executableTarget("pe", "arm64")), + ).toMatchObject({ + status: "unsupported", + code: "architecture_unsupported", + }); + const client = ghidra.createClient(nativeApplication); + const health = await client.execute("health", {}); + if (health.ok) + throw new Error( + "Failed native admission must not create a ready client", + ); + expect(projectAnalysisError(health.error)).toMatchObject({ + code: "capability_unavailable", + details: { + reason: "Windows native controls are unavailable: missing manifest", + }, + }); + expect(load).toHaveBeenCalledTimes(3); + }, + ); +}); + describe("Ghidra Windows PE fixture admission", () => { it("admits generated PE32 and PE32+ DLLs and applications while preserving native-only Windows admission", async () => { const root = fileURLToPath(new URL("../../", import.meta.url)); @@ -633,7 +759,7 @@ describe("Ghidra client projection", () => { expect(resolved.value.profile.parameters).toMatchObject({ language_id: "auto-from-header", compiler_spec_id: "auto-default", - call_relationship_coverage: "resolved-call-reference-diagnostics-v1", + call_relationship_coverage: "function-body-call-coverage-v5", function_dossier_coverage: "observed-facets-and-call-relationships-v1", }); @@ -870,7 +996,16 @@ describe("Ghidra relationship coverage", () => { Promise.resolve( ok( operation === "procedure_callees" - ? { value: ["0x2000"], limitations: [] } + ? { + value: ["0x2000"], + limitations: [], + relationship_coverage: { + status: "complete", + diagnostics: [ + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals.", + ], + }, + } : { value: [], limitations: [] }, ), ), @@ -2228,3 +2363,104 @@ describe("Ghidra disabled analyzer options", () => { } }); }); + +describe("Ghidra queries after the owned provider is lost", () => { + const lostProviderSession = async ( + state: "exited" | "unreachable" | "idle", + failureKind: "process" | "protocol" | "remote" = "process", + answeredBeforeLoss = true, + ) => { + let calls = 0; + const ghidra = provider(installationHost(), () => ({ + start: async () => + err(new GhidraSessionError("start", "startup is not exercised")), + callTool: async () => { + calls += 1; + if (answeredBeforeLoss && calls === 1) return ok(null); + return err( + new GhidraSessionError( + failureKind, + "Ghidra bridge socket closed", + { failure_kind: failureKind }, + failureKind === "remote" ? { remoteCode: "internal" } : {}, + ), + ); + }, + close: async () => ok(null), + operationHealthSnapshot: () => + state === "idle" + ? undefined + : { + state, + stage: null, + retryAction: "restart_provider", + exitCode: state === "exited" ? 137 : null, + requests: [], + }, + })); + const target = executableTarget("elf", "x86_64"); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok || profile.value.profile === null) + throw new Error("Expected Ghidra analysis profile"); + const client = ghidra.createClient(target, profile.value.profile); + if (answeredBeforeLoss) await client.execute("list_documents", {}); + return client; + }; + + it("tells the caller to reopen the target when Ghidra exited", async () => { + const client = await lostProviderSession("exited"); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "execution_failure", + retryable: false, + details: { provider_id: "ghidra", operation: "list_documents" }, + }); + expect(projected.message).toContain("exited with code 137"); + expect(projected.message).toContain("Call open_binary"); + expect(projected.remediation.action).toContain("Call open_binary"); + expect(projected.remediation.action).not.toContain("rea doctor"); + }); + + it("tells the caller to reopen the target when the bridge became unreachable", async () => { + const client = await lostProviderSession("unreachable", "protocol"); + const result = await client.execute("list_procedures", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + const projected = projectAnalysisError(result.error); + expect(projected.message).toContain("became unreachable"); + expect(projected.remediation.action).toContain("Call open_binary"); + }); + + it("keeps installation guidance when Ghidra dies before it ever answered", async () => { + const client = await lostProviderSession("exited", "process", false); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + const projected = projectAnalysisError(result.error); + expect(projected.message).not.toContain("Call open_binary"); + expect(projected.remediation.action).toContain("rea doctor"); + }); + + it("keeps the generic failure when the provider is not known to be lost", async () => { + const client = await lostProviderSession("idle"); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(projectAnalysisError(result.error).remediation.action).toContain( + "rea doctor", + ); + }); + + it("does not reinterpret a bridge-reported failure as a lost provider", async () => { + const client = await lostProviderSession("exited", "remote"); + const result = await client.execute("list_documents", {}); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(projectAnalysisError(result.error).message).not.toContain( + "Call open_binary", + ); + }); +}); diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index b0ee8787d4..3cee8bacec 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -4,6 +4,7 @@ import type { AnalysisClientContext, AnalysisProfileResolutionOptions, AnalysisProviderCandidate, + AnalysisProviderReadinessSnapshot, CapabilityDescriptor, ProviderAvailability, ProviderIdentity, @@ -25,6 +26,7 @@ import { } from "../domain/jsonValue.js"; import { err, ok } from "../domain/result.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import type { Logger } from "pino"; import { GhidraClient } from "./GhidraClient.js"; import { @@ -38,6 +40,7 @@ import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { resolveGhidraAnalysisSeeds } from "./GhidraAnalysisSeeds.js"; import { admitGhidraLanguageOverride, + GhidraLanguageCatalogCleanupError, readGhidraLanguageCatalog, } from "./GhidraLanguageCatalog.js"; import { @@ -55,8 +58,8 @@ import { type GhidraProviderClientFactory, } from "./GhidraProviderClient.js"; import { - windowsNativeAuthorityUnavailableReason, - hasWindowsNativeAuthority, + inspectWindowsNativeReadiness, + type WindowsNativeReadiness, windowsNativeCapabilities, } from "../process/WindowsAuthority.js"; @@ -101,22 +104,43 @@ export class GhidraProvider implements AnalysisProviderCandidate { } inspectAvailability(): ProviderAvailability { + return this.inspectReadinessSnapshot().availability; + } + + inspectReadinessSnapshot(): AnalysisProviderReadinessSnapshot { const installation = this.#inspectInstallation(); + const readiness = + installation.platform === "win32" + ? inspectWindowsNativeReadiness(installation.platform) + : undefined; + return { + availability: this.#availability(installation, readiness), + capabilities: + readiness === undefined + ? CAPABILITIES + : windowsP0Capabilities(readiness), + }; + } + + #availability( + installation: GhidraInstallationInspection, + readiness: WindowsNativeReadiness | undefined, + ): ProviderAvailability { const diagnostics = ghidraInstallationDiagnostics(installation); - if ( - installation.status === "available" && - installation.platform === "win32" && - !hasWindowsNativeAuthority(installation.platform) - ) + if (installation.status === "available" && readiness?.available === false) return { status: "unavailable", - code: "unsupported_host", - reason: windowsNativeAuthorityUnavailableReason(installation.platform), + code: readiness.code, + reason: readiness.reason, diagnostics: { ...diagnostics, windows_security: jsonObjectSchema.parse( - windowsNativeCapabilities(installation.platform), + windowsNativeCapabilities(installation.platform, readiness), ), + windows_native_failure: { + stage: readiness.failureStage, + remediation: readiness.remediation, + }, }, }; return installation.status === "available" @@ -217,6 +241,22 @@ export class GhidraProvider implements AnalysisProviderCandidate { try { catalog = await readGhidraLanguageCatalog(installation.installDir); } catch (cause: unknown) { + if (cause instanceof GhidraLanguageCatalogCleanupError) + return err( + new ProviderCleanupError( + "ghidra", + cause.paths, + { + reason: cause.message, + paths: [...cause.paths], + close_failures: cause.causes.map(providerFailureText), + ...(cause.readFailure === undefined + ? {} + : { read_failure: providerFailureText(cause.readFailure) }), + }, + { operation: "resolve_analysis_profile", cause }, + ), + ); return err( new ProviderAdapterError("ghidra", "resolve_analysis_profile", { cause, @@ -302,6 +342,9 @@ export class GhidraProvider implements AnalysisProviderCandidate { } } +const providerFailureText = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); + const inspectWindowsP0TargetSupport = ( target: BinaryTarget, diagnostics: Readonly>, diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index f4bff8d9c9..6272802d30 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -2,10 +2,9 @@ import type { CapabilityDescriptor, ProviderIdentity, } from "../application/AnalysisProvider.js"; -import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; import { - hasWindowsNativeAuthority, - windowsNativeAuthorityUnavailableReason, + inspectWindowsNativeReadiness, + type WindowsNativeReadiness, } from "../process/WindowsAuthority.js"; import { GHIDRA_FUNCTION_OPERATIONS } from "./GhidraFunctionValues.js"; import { GHIDRA_INVENTORY_OPERATIONS } from "./GhidraInventoryValues.js"; @@ -44,22 +43,6 @@ const stringInventoryLimitation = const searchLimitation = "Regex mode uses Java Pattern semantics. Matcher or compiler stack exhaustion returns a resource constraint without closing the session; literal mode avoids regex recursion and returns complete matching values."; -const resolvedCallReferenceLimitation = - "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals."; - -/** Relationship-only caveats belong to each observed caller/callee result. */ -export const relationshipCoverageFor = ( - operation: string, -): RelationshipCoverage | undefined => - operation === "procedure_callers" || - operation === "procedure_callees" || - operation === "analyze_function" - ? { - status: "complete", - diagnostics: [resolvedCallReferenceLimitation], - } - : undefined; - /** Limitation text for one admitted operation, including the common base. */ export const limitationsFor = (operation: string): readonly string[] => { const common = [ @@ -214,8 +197,9 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( ); /** Capabilities advertised for the experimental Windows x64 P0 boundary. */ -export const windowsP0Capabilities = (): readonly CapabilityDescriptor[] => { - const available = hasWindowsNativeAuthority("win32"); +export const windowsP0Capabilities = ( + readiness: WindowsNativeReadiness = inspectWindowsNativeReadiness("win32"), +): readonly CapabilityDescriptor[] => { return Object.freeze( CAPABILITIES.map((capability): CapabilityDescriptor => { const limitations = Object.freeze([ @@ -230,13 +214,13 @@ export const windowsP0Capabilities = (): readonly CapabilityDescriptor[] => { availabilityCode: "unsupported_host", limitations, }); - return available + return readiness.available ? Object.freeze({ ...capability, limitations }) : Object.freeze({ ...capability, available: false, - reason: windowsNativeAuthorityUnavailableReason("win32"), - availabilityCode: "unsupported_host", + reason: readiness.reason, + availabilityCode: readiness.code, limitations, }); }), diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 9db1b0c5f6..80c37c0be2 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -34,6 +34,11 @@ import { } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { EvidenceLocation } from "../domain/evidence.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; +import { + providerRetryAction, + type ProviderOperationHealth, +} from "../domain/providerOperationHealth.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -86,7 +91,6 @@ import { healthLimitations, windowsP0Limitations, limitationsFor, - relationshipCoverageFor, } from "./GhidraProviderCapabilities.js"; import type { GhidraSessionError } from "./GhidraSessionError.js"; import { @@ -99,10 +103,7 @@ import { validateGhidraExtensionProfile, ghidraExtensionLimitations, } from "./extensions/GhidraExtensions.js"; -import { - windowsNativeAuthorityUnavailableReason, - hasWindowsNativeAuthority, -} from "../process/WindowsAuthority.js"; +import { inspectWindowsNativeReadiness } from "../process/WindowsAuthority.js"; import { ghidraRawBinaryLanguage } from "./GhidraRawBinaryLanguage.js"; /** Production seam for exercising provider projection without a real process. */ @@ -652,6 +653,11 @@ export const createGhidraProviderClient = (input: { ...(context === undefined ? {} : { runId: context.runId }), logger: logger.child({ layer: "ghidra-bridge" }), }); + // Startup failures retain installation guidance; only a previously running + // provider can be described as lost and reopened. + let providerStarted = false; + const lostProviderHealth = (): ProviderOperationHealth | undefined => + providerStarted ? client.operationHealthSnapshot?.() : undefined; const checkSessionObservations = async ( operation: AnalysisOperation, info: GhidraSessionInfo, @@ -733,8 +739,8 @@ export const createGhidraProviderClient = (input: { rawResult: JsonValue, observationLimitations: readonly string[] = [], locations: readonly EvidenceLocation[] = [], + relationshipCoverage?: RelationshipCoverage, ) => { - const relationshipCoverage = relationshipCoverageFor(operation); return createAnalysisExecution(normalized, committedProfile.provider, { rawResult, locations, @@ -757,6 +763,7 @@ export const createGhidraProviderClient = (input: { operation, called.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const parsed = parseGhidraInventoryResult(operation, called.value); @@ -780,12 +787,14 @@ export const createGhidraProviderClient = (input: { ); if (operation === "health") { const started = await client.start(options?.signal, options?.progress); + if (started.ok) providerStarted = true; if (!started.ok) return err( projectSessionError( operation, started.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const failed = await checkSessionObservations(operation, started.value); @@ -808,12 +817,14 @@ export const createGhidraProviderClient = (input: { softFloat ) { const started = await client.start(options?.signal, options?.progress); + if (started.ok) providerStarted = true; if (!started.ok) return err( projectSessionError( operation, started.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const failed = await checkSessionObservations(operation, started.value); @@ -825,6 +836,7 @@ export const createGhidraProviderClient = (input: { ? {} : { progress: options.progress }), }); + if (called.ok) providerStarted = true; if (operation === "list_documents" || operation === "list_procedures") return inventoryExecution(operation, called); if (!called.ok) @@ -833,17 +845,20 @@ export const createGhidraProviderClient = (input: { operation, called.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); let normalized: JsonValue; let observationLimitations: readonly string[] = []; let observationLocations: readonly EvidenceLocation[] = []; + let relationshipCoverage: RelationshipCoverage | undefined; if (isGhidraFunctionOperation(operation)) { const result = parseGhidraFunctionResult(operation, called.value); if (!result.ok) return result; normalized = result.value.value; observationLimitations = result.value.limitations; observationLocations = result.value.locations ?? []; + relationshipCoverage = result.value.relationshipCoverage; } else { const result = parseGhidraInventoryResult(operation, called.value); if (!result.ok) return result; @@ -863,6 +878,7 @@ export const createGhidraProviderClient = (input: { operation, started.error, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), ); const derived = await nativeAotOwner.derive( @@ -925,6 +941,7 @@ export const createGhidraProviderClient = (input: { operation, failure, config.ghidraStartupTimeoutMs, + lostProviderHealth(), ), nativeAotResult: derived, signal: options?.signal, @@ -939,6 +956,7 @@ export const createGhidraProviderClient = (input: { called.value, observationLimitations, observationLocations, + relationshipCoverage, ), ); }, @@ -997,15 +1015,16 @@ const ghidraClientPrerequisites = ( diagnostics: ghidraInstallationDiagnostics(installation), }), ); - if ( - installation.platform === "win32" && - !hasWindowsNativeAuthority(installation.platform) - ) + const readiness = + installation.platform === "win32" + ? inspectWindowsNativeReadiness(installation.platform) + : undefined; + if (readiness?.available === false) return err( new AnalysisCapabilityUnavailableError( "ghidra", "health", - windowsNativeAuthorityUnavailableReason(installation.platform), + readiness.reason, ), ); if (profile === undefined || profile.provider.id !== "ghidra") @@ -1027,11 +1046,13 @@ const projectSessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, startupTimeoutMs: number, + health?: ProviderOperationHealth, ): AnalysisError => { const primary = projectPrimarySessionError( operation, failure, startupTimeoutMs, + health, ); return failure.cleanupFailure === undefined ? primary @@ -1046,6 +1067,7 @@ const projectPrimarySessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, startupTimeoutMs: number, + health?: ProviderOperationHealth, ): AnalysisError => { const context = { cause: failure, diagnostics: failure.diagnostics }; if (failure.cause instanceof AnalysisAccessDeniedError) @@ -1112,5 +1134,31 @@ const projectPrimarySessionError = ( failure.message, context, ); + const exited = exitedProviderMessage(operation, failure, health); + if (exited !== undefined) + return new ProviderAdapterError("ghidra", operation, { + ...context, + userMessage: exited, + }); return new ProviderAdapterError("ghidra", operation, context); }; + +/** A provider already observed as running can be restarted by reopening. */ +const exitedProviderMessage = ( + operation: AnalysisOperation, + failure: GhidraSessionError, + health: ProviderOperationHealth | undefined, +): string | undefined => { + if (failure.kind !== "process" && failure.kind !== "protocol") + return undefined; + if ( + health === undefined || + providerRetryAction(health.state) !== "restart_provider" + ) + return undefined; + const observed = + health.state === "exited" + ? `exited${health.exitCode === null ? "" : ` with code ${String(health.exitCode)}`}` + : "became unreachable"; + return `The Ghidra provider for this session ${observed} before ${operation} could complete. Call open_binary for the same target to start a new Ghidra session, then retry; session-only annotations from the previous provider are not retained.`; +}; diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index ca3b4f6aca..dedf27bd42 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -2,6 +2,7 @@ import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; +import { GhidraLaunchSupportError } from "./GhidraJavaLaunch.js"; import { GhidraLaunchError, GHIDRA_LAUNCH_COMMAND_OUTPUT_BYTES, @@ -11,6 +12,7 @@ const MAX_CAUSE_DEPTH = 3; const MAX_CAUSE_MESSAGE_BYTES = 2 * 1024; const MAX_CAUSE_FIELD_BYTES = 1024; const MAX_COMPILER_STDERR_BYTES = 8 * 1024; +const MAX_COMMAND_OUTPUT_BYTES = 8 * 1024; /** Stable failure phase for one private Ghidra headless session. */ export type GhidraSessionFailureKind = @@ -59,7 +61,7 @@ const createGhidraSessionError = (failure: { readonly cause?: unknown; readonly timeoutMs?: number; readonly remoteCode?: string; - readonly redact: (value: string) => string; + readonly redact: (value: string, incomplete?: boolean) => string; }): GhidraSessionError => new GhidraSessionError( failure.kind, @@ -90,7 +92,7 @@ const createGhidraSessionError = (failure: { export const bindGhidraSessionFailure = ( diagnostics: () => Readonly>, - redact: (value: string) => string = (value) => value, + redact: (value: string, incomplete?: boolean) => string = (value) => value, ) => ( kind: GhidraSessionFailureKind, @@ -114,7 +116,7 @@ export const bindGhidraSessionFailure = const describeCause = ( cause: unknown, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, depth = 0, ): JsonValue => { if (cause instanceof Error) { @@ -137,7 +139,7 @@ const describeCause = ( const describeErrorCause = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, depth: number, ): JsonValue => { const message = boundedUtf8Text( @@ -152,15 +154,45 @@ const describeErrorCause = ( ...(message.truncated ? { message_truncated: true } : {}), ...causeCode(cause, redact), ...causeSignal(cause, redact), + ...causeCommandOutcome(cause, redact), ...causeCompilerOutcome(cause, redact), ...causeLaunchOutcome(cause, redact), ...nestedErrorCause(cause, redact, depth), }; }; +const causeCommandOutcome = ( + cause: Error, + redact: (value: string, incomplete?: boolean) => string, +): Record => { + if (!(cause instanceof GhidraLaunchSupportError)) return {}; + const outcome = cause.outcome; + if (outcome === undefined) return {}; + const stdout = boundedUtf8Text( + redact(outcome.stdout, outcome.outputTruncated !== false), + MAX_COMMAND_OUTPUT_BYTES, + ); + const stderr = boundedUtf8Text( + redact(outcome.stderr, outcome.outputTruncated !== false), + MAX_COMMAND_OUTPUT_BYTES, + ); + return { + command_outcome: { + stdout: stdout.text, + stderr: stderr.text, + stdoutBytes: outcome.stdoutBytes ?? Buffer.byteLength(outcome.stdout), + stderrBytes: outcome.stderrBytes ?? Buffer.byteLength(outcome.stderr), + stdoutTruncated: stdout.truncated, + stderrTruncated: stderr.truncated, + outputTruncated: outcome.outputTruncated, + killed: outcome.killed, + }, + }; +}; + const causeCode = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { const code = "code" in cause ? cause.code : undefined; if (typeof code === "string") { @@ -176,7 +208,7 @@ const causeCode = ( const causeSignal = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { const signal = "signal" in cause ? cause.signal : undefined; if (typeof signal === "string") { @@ -192,7 +224,7 @@ const causeSignal = ( const causeCompilerOutcome = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { if (!(cause instanceof DarwinProcessOwnershipInspectionError)) return {}; const outcome = cause.compilerOutcome; @@ -218,7 +250,7 @@ const causeCompilerOutcome = ( const boundedCompilerField = ( value: string | number | null, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): { readonly value: string | number | null; readonly truncated: boolean } => { if (typeof value !== "string") return { value, truncated: false }; const bounded = boundedUtf8Text(redact(value), MAX_CAUSE_FIELD_BYTES); @@ -228,9 +260,10 @@ const boundedCompilerField = ( /** Retain a launch failure's launcher command outcome in its cause envelope. */ const causeLaunchOutcome = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, ): Record => { if (!(cause instanceof GhidraLaunchError)) return {}; + if (cause.cause instanceof GhidraLaunchSupportError) return {}; const launchSupport = cause.launchSupport.length === 0 ? {} @@ -273,7 +306,7 @@ const causeLaunchOutcome = ( const nestedErrorCause = ( cause: Error, - redact: (value: string) => string, + redact: (value: string, incomplete?: boolean) => string, depth: number, ): Record => { if (!("cause" in cause)) return {}; diff --git a/src/ghidra/GhidraSnapshotCache.test.ts b/src/ghidra/GhidraSnapshotCache.test.ts index 83d293aa21..fa7b35c224 100644 --- a/src/ghidra/GhidraSnapshotCache.test.ts +++ b/src/ghidra/GhidraSnapshotCache.test.ts @@ -11,6 +11,7 @@ import { silentLogger } from "../logger.js"; import { GhidraProvider } from "./GhidraProvider.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; import { GHIDRA_SESSION_CAPABILITIES } from "./GhidraSessionValues.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; const TARGET: BinaryTarget = { path: "/tmp/ghidra-cache-fixture", @@ -23,7 +24,7 @@ const TARGET: BinaryTarget = { // Exercise the real provider descriptors, profile, input/output projection and // session cache; only the installed Ghidra process is replaced at its seam. -const fixture = (version = "12.1.4") => { +const fixture = (version = "12.1.4", coverage?: RelationshipCoverage) => { const config = parseConfig({ GHIDRA_INSTALL_DIR: "/opt/ghidra" }); if (!config.ok) throw config.error; const calls: string[] = []; @@ -113,7 +114,13 @@ const fixture = (version = "12.1.4") => { return ok( operation === "procedure_pseudo_code" || operation === "analyze_function" - ? { value, limitations: [] } + ? { + value, + limitations: [], + ...(operation === "analyze_function" && coverage !== undefined + ? { relationship_coverage: coverage } + : {}), + } : value, ); }, @@ -133,42 +140,119 @@ const open = async (session: ReturnType["session"]) => { expect((await session.openResolvedTarget(preview.value)).ok).toBe(true); }; -describe("Ghidra snapshot cache", () => { - it("refuses snapshots whose profile predates typed incoming references", async () => { - const initial = fixture(); - await open(initial.session); - const snapshot = initial.session.exportAnalysisSnapshot(); - if (!snapshot.ok) throw snapshot.error; - await initial.session.close(); - const profile = snapshot.value.binding.analysis_profile; - expect(profile.parameters.instruction_references).toBe( - "exact-address-incoming-v1", - ); - const legacy = { - ...snapshot.value, - binding: { - ...snapshot.value.binding, - analysis_profile: createAnalysisProfile( - profile.provider, - Object.fromEntries( - Object.entries(profile.parameters).filter( - ([key]) => key !== "instruction_references", - ), - ), - ), - }, - }; - const replay = fixture(); - const imported = replay.session.importAnalysisSnapshot(legacy); - if (!imported.ok) throw imported.error; - expect(await replay.session.previewTarget(TARGET)).toMatchObject({ - ok: false, - error: { _tag: "EvidenceIntegrityError" }, - }); - expect(replay.calls).toEqual([]); - await replay.session.close(); - }); +describe("Ghidra relationship coverage replay", () => { + it.each([ + undefined, + { status: "complete", diagnostics: ["Resolved static references only."] }, + { status: "partial", limitations: ["Instruction coverage was truncated."] }, + { status: "unknown", reason: "Relationship inventory was unavailable." }, + { + status: "partial", + limitations: ["Missing callers."], + limited_directions: ["callers"], + }, + ] satisfies (RelationshipCoverage | undefined)[])( + "preserves relationship coverage %j through live execution, cache and snapshot replay", + async (coverage) => { + const initial = fixture("12.1.4", coverage); + await open(initial.session); + const parameters = { procedure: "0x1000" }; + for (let i = 0; i < 2; i++) { + const execution = await initial.session.execute( + "analyze_function", + parameters, + ); + if (!execution.ok) throw execution.error; + expect(execution.value.relationshipCoverage).toEqual(coverage); + } + expect(initial.calls).toEqual(["analyze_function"]); + const snapshot = initial.session.exportAnalysisSnapshot(); + if (!snapshot.ok) throw snapshot.error; + await initial.session.close(); + const replay = fixture(); + expect(replay.session.importAnalysisSnapshot(snapshot.value).ok).toBe( + true, + ); + await open(replay.session); + const execution = await replay.session.execute( + "analyze_function", + parameters, + ); + if (!execution.ok) throw execution.error; + expect(execution.value.relationshipCoverage ?? undefined).toEqual( + coverage, + ); + expect(replay.calls).toEqual([]); + await replay.session.close(); + }, + ); +}); +describe("Ghidra snapshot profile admission", () => { + it.each([ + { + parameter: "instruction_references", + current: "exact-address-incoming-v1", + legacy: undefined, + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "resolved-call-reference-diagnostics-v1", + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "decoded-call-target-coverage-v2", + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "decoded-call-target-coverage-v3", + }, + { + parameter: "call_relationship_coverage", + current: "function-body-call-coverage-v5", + legacy: "function-body-call-coverage-v4", + }, + ])( + "refuses snapshots with outdated $parameter semantics", + async ({ parameter, current, legacy: prior }) => { + const initial = fixture(); + await open(initial.session); + const snapshot = initial.session.exportAnalysisSnapshot(); + if (!snapshot.ok) throw snapshot.error; + await initial.session.close(); + const profile = snapshot.value.binding.analysis_profile; + expect(profile.parameters[parameter]).toBe(current); + const legacy = { + ...snapshot.value, + binding: { + ...snapshot.value.binding, + analysis_profile: createAnalysisProfile(profile.provider, { + ...Object.fromEntries( + Object.entries(profile.parameters).filter( + ([key]) => key !== parameter, + ), + ), + ...(prior === undefined ? {} : { [parameter]: prior }), + }), + }, + }; + const replay = fixture(); + const imported = replay.session.importAnalysisSnapshot(legacy); + if (!imported.ok) throw imported.error; + expect(await replay.session.previewTarget(TARGET)).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); + expect(replay.calls).toEqual([]); + await replay.session.close(); + }, + ); +}); + +describe("Ghidra snapshot cache", () => { it.each([ { operation: "analyze_function", parameters: { procedure: "0x1000" } }, { diff --git a/src/go/GoBinaryProvider.ts b/src/go/GoBinaryProvider.ts index 466136f3f6..9e2ac1f301 100644 --- a/src/go/GoBinaryProvider.ts +++ b/src/go/GoBinaryProvider.ts @@ -55,7 +55,7 @@ const failureForPrimary = ( if (cause instanceof GoBinaryResourceFailure) return new AnalysisResourceConstraintError( OPERATION, - "memory", + cause.boundary === "search" ? "cpu" : "memory", `Selected artifact ${path}: ${cause.message}`, { boundary: cause.boundary, maximum_bytes: cause.maximum_bytes }, { cause }, @@ -194,7 +194,7 @@ export class GoBinaryProvider implements GoBinaryPort { limitations: [ "Embedded build metadata is observed file content, not proof of compiler identity, dependency authenticity or runtime behavior.", "Only Go build information is inspected; function names, source mappings, type metadata and decompiled code are not recovered.", - "Resource guards bound stable artifact reads to 256 MiB, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.", + "Resource guards bound stable artifact reads and cumulative header searches to 256 MiB each, aggregate embedded strings to 1 MiB, structural table decoding to 16 MiB and complete metadata payloads to 8 MiB; these are not operating-system memory or CPU limits.", ...(info !== null && (info.go_version === null || info.module_text === null) ? [ diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index e4d7c83814..e446e40448 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -114,6 +114,14 @@ export class HopperProvider implements AnalysisProviderCandidate { reason: `Hopper cannot directly analyze ${target.kind} targets.`, diagnostics, }; + if (target.kind === "database" && this.platform === "darwin") + return { + status: "unsupported", + code: "target_kind_unsupported", + reason: + "This Hopper macOS launcher cannot start REA's analysis bridge when opening a saved Hopper database. Open an executable target instead; saved-database analysis is unavailable until Hopper exposes a database launch route that runs the bridge script.", + diagnostics, + }; if (target.kind === "database") return { status: "supported", diff --git a/src/hopper/HopperProviderDiscovery.test.ts b/src/hopper/HopperProviderDiscovery.test.ts index 82acf8888a..5df7eacad0 100644 --- a/src/hopper/HopperProviderDiscovery.test.ts +++ b/src/hopper/HopperProviderDiscovery.test.ts @@ -46,7 +46,7 @@ describe("Hopper provider discovery", () => { }); it("separates target kind, format, and architecture support", () => { - const hopper = provider(process.execPath); + const hopper = provider(process.execPath, "linux"); expect(hopper.inspectTargetSupport(databaseTarget())).toMatchObject({ status: "supported", }); @@ -59,12 +59,31 @@ describe("Hopper provider discovery", () => { diagnostics: { target_kind: "archive", target_format: "asar" }, }); }); + + it("fails closed for saved Hopper databases on macOS without a bridge launch route", () => { + expect( + provider(process.execPath, "darwin").inspectTargetSupport( + databaseTarget(), + ), + ).toMatchObject({ + status: "unsupported", + code: "target_kind_unsupported", + reason: expect.stringContaining("cannot start REA's analysis bridge"), + diagnostics: { + target_kind: "database", + target_format: "analysis-database", + }, + }); + }); }); -const provider = (launcherPath: string): HopperProvider => { +const provider = ( + launcherPath: string, + platform: NodeJS.Platform = process.platform, +): HopperProvider => { const config = parseConfig({ HOPPER_LAUNCHER_PATH: launcherPath }); if (!config.ok) throw config.error; - return new HopperProvider(config.value, silentLogger, {}); + return new HopperProvider(config.value, silentLogger, {}, platform); }; const databaseTarget = (): BinaryTarget => ({ diff --git a/src/ida/IdaContainment.test.ts b/src/ida/IdaContainment.test.ts index 069388df82..f81d7b9cdf 100644 --- a/src/ida/IdaContainment.test.ts +++ b/src/ida/IdaContainment.test.ts @@ -148,7 +148,7 @@ const found = (query: string, fn: typeof function_) => ({ }); describe("IDA containment capabilities", () => { - it("declares both operations for the headless profile only", () => { + it("declares document identity in both profiles and address resolution only in headless", () => { const declared = (mode: "attached" | "headless", native = false) => idaCapabilities(mode, false, native) .map(({ operation }) => operation) @@ -172,35 +172,30 @@ describe("IDA containment capabilities", () => { "resolve_containing_procedure", "list_documents", ]); - expect(declared("attached")).toEqual([]); - expect(declared("attached", true)).toEqual([]); + expect(declared("attached")).toEqual(["list_documents"]); + expect(declared("attached", true)).toEqual(["list_documents"]); expect( idaCapabilities("attached", false).map(({ operation }) => operation), - "attached profiles do not even declare them", - ).not.toContain("list_documents"); + "attached profiles declare the bound document but not address resolution", + ).toContain("list_documents"); }); - it.each(["resolve_containing_procedure", "list_documents"] as const)( - "refuses %s on the legacy attached profile without calling upstream", - async (operation) => { - const { client, producer } = await session({}, "attached"); - try { - const result = await client.execute( - operation, - operation === "list_documents" ? {} : { address: "0x1000" }, - ); - - expect(result.ok).toBe(false); - if (!result.ok) - expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); - expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( - false, - ); - } finally { - expect((await client.close()).ok).toBe(true); - } - }, - ); + it("refuses address resolution on the legacy attached profile", async () => { + const { client, producer } = await session({}, "attached"); + try { + const result = await client.execute("resolve_containing_procedure", { + address: "0x1000", + }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( + false, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); }); describe("headless IDA containing procedure", () => { diff --git a/src/ida/IdaDiagnosticRedactor.test.ts b/src/ida/IdaDiagnosticRedactor.test.ts new file mode 100644 index 0000000000..0c65e37d31 --- /dev/null +++ b/src/ida/IdaDiagnosticRedactor.test.ts @@ -0,0 +1,563 @@ +import { describe, expect, it } from "vitest"; +import { + createIdaDiagnosticRedactor, + IDA_DIAGNOSTIC_SOURCE_BYTES, +} from "./IdaDiagnosticRedactor.js"; + +describe("IDA diagnostic credential redaction", () => { + it.each([false, true])( + "redacts literal credentials once regardless of header order (reversed: %s)", + (reversed) => { + const entries = [ + ["Authorization", "Bearer fixture"], + ["proxy-AUTHORIZATION", "bAsIc fixture-long.$[token]"], + ["cOoKiE", "session=fixture-long.$[token]; marker=redacted"], + ["X-Evidence", "caller-selected-evidence"], + ]; + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: Object.fromEntries(reversed ? entries.reverse() : entries), + mode: "attached", + timeoutMs: 1000, + }).text; + expect( + redact( + "Bearer fixture / fixture / bAsIc fixture-long.$[token] / fixture-long.$[token] / session=fixture-long.$[token]; marker=redacted /caller/input?token=visible API_KEY=visible caller-selected-evidence", + ), + ).toBe( + "␟ / ␟ / ␟ / ␟ / ␟ /caller/input?token=visible API_KEY=visible caller-selected-evidence", + ); + }, + ); + it("redacts configured and HTTP-normalized header values", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { + aUtHoRiZaTiOn: "\r\n \tBearer fixture-secret \t\r\n", + "pRoXy-AuThOrIzAtIoN": "\tBasic proxy-secret ", + CoOkIe: " session=cookie-secret\t", + }, + mode: "attached", + timeoutMs: 1000, + }).text; + expect( + redact( + "raw=\r\n \tBearer fixture-secret \t\r\n; wire=Bearer fixture-secret; token=fixture-secret; proxy=Basic proxy-secret; bare=proxy-secret; cookie=session=cookie-secret", + ), + ).toBe( + "raw=[redacted transport credential]; wire=[redacted transport credential]; token=[redacted transport credential]; proxy=[redacted transport credential]; bare=[redacted transport credential]; cookie=[redacted transport credential]", + ); + }); + it("does not treat empty HTTP authentication as diagnostic content", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { + Authorization: "\r\n \t \r\n", + "Proxy-Authorization": "", + Cookie: "\t", + }, + mode: "attached", + timeoutMs: 1000, + }).text; + const message = "Failed at /caller/input \t with ordinary whitespace"; + expect(redact(message)).toBe(message); + }); + it("does not reprocess the replacement marker as another credential", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Bearer alpha", Cookie: "redacted" }, + mode: "attached", + timeoutMs: 1000, + }).text; + expect(redact("alpha redacted")).toBe("␟ ␟"); + }); +}); + +const credentialCases: readonly { + readonly name: string; + readonly headers: Readonly>; + readonly message: string; + readonly expected: string; +}[] = [ + { + name: "duplicate cookie names, quoted values, and equals padding", + headers: { + Cookie: + 'session=first-secret; session=second-secret; signed="padded=value=="', + }, + message: "first-secret / second-secret / padded=value== / session signed", + expected: + "[redacted transport credential] / [redacted transport credential] / [redacted transport credential] / session signed", + }, + { + name: "empty cookie and auth-param values", + headers: { Cookie: 'a=; b=""', Authorization: 'Custom empty=""' }, + message: 'Ordinary spaces and "" remain; a b empty are names', + expected: 'Ordinary spaces and "" remain; a b empty are names', + }, + { + name: "opaque token68 and explicit one-character credentials", + headers: { Authorization: "Negotiate opaque+/==", Cookie: "a=Q" }, + message: "opaque+/== Q Negotiate a", + expected: + "[redacted transport credential] [redacted transport credential] Negotiate a", + }, + { + name: "auth-param values retain quoted commas and decode quoted pairs", + headers: { + Authorization: String.raw`Digest first="nonce,part", response="quote\"part", opaque="back\\part", tail=last-secret`, + }, + message: String.raw`nonce,part / quote"part / quote\"part / back\part / back\\part / last-secret / Digest first response opaque tail`, + expected: + "[redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / Digest first response opaque tail", + }, + { + name: "complete parameters before a malformed quoted member", + headers: { + Authorization: + 'Digest first=known-secret, broken="unterminated, second=visible', + }, + message: "known-secret visible broken second", + expected: "[redacted transport credential] visible broken second", + }, + { + name: "unclosed quotes do not create parameters from their contents", + headers: { Authorization: 'Digest broken="unterminated, second=visible' }, + message: "unterminated visible broken second", + expected: "unterminated visible broken second", + }, + { + name: "malformed cookie members do not hide valid later pairs", + headers: { + Cookie: 'broken; =not-a-value; malformed="unclosed; session=valid-secret', + }, + message: "valid-secret broken not-a-value unclosed session", + expected: + "[redacted transport credential] broken not-a-value unclosed session", + }, + { + name: "cookie octets remain opaque without URL decoding", + headers: { Cookie: "session=encoded%2Fcredential" }, + message: "encoded%2Fcredential / encoded/credential", + expected: "[redacted transport credential] / encoded/credential", + }, +]; +it.each(credentialCases)( + "parses $name at the configured credential boundary", + ({ headers, message, expected }) => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers, + mode: "attached", + timeoutMs: 1000, + }).text; + expect(redact(message)).toBe(expected); + }, +); + +it("redacts full configured malformed headers without guessing their internal values", () => { + const header = 'Digest broken="unterminated, second=visible'; + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: header }, + mode: "attached", + timeoutMs: 1000, + }); + expect(redactor.text(header)).toBe("[redacted transport credential]"); + expect(redactor.json(JSON.stringify(header))).toBe( + '"[redacted transport credential]"', + ); +}); + +const marker = "[redacted transport credential]"; +const redactorFor = (header: string, name = "Authorization") => + createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { [name]: header }, + mode: "attached", + timeoutMs: 1000, + }); + +describe("IDA Basic authentication payload components", () => { + it.each(["aUtHoRiZaTiOn", "pRoXy-AuThOrIzAtIoN"])( + "protects decoded Basic pairs and passwords while preserving user names for %s", + (name) => { + const pair = "fixture-user:fixture-password:tail"; + const encoded = Buffer.from(pair, "utf8").toString("base64"); + const redactor = redactorFor("bAsIc " + encoded, name); + const reason = + "pair=" + + pair + + "; user=fixture-user; password=fixture-password:tail; wire=" + + encoded; + const safe = + "pair=" + + marker + + "; user=fixture-user; password=" + + marker + + "; wire=" + + marker; + expect(redactor.text(reason + "; path=/caller/input?token=visible")).toBe( + safe + "; path=/caller/input?token=visible", + ); + expect( + redactor.json(JSON.stringify({ reason, path: "/caller/input" })), + ).toBe(JSON.stringify({ reason: safe, path: "/caller/input" })); + }, + ); + + it.each([ + [ + ":fixture-password", + "pair=:fixture-password; password=fixture-password; empty values leave spaces", + "pair=" + marker + "; password=" + marker + "; empty values leave spaces", + ], + [ + "fixture-user:", + "pair=fixture-user:; user=fixture-user; empty values leave spaces", + "pair=" + marker + "; user=fixture-user; empty values leave spaces", + ], + [ + ":", + "pair=:; empty values leave spaces", + "pair=" + marker + "; empty values leave spaces", + ], + ])( + "protects a decoded pair with empty components: %s", + (pair, reason, safe) => { + const encoded = Buffer.from(pair, "utf8").toString("base64"); + expect(redactorFor("Basic " + encoded).text(reason)).toBe(safe); + }, + ); +}); + +describe("IDA Basic authentication charset and invalid data", () => { + it("retains both strict UTF-8 and exact octet-compatible interpretations when charset is unknown", () => { + const pair = "tést-user:päss£word"; + const bytes = Buffer.from(pair, "utf8"); + const octets = bytes.toString("latin1"); + const redactor = redactorFor("Basic " + bytes.toString("base64")); + const reason = + "utf8=" + + pair + + "; octets=" + + octets + + "; user=tést-user; password=päss£word; octet-user=tést-user; octet-password=päss£word; caller=/caller/input"; + expect(redactor.text(reason)).toBe( + "utf8=" + + marker + + "; octets=" + + marker + + "; user=tést-user; password=" + + marker + + "; octet-user=tést-user; octet-password=" + + marker + + "; caller=/caller/input", + ); + }); + + it("preserves exact Latin-1 octets without inventing lossy UTF-8 replacement credentials", () => { + const bytes = Buffer.from("latin-user:pässword", "latin1"); + const redactor = redactorFor("Basic " + bytes.toString("base64")); + expect( + redactor.text( + "latin-user:pässword / latin-user / pässword / p\ufffdssword /caller/input", + ), + ).toBe( + marker + " / latin-user / " + marker + " / p\ufffdssword /caller/input", + ); + }); + + it("preserves a UTF-8 BOM as credential data rather than dropping it", () => { + const pair = "\ufefffixture-user:fixture-password"; + const encoded = Buffer.from(pair, "utf8").toString("base64"); + expect( + redactorFor("Basic " + encoded).text(pair + " / \ufefffixture-user"), + ).toBe(marker + " / \ufefffixture-user"); + }); + + it("keeps decoded Basic values opaque without URL decoding", () => { + const encoded = Buffer.from("fixture-user:p%2Fsecret", "utf8").toString( + "base64", + ); + expect(redactorFor("Basic " + encoded).text("p%2Fsecret / p/secret")).toBe( + marker + " / p/secret", + ); + }); + + const canonical = Buffer.from( + "fixture-user:fixture-password", + "utf8", + ).toString("base64"); + const alphabet = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + const nonzeroPadBits = canonical.replace( + /(.)(=)$/u, + (_match, symbol: string, padding: string) => + (alphabet[alphabet.indexOf(symbol) + 1] ?? "!") + padding, + ); + it.each([ + { name: "unpadded", payload: canonical.slice(0, -1) }, + { name: "at most two padding characters", payload: canonical + "=" }, + ])( + "accepts the supported normalized Basic Base64 spelling: $name", + ({ payload }) => { + const redactor = redactorFor("Basic " + payload); + expect( + redactor.text( + "fixture-user:fixture-password / fixture-password / fixture-user", + ), + ).toBe(marker + " / " + marker + " / fixture-user"); + }, + ); + + it.each([ + { + name: "non-alphabet character", + payload: canonical.slice(0, 4) + "!" + canonical.slice(4), + }, + { + name: "embedded whitespace", + payload: canonical.slice(0, 4) + " " + canonical.slice(4), + }, + { name: "more than two pad characters", payload: canonical + "==" }, + { name: "nonzero pad bits", payload: nonzeroPadBits }, + { + name: "missing colon separator", + payload: Buffer.from("fixture-user", "utf8").toString("base64"), + }, + { + name: "ASCII control in user-pass", + payload: Buffer.from("fixture-user:fixture-password\n", "utf8").toString( + "base64", + ), + }, + { + name: "auth-param-shaped malformed Basic data", + payload: "credential=fixture-password", + }, + ])("retains only opaque configured Basic data for $name", ({ payload }) => { + const header = "Basic " + payload; + const redactor = redactorFor(header); + expect(redactor.text(header + " / " + payload)).toBe( + marker + " / " + marker, + ); + expect(redactor.text("fixture-user / fixture-password /caller/input")).toBe( + "fixture-user / fixture-password /caller/input", + ); + }); + + it("does not decode another scheme's base64-shaped opaque payload", () => { + const encoded = Buffer.from( + "fixture-user:fixture-password", + "utf8", + ).toString("base64"); + const redactor = redactorFor("Custom " + encoded); + expect(redactor.text(encoded + " / fixture-user / fixture-password")).toBe( + marker + " / fixture-user / fixture-password", + ); + }); +}); + +it("redacts protocol-decoded Basic credentials and password without treating ordinary names as secrets", () => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { + Authorization: `Basic ${Buffer.from("user:secret").toString("base64")}`, + }, + }); + expect(redactor.text("user:secret / secret / user")).toBe( + "[redacted transport credential] / [redacted transport credential] / user", + ); + expect(redactor.json('"\\u0073ecret"')).toBe( + '"[redacted transport credential]"', + ); +}); + +it.each(["text", "json"] as const)( + "bounds short-secret diagnostic expansion before %s redaction", + (format) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Cookie: "a=Q" }, + }); + const result = redactor[format]("Q".repeat(2 * 1024 * 1024)); + expect(result).toBe("[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]"); + }, +); + +it.each(["text", "json"] as const)( + "applies the UTF-8 source-byte budget before %s processing even without credentials", + (format) => { + const atBudget = "£".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES / 2); + const overBudget = atBudget + "£"; + const configuredHeaders: readonly Readonly>[] = [ + {}, + { Cookie: "a=Q" }, + ]; + for (const headers of configuredHeaders) { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers, + }); + expect(redactor[format](atBudget)).toBe(atBudget); + expect(redactor[format](overBudget)).toBe( + "[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]", + ); + } + }, +); + +it.each(["dXNlcjpzZWNyZXQ=", "dXNlcjpzZWNyZXQ", "dXNlcjpzZWNyZXQ=="])( + "redacts accepted canonical Basic spellings for %s", + (payload) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Authorization: `Basic ${payload}` }, + }); + const marker = "[redacted transport credential]"; + expect( + redactor.text( + "dXNlcjpzZWNyZXQ= / dXNlcjpzZWNyZXQ / user:secret / secret", + ), + ).toBe(`${marker} / ${marker} / ${marker} / ${marker}`); + }, +); + +describe("overlapping IDA credential source spans", () => { + it.each([false, true])( + "unions all overlapping plain credentials before writing (reversed: %s)", + (reversed) => { + const entries = [ + ["Authorization", "Custom X]"], + ["Cookie", "a=]SECRET"], + ]; + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: Object.fromEntries(reversed ? entries.reverse() : entries), + mode: "attached", + timeoutMs: 1000, + }); + const result = redactor.text( + "Rejected X]SECRET at /caller/input?cursor=visible", + ); + expect(result).toBe("Rejected ␟ at /caller/input?cursor=visible"); + expect(result).not.toContain("]SECRET"); + }, + ); + + it("unions a chain of different overlapping credentials", () => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Custom abc", Cookie: "a=cde; b=def" }, + mode: "attached", + timeoutMs: 1000, + }); + expect(redactor.text("prefix abcdefg suffix")).toBe( + "prefix " + marker + "g suffix", + ); + }); + + it("merges dense same-credential overlaps without retaining a per-match collection", () => { + const redactor = redactorFor("Custom AA"); + const result = redactor.text("A".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES)); + expect(result.length).toBe(marker.length); + expect(result).toBe(marker); + }); + + it("retains adjacent disjoint credential occurrences", () => { + expect(redactorFor("Custom AA").text("AA-AA")).toBe(marker + "-" + marker); + }); +}); + +describe("credential-safe redaction markers", () => { + it.each([ + { secret: "]SECRET", source: "XSECRET", expected: "␟SECRET" }, + { secret: "SECRET[", source: "SECRETX", expected: "SECRET␟" }, + { + secret: "a[redacted transport credential]b", + source: "aXb", + expected: "a␟b", + }, + { secret: "redacted", source: "redacted", expected: "␟" }, + ])( + "does not create $secret from replacement text or boundaries", + ({ secret, source, expected }) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Authorization: "Custom " + secret, Cookie: "a=X" }, + }); + expect(redactor.text(source)).toBe(expected); + expect( + redactor.json( + JSON.stringify({ reason: source, context: "/caller/input" }), + ), + ).toBe(JSON.stringify({ reason: expected, context: "/caller/input" })); + expect(redactor.json('"' + source + '",x')).toBe('"' + expected + '",x'); + }, + ); + it("chooses a different graphic scalar when a decoded Basic credential contains the first marker", () => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { + Authorization: + "Basic " + Buffer.from("user:␟", "utf8").toString("base64"), + Cookie: "a=]SECRET", + }, + }); + expect(redactor.text("␟ at /caller/input")).toBe("␠ at /caller/input"); + expect(redactor.json(String.raw`{"reason":"␟","context":"kept"}`)).toBe( + '{"reason":"␠","context":"kept"}', + ); + }); +}); + +describe("unclassified diagnostic blocks and message ownership", () => { + it.each>>([{}, { Cookie: "a=Q" }])( + "budgets the entire source block list before redaction: %s", + (headers) => { + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers, + }); + const first = "£".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES / 4); + expect(redactor.blocks([first, first])).toBe( + "[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]", + ); + expect(redactor.blocks([first, first.slice(1)])).toBe( + first + "\n" + first.slice(1), + ); + }, + ); + it("keeps complete quoted plain credentials while preserving explicit JSON semantics", () => { + const redactor = redactorFor(String.raw`Custom folder\tmp`); + const source = String.raw`"folder\tmp at /caller/input"`; + expect(redactor.blocks([source])).toBe( + '"[redacted transport credential] at /caller/input"', + ); + expect(redactor.json(source)).toBe(source); + }); + it("sanitizes the omission notice as diagnostic text", () => { + const redactor = redactorFor("Custom omitted"); + expect( + redactor.blocks(["Q".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES + 1)]), + ).toContain("[redacted transport credential]"); + expect( + redactor.text("Q".repeat(IDA_DIAGNOSTIC_SOURCE_BYTES + 1)), + ).not.toContain("omitted"); + }); +}); diff --git a/src/ida/IdaDiagnosticRedactor.ts b/src/ida/IdaDiagnosticRedactor.ts new file mode 100644 index 0000000000..d332d15488 --- /dev/null +++ b/src/ida/IdaDiagnosticRedactor.ts @@ -0,0 +1,243 @@ +import { + redactIdaJsonDiagnostic, + redactIdaUnclassifiedDiagnostic, +} from "./IdaJsonDiagnostic.js"; +import { + IDA_REDACTION_MARKER, + joinIdaDiagnosticParts, + writeIdaDiagnostic, + type IdaDiagnosticMatch, +} from "./IdaDiagnosticWriter.js"; +import type { IdaConfiguration } from "./IdaConfiguration.js"; +import { AnalysisProtocolError } from "../domain/analysisErrorCore.js"; + +// Diagnostic text is bounded before decoding or replacement expansion. +export const IDA_DIAGNOSTIC_SOURCE_BYTES = 64 * 1024; +const omittedDiagnostic = "[IDA diagnostic omitted: exceeds 65536 UTF-8 bytes]"; + +const boundedDiagnostic = + (redact: (message: string) => string) => + (message: string): string => + Buffer.byteLength(message, "utf8") > IDA_DIAGNOSTIC_SOURCE_BYTES + ? redact(omittedDiagnostic) + : redact(message); + +/** RFC 7617 user-pass candidates; the configured request carries no charset. */ +const basicPayloads = (payload: string): readonly string[] => { + if (!/^[A-Za-z\d+/]+={0,2}$/u.test(payload)) return []; + const octets = Buffer.from(payload, "base64"); + const canonical = octets.toString("base64"); + // Match the supported padded/unpadded policy without accepting Buffer's + // permissive alphabet or discarding nonzero pad bits. + if ( + canonical.replace(/=+$/u, "") !== payload.replace(/=+$/u, "") || + octets.indexOf(0x3a) < 0 + ) + return []; + // Basic user-id and password exclude ASCII control characters. + if (octets.some((value) => value < 0x20 || value === 0x7f)) return []; + const candidates = [octets.toString("latin1")]; + try { + candidates.push( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(octets), + ); + } catch { + // Invalid UTF-8 has no decoded Unicode candidate. Exact Latin-1 preserves + // each octet; do not invent a replacement-character credential. + } + // The default charset is unknown. Retain both supported interpretations + // rather than selecting one or normalizing their configured source bytes. + return [ + canonical, + canonical.replace(/=+$/u, ""), + ...candidates.flatMap((pair) => { + const separator = pair.indexOf(":"); + // The complete pair and password are credentials; a bare user-id is not. + return [pair, pair.slice(separator + 1)].filter( + (value) => value.length > 0, + ); + }), + ]; +}; + +/** RFC 9110 credentials: any scheme, followed by opaque data or auth-params. */ +const authorizationPayloads = (header: string): readonly string[] => { + const credentials = header.match( + /^([!#$%&'*+.^_`|~\dA-Za-z-]+)[ \t]+([\s\S]+)$/u, + ); + const payload = credentials?.[2]; + if (payload === undefined) return []; + if (credentials?.[1]?.toLowerCase() === "basic") + return [payload, ...basicPayloads(payload)]; + const values = [payload]; + // A sticky parse cannot reinterpret commas inside an incomplete quoted value. + // Keep complete parameters preceding malformed input; the opaque payload above + // remains a known credential even when the parameter grammar is not satisfied. + const parameter = + /[ \t,]*[!#$%&'*+.^_`|~\dA-Za-z-]+[ \t]*=[ \t]*("(?:[^"\\\r\n]|\\[^\r\n])*"|[!#$%&'*+.^_`|~\dA-Za-z-]+)[ \t]*(?:,|$)/uy; + let cursor = 0; + while (cursor < payload.length) { + parameter.lastIndex = cursor; + const match = parameter.exec(payload); + if (match === null) break; + cursor = parameter.lastIndex; + const value = match[1] ?? ""; + if (!value.startsWith('"')) { + values.push(value); + continue; + } + const quoted = value.slice(1, -1); + if (quoted.length > 0) + values.push(value, quoted, quoted.replace(/\\([^\r\n])/gu, "$1")); + } + return values; +}; + +/** RFC 6265 cookie-pairs: retain duplicates and '=' inside values. */ +const cookiePayloads = (header: string): readonly string[] => + header.split(";").flatMap((pair) => { + const value = pair.match( + /^[ \t]*[!#$%&'*+.^_`|~\dA-Za-z-]+[ \t]*=[ \t]*(.*?)[ \t]*$/u, + )?.[1]; + if (value === undefined) return []; + const payload = + value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value; + // Cookie quotes wrap cookie-octets; they do not introduce escape syntax. + if (!/^[\x21\x23-\x2b\x2d-\x3a\x3c-\x5b\x5d-\x7e]+$/u.test(payload)) + return []; + return [value, payload]; + }); + +/** Choose one marker for this credential owner, including replacement boundaries. */ +const redactionMarker = (secrets: readonly string[]): string => { + // Crossing either edge of the default marker requires a square bracket. + // A credential wholly inside the marker must also be excluded. + if ( + secrets.every( + (secret) => + !secret.includes("[") && + !secret.includes("]") && + !IDA_REDACTION_MARKER.includes(secret), + ) + ) + return IDA_REDACTION_MARKER; + const characters = new Set(); + for (const secret of secrets) + for (const character of secret) characters.add(character); + // A graphic scalar absent from every credential cannot contain or connect + // a credential, and can be written verbatim inside a JSON string. + const graphic = /[\p{L}\p{N}\p{P}\p{S}]/u; + for (let point = 0x241f; point <= 0x10ffff; point++) { + if (point >= 0xd800 && point <= 0xdfff) continue; + const character = String.fromCodePoint(point); + if (graphic.test(character) && !characters.has(character)) return character; + } + throw new AnalysisProtocolError( + "Configured IDA credentials leave no available diagnostic redaction character.", + ); +}; + +/** Own configured credential interpretation for all IDA error diagnostics. */ +export const createIdaDiagnosticRedactor = ( + config?: IdaConfiguration, +): Readonly<{ + text(message: string): string; + json(message: string): string; + blocks(messages: readonly string[]): string; + message(prefix: string, ownedBody: string): string; +}> => { + const headers = + config !== undefined && "headers" in config + ? Object.entries(config.headers) + : []; + const values = headers.flatMap(([name, value]) => { + if (!/^(?:authorization|proxy-authorization|cookie)$/iu.test(name)) + return []; + // Fetch normalizes surrounding HTTP whitespace before sending headers. + const normalized = value.replace(/^[ \t\r\n]+|[ \t\r\n]+$/gu, ""); + if (normalized.length === 0) return []; + return [ + value, + normalized, + ...(name.toLowerCase() === "cookie" + ? cookiePayloads(normalized) + : authorizationPayloads(normalized)), + ]; + }); + const secrets = [...new Set(values.filter((value) => value.length > 0))].sort( + (left, right) => right.length - left.length, + ); + const replacement = + secrets.length === 0 ? IDA_REDACTION_MARKER : redactionMarker(secrets); + const alternatives = secrets + .map((secret) => secret.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")) + .join("|"); + // Look ahead at each source position; a global consuming match loses overlap. + // Longest-first alternatives cover every shorter match at the same position. + const pattern = + alternatives.length === 0 + ? undefined + : new RegExp("(?=(" + alternatives + "))", "gu"); + const matches = function* (value: string): Iterable { + if (pattern === undefined) return; + let pending: IdaDiagnosticMatch | undefined; + for (const match of value.matchAll(pattern)) { + const credential = match[1]; + if (credential === undefined || credential.length === 0) + throw new AnalysisProtocolError( + "IDA diagnostic matching omitted its credential span.", + ); + const span = { start: match.index, end: match.index + credential.length }; + if (pending !== undefined && span.start < pending.end) { + pending = { + start: pending.start, + end: Math.max(pending.end, span.end), + }; + } else { + if (pending !== undefined) yield pending; + pending = span; + } + } + if (pending !== undefined) yield pending; + }; + const text = boundedDiagnostic((message) => + writeIdaDiagnostic(message, matches(message), replacement), + ); + const json = boundedDiagnostic((message) => + secrets.length === 0 + ? message + : redactIdaJsonDiagnostic(message, matches, replacement), + ); + return { + text, + json, + blocks(messages) { + let bytes = 0; + let sourceUnits = 0; + for (let index = 0; index < messages.length; index++) { + const value = messages[index]!; + bytes += Buffer.byteLength(value, "utf8") + (index === 0 ? 0 : 1); + sourceUnits += value.length + (index === 0 ? 0 : 1); + if (bytes > IDA_DIAGNOSTIC_SOURCE_BYTES) return text(omittedDiagnostic); + } + function* parts(): Generator { + for (let index = 0; index < messages.length; index++) { + if (index > 0) yield "\n"; + const value = messages[index]!; + yield secrets.length === 0 || value.length === 0 + ? value + : redactIdaUnclassifiedDiagnostic(value, matches, replacement); + } + } + return joinIdaDiagnosticParts(parts(), sourceUnits); + }, + message(prefix, ownedBody) { + // Normalized wire headers and supported decoded candidates have no LF. + // Preserve the already-owned body without inspecting its replacement text. + return joinIdaDiagnosticParts( + [text(prefix), "\n", ownedBody], + prefix.length + 1 + ownedBody.length, + ); + }, + }; +}; diff --git a/src/ida/IdaDiagnosticWriter.ts b/src/ida/IdaDiagnosticWriter.ts new file mode 100644 index 0000000000..7507405074 --- /dev/null +++ b/src/ida/IdaDiagnosticWriter.ts @@ -0,0 +1,85 @@ +import { constants } from "node:buffer"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { bufferedJsonParts } from "../domain/jsonSerialization.js"; + +export const IDA_REDACTION_MARKER = "[redacted transport credential]"; +export type IdaDiagnosticMatch = Readonly<{ start: number; end: number }>; + +type DiagnosticRepresentation = "redacted-output" | "json-character-run"; + +/** An owned diagnostic cannot be represented as one runtime string. */ +export class IdaDiagnosticResourceError extends AnalysisResourceConstraintError { + constructor( + inputCodeUnits: number, + representedCodeUnits: number, + representation: DiagnosticRepresentation = "redacted-output", + ) { + super( + "ida_diagnostic_redaction", + "memory", + representation === "redacted-output" + ? "Redacted IDA diagnostics exceed the runtime's maximum string length." + : "Temporary IDA diagnostic JSON character decoding exceeds the runtime's maximum string length.", + { + boundary: "ida-diagnostic-redaction", + representation, + input_code_units: inputCodeUnits, + ...(representation === "redacted-output" + ? { output_code_units_at_least: representedCodeUnits } + : { json_decoding_code_units_at_least: representedCodeUnits }), + max_string_code_units: constants.MAX_STRING_LENGTH, + }, + ); + } +} + +/** Account for the complete representation before allocating its final string. */ +export const joinIdaDiagnosticParts = ( + parts: Iterable, + inputCodeUnits: number, +): string => { + function* checkedParts(): Generator { + let outputCodeUnits = 0; + for (const part of parts) { + outputCodeUnits += part.length; + if (outputCodeUnits > constants.MAX_STRING_LENGTH) + throw new IdaDiagnosticResourceError(inputCodeUnits, outputCodeUnits); + yield part; + } + } + return [...bufferedJsonParts(checkedParts())].join(""); +}; + +/** Write ordered source spans without retaining a per-match collection. */ +export const writeIdaDiagnostic = ( + text: string, + matches: Iterable, + replacement: string = IDA_REDACTION_MARKER, +): string => { + function* parts(): Generator { + let cursor = 0; + let pending: IdaDiagnosticMatch | undefined; + for (const match of matches) { + if (pending !== undefined && match.start < pending.end) { + pending = { + start: pending.start, + end: Math.max(pending.end, match.end), + }; + continue; + } + if (pending !== undefined) { + yield text.slice(cursor, pending.start); + yield replacement; + cursor = pending.end; + } + pending = match; + } + if (pending !== undefined) { + yield text.slice(cursor, pending.start); + yield replacement; + cursor = pending.end; + } + yield text.slice(cursor); + } + return joinIdaDiagnosticParts(parts(), text.length); +}; diff --git a/src/ida/IdaDoctor.ts b/src/ida/IdaDoctor.ts index 750ca757ec..8a711aff14 100644 --- a/src/ida/IdaDoctor.ts +++ b/src/ida/IdaDoctor.ts @@ -52,13 +52,16 @@ const inspectPrivateWorkspace = (): DoctorProviderCheck => { return { name: "private_workspace", ok: privacy.available, - code: privacy.available ? null : "unsupported_host", + code: privacy.available ? null : privacy.code, detail: privacy.available ? `Private workspace authority is available (${privacy.proof}).` : privacy.reason, - remediation: privacy.available - ? null - : "Use the matching Windows x64 REA package and a writable local NTFS temporary directory.", - classification: "unsupported_host", + remediation: privacy.available ? null : privacy.remediation, + classification: + !privacy.available && privacy.code === "runtime_missing" + ? "missing_dependency" + : !privacy.available && privacy.code === "runtime_unavailable" + ? "runtime_failure" + : "unsupported_host", }; }; diff --git a/src/ida/IdaJsonDiagnostic.test.ts b/src/ida/IdaJsonDiagnostic.test.ts new file mode 100644 index 0000000000..3929551d33 --- /dev/null +++ b/src/ida/IdaJsonDiagnostic.test.ts @@ -0,0 +1,337 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { + IdaDiagnosticResourceError, + writeIdaDiagnostic, +} from "./IdaDiagnosticWriter.js"; +import { createIdaDiagnosticRedactor } from "./IdaDiagnosticRedactor.js"; +import { redactIdaJsonDiagnostic } from "./IdaJsonDiagnostic.js"; + +const runtime = vi.hoisted(() => ({ limit: undefined as number | undefined })); +vi.mock("node:buffer", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + constants: { + ...original.constants, + get MAX_STRING_LENGTH() { + return runtime.limit ?? original.constants.MAX_STRING_LENGTH; + }, + }, + }; +}); +afterEach(() => { + runtime.limit = undefined; +}); + +const marker = "[redacted transport credential]"; +const forCredential = (credential: string) => + createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: `Custom ${credential}` }, + mode: "attached", + timeoutMs: 1000, + }); + +describe("IDA JSON diagnostic source preservation", () => { + it("matches equivalent JSON encodings without enumerating credential spellings", () => { + const redactor = forCredential(""); + const encodings = [ + ["<", String.raw`\u003c`, String.raw`\u003C`], + ["s", String.raw`\u0073`], + ["/", String.raw`\/`, String.raw`\u002f`, String.raw`\u002F`], + [">", String.raw`\u003e`, String.raw`\u003E`], + ].reduce( + (prefixes, units) => + prefixes.flatMap((prefix) => units.map((unit) => prefix + unit)), + [""], + ); + for (const encoded of encodings) + expect(redactor.json(`{"reason":"${encoded}"}`)).toBe( + `{"reason":"${marker}"}`, + ); + }); + + it("preserves unrelated spelling inside a matching string", () => { + const source = String.raw`{"reason":"\u003cpublic\u003e\/asset \u003csecret\u003e \ud83d\uDE00"}`; + expect(forCredential("").json(source)).toBe( + String.raw`{"reason":"\u003cpublic\u003e\/asset ` + + marker + + String.raw` \ud83d\uDE00"}`, + ); + }); + + it("retains duplicate fields, whitespace and exact numeric lexemes", () => { + const source = + '{\n "\\u003csecret\\u003e": 9007199254740993, "n": -0, "n": 1.00e+03, "value": ""\n}'; + expect(forCredential("").json(source)).toBe( + `{\n "${marker}": 9007199254740993, "n": -0, "n": 1.00e+03, "value": "${marker}"\n}`, + ); + }); + + it.each([ + { credential: "a😀b", encoded: String.raw`a\ud83d\uDE00b` }, + { credential: "a\ud800b", encoded: String.raw`a\uD800b` }, + { credential: "a\nb", encoded: String.raw`a\nb` }, + { credential: 'a"b', encoded: String.raw`a\u0022b` }, + ])("maps UTF-16 source spans for $encoded", ({ credential, encoded }) => { + expect(forCredential(credential).json(`"before ${encoded} after"`)).toBe( + `"before ${marker} after"`, + ); + }); + + it.each([ + { + source: String.raw`{broken: "\u003csecret\u003e",`, + expected: `{broken: "${marker}",`, + }, + { + source: String.raw`{"x":"\u003csecret\u003e`, + expected: `{"x":"${marker}`, + }, + { + source: String.raw`{"x":"\u003csecret\u003e\q"}`, + expected: `{"x":"${marker}\\q"}`, + }, + ])( + "preserves malformed source around known complete characters: $source", + ({ source, expected }) => { + expect(forCredential("").json(source)).toBe(expected); + }, + ); + + it("does not join decoded fragments across invalid escapes", () => { + const source = String.raw`{"x":""}`; + expect(forCredential("").json(source)).toBe(source); + }); + + it("retains literal matches crossing invalid JSON units", () => { + expect( + forCredential(String.raw`foo\qbar`).json(String.raw`{"x":"foo\qbar"}`), + ).toBe(`{"x":"${marker}"}`); + }); + + it.each( + ["b", "f", "n", "r", "t", "/", '"', "\\", "u0061"].map( + (escape) => "folder\\" + escape + "tmp", + ), + )( + "retains literal credentials in ambiguous malformed fragments: %s", + (credential) => { + const redactor = forCredential(credential); + for (const [before, after] of [ + ["Rejected ", ""], + ['Rejected "', '"'], + ['{"reason":"Rejected ', ""], + ['{"reason":"Rejected ', String.raw`\q"}`], + ]) + expect(redactor.json(before + credential + after)).toBe( + before + marker + after, + ); + expect(redactor.json('"' + credential + '"')).toBe( + '"' + credential + '"', + ); + expect(redactor.json(JSON.stringify("Rejected " + credential))).toBe( + JSON.stringify("Rejected " + marker), + ); + }, + ); + + it("uses decoded values instead of matching unrelated escape syntax", () => { + const source = String.raw`{"x":"\u003cpublic\u003e","y":"u003c"}`; + expect(forCredential("u003c").json(source)).toBe( + String.raw`{"x":"\u003cpublic\u003e","y":"` + marker + '"}', + ); + }); + + it("retains literal opaque matches spanning quoted fragments in plain bodies", () => { + const credential = 'prefix"quoted"suffix'; + expect( + forCredential(credential).json( + `Auth Custom ${credential}; payload ${credential}; path /caller/input`, + ), + ).toBe(`Auth ${marker}; payload ${marker}; path /caller/input`); + }); + + it("replaces adjacent matches once without rescanning the marker", () => { + expect( + forCredential("redacted").json(String.raw`"redacted\u0072edacted"`), + ).toBe('"␟␟"'); + }); + + it("handles unquoted JSON fragments while retaining ordinary text semantics", () => { + const source = String.raw`\u003csecret\u003e`; + const redactor = forCredential(""); + expect(redactor.text(source)).toBe(source); + expect(redactor.json(source)).toBe(marker); + const invalid = String.raw``; + expect(redactor.json(invalid)).toBe(invalid); + const unrelated = String.raw`"\u003cpublic\u003e"`; + expect(forCredential("u003c").json(unrelated)).toBe(unrelated); + expect(forCredential("u003c").json(unrelated.slice(1, -1))).toBe( + "\\" + marker + String.raw`public\u003e`, + ); + }); + it("leaves ordinary text diagnostics and credential-free configurations unchanged", () => { + const source = String.raw`{"x":"\u003csecret\u003e"}`; + expect(forCredential("").text(source)).toBe(source); + const redactor = createIdaDiagnosticRedactor({ + headers: {}, + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + }); + expect(redactor.json(source)).toBe(source); + }); +}); + +describe("IDA diagnostic output resource bounds", () => { + it.each(["text", "json"] as const)( + "bounds %s expansion using the runtime string limit", + (format) => { + runtime.limit = 64; + const redactor = forCredential("Q"); + const prefix = format === "json" ? '"' : ""; + const suffix = prefix; + expect(redactor[format](prefix + "QQ" + suffix)).toBe( + prefix + marker + marker + suffix, + ); + let failure: unknown; + try { + redactor[format](prefix + "QQQ" + suffix); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(IdaDiagnosticResourceError); + if (!(failure instanceof IdaDiagnosticResourceError)) + throw new Error("Expected an owned diagnostic resource failure"); + expect(failure.reportedLimits).toMatchObject({ + boundary: "ida-diagnostic-redaction", + max_string_code_units: 64, + }); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "resource_constraint", + details: { resource: "memory" }, + }); + }, + ); + + it("bounds temporary JSON character decoding before wrapping an oversized run", () => { + runtime.limit = 64; + const redactor = forCredential("Z"); + expect(redactor.json("Q".repeat(62))).toBe("Q".repeat(62)); + let failure: unknown; + try { + redactor.json("Q".repeat(63)); + } catch (cause: unknown) { + failure = cause; + } + expect(failure).toBeInstanceOf(IdaDiagnosticResourceError); + if (!(failure instanceof IdaDiagnosticResourceError)) + throw new Error("Expected a temporary decoding resource failure"); + expect(failure.message).toContain("Temporary"); + expect(failure.reportedLimits).toEqual({ + boundary: "ida-diagnostic-redaction", + representation: "json-character-run", + input_code_units: 63, + json_decoding_code_units_at_least: 65, + max_string_code_units: 64, + }); + }); + + it("closes the first match stream if priming the second stream fails", () => { + runtime.limit = 64; + const source = "Q".repeat(63); + let closed = false; + function* matches() { + try { + yield { start: 0, end: 1 }; + } finally { + closed = true; + } + } + expect(() => redactIdaJsonDiagnostic(source, matches)).toThrow( + IdaDiagnosticResourceError, + ); + expect(closed).toBe(true); + }); + + it("stops consuming a lazy match stream before oversized output is allocated", () => { + runtime.limit = 32; + let consumed = 0; + function* matches() { + for (let index = 0; index < 10000; index++) { + consumed++; + yield { start: index, end: index + 1 }; + } + } + expect(() => writeIdaDiagnostic("Q".repeat(10000), matches())).toThrow( + IdaDiagnosticResourceError, + ); + expect(consumed).toBeLessThan(10); + }); +}); + +describe("overlapping raw and decoded JSON credential spans", () => { + const overlappingMarker = "␟"; + const overlappingRedactor = () => + createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Custom X]", Cookie: "a=]SECRET" }, + mode: "attached", + timeoutMs: 1000, + }); + it.each([ + String.raw`{"reason":"X]SECRET","n":1.00e+03}`, + String.raw`{"reason":"X\u005dSECRET","n":1.00e+03}`, + ])("unions decoded overlaps before monotone source mapping: %s", (source) => { + expect(overlappingRedactor().json(source)).toBe( + '{"reason":"' + overlappingMarker + '","n":1.00e+03}', + ); + }); + it.each(["X]SECRET", String.raw`X\u005dSECRET`])( + "unions raw/recovered overlaps in a malformed document: %s", + (source) => { + expect( + overlappingRedactor().json("Rejected " + source + " at /caller/input"), + ).toBe("Rejected " + overlappingMarker + " at /caller/input"); + }, + ); + it("maps a merged overlap containing an escaped surrogate pair", () => { + const encoded = Buffer.from("fixture-user:😀X]", "utf8").toString("base64"); + const redactor = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Basic " + encoded, Cookie: "a=]SECRET" }, + mode: "attached", + timeoutMs: 1000, + }); + expect( + redactor.json( + String.raw`{"reason":"\ud83d\ude00X\u005dSECRET","context":"kept"}`, + ), + ).toBe('{"reason":"' + overlappingMarker + '","context":"kept"}'); + }); + it("unions dense escaped decoded overlaps lazily", () => { + const source = '"' + String.raw`\u0041`.repeat(4096) + '"'; + const result = forCredential("AA").json(source); + expect(result.length).toBe(marker.length + 2); + expect(result).toBe('"' + marker + '"'); + }); +}); + +it("bounds the final multi-block diagnostic representation", () => { + runtime.limit = 64; + const redactor = forCredential("Q"); + expect(() => redactor.blocks(["QQ", "QQ"])).toThrow( + IdaDiagnosticResourceError, + ); +}); + +it("bounds context plus the already-owned diagnostic body before joining", () => { + runtime.limit = 64; + const redactor = forCredential("Q"); + const body = redactor.text("QQ"); + expect(() => redactor.message("ctx", body)).toThrow( + IdaDiagnosticResourceError, + ); +}); diff --git a/src/ida/IdaJsonDiagnostic.ts b/src/ida/IdaJsonDiagnostic.ts new file mode 100644 index 0000000000..4f0823b44f --- /dev/null +++ b/src/ida/IdaJsonDiagnostic.ts @@ -0,0 +1,216 @@ +import { constants } from "node:buffer"; +import { createScanner, type ScanError, type SyntaxKind } from "jsonc-parser"; +import { + IDA_REDACTION_MARKER, + IdaDiagnosticResourceError, + writeIdaDiagnostic, + type IdaDiagnosticMatch, +} from "./IdaDiagnosticWriter.js"; + +// jsonc-parser declares ambient const enums, which cannot be imported as values +// under verbatimModuleSyntax. Keep its scanner codes local and type-checked. +const JSON_STRING_LITERAL = 10 satisfies SyntaxKind; +const JSON_EOF = 17 satisfies SyntaxKind; +const JSON_NO_ERROR = 0 satisfies ScanError; + +/** Matches are non-overlapping and ordered by their UTF-16 source offsets. */ +type Matches = (value: string) => Iterable; + +// Complete JSON character runs remain meaningful even in an unfinished or +// malformed string. Invalid units separate runs instead of joining their text. +const JSON_CHARACTER_RUN = + /(?:[^"\\\u0000-\u001f]|\\(?:["\\/bfnrt]|u[\dA-Fa-f]{4}))+/gu; + +function* stringMatches( + encoded: string, + decoded: string, + offset: number, + matches: Matches, +): Generator { + // Match offsets and JSON escape widths are both UTF-16 units. No source bytes + // outside a matched value need to be normalized or serialized again. + let sourceCursor = 0; + let decodedCursor = 0; + const sourceOffset = (boundary: number): number => { + while (decodedCursor < boundary) { + sourceCursor += + encoded[sourceCursor] === "\\" + ? encoded[sourceCursor + 1] === "u" + ? 6 + : 2 + : 1; + decodedCursor++; + } + return offset + sourceCursor; + }; + for (const match of matches(decoded)) + yield { + start: sourceOffset(match.start), + end: sourceOffset(match.end), + }; +} + +function* characterRunMatches( + encoded: string, + offset: number, + matches: Matches, + inputCodeUnits: number, +): Generator { + for (const run of encoded.matchAll(JSON_CHARACTER_RUN)) { + // The run grammar admits only complete JSON characters. JSON.parse owns + // their decoded meaning; malformed units remain verbatim between runs. + const decodingCodeUnits = run[0].length + 2; + if (decodingCodeUnits > constants.MAX_STRING_LENGTH) + throw new IdaDiagnosticResourceError( + inputCodeUnits, + decodingCodeUnits, + "json-character-run", + ); + const decoded: unknown = JSON.parse('"' + run[0] + '"'); + if (typeof decoded !== "string") + throw new TypeError("A JSON character run did not decode to a string."); + yield* stringMatches(run[0], decoded, offset + run.index, matches); + } +} + +function* decodedMatches( + text: string, + matches: Matches, +): Generator { + let scannedEnd = 0; + const scanner = createScanner(text); + while (scanner.scan() !== JSON_EOF) { + if (scanner.getToken() !== JSON_STRING_LITERAL) continue; + const start = scanner.getTokenOffset(); + const end = start + scanner.getTokenLength(); + if (scannedEnd < start) + yield* characterRunMatches( + text.slice(scannedEnd, start), + scannedEnd, + matches, + text.length, + ); + if (scanner.getTokenError() === JSON_NO_ERROR) + yield* stringMatches( + text.slice(start + 1, end - 1), + scanner.getTokenValue(), + start + 1, + matches, + ); + else + yield* characterRunMatches( + text.slice(start + 1, end), + start + 1, + matches, + text.length, + ); + scannedEnd = end; + } + yield* characterRunMatches( + text.slice(scannedEnd), + scannedEnd, + matches, + text.length, + ); +} + +function* literalMatches( + text: string, + matches: Matches, + validDocument: boolean, +): Generator { + if (!validDocument) { + // Recovery cannot establish a unique meaning for a malformed document. + yield* matches(text); + return; + } + const scanner = createScanner(text); + let complete: IdaDiagnosticMatch | undefined; + let ended = false; + for (const match of matches(text)) { + while (!ended && (complete === undefined || complete.end <= match.start)) { + complete = undefined; + while (scanner.scan() !== JSON_EOF) { + if (scanner.getToken() !== JSON_STRING_LITERAL) continue; + const start = scanner.getTokenOffset(); + complete = { start, end: start + scanner.getTokenLength() }; + break; + } + if (complete === undefined) ended = true; + } + // Only a complete string in a valid document owns decoded-only semantics. + // Raw matches crossing tokens remain visible to the credential boundary. + if ( + complete === undefined || + match.start < complete.start || + match.end > complete.end + ) + yield match; + } +} + +function* mergeMatches( + left: Iterable, + right: Iterable, +): Generator { + const leftIterator = left[Symbol.iterator](); + const rightIterator = right[Symbol.iterator](); + try { + let leftMatch = leftIterator.next(); + let rightMatch = rightIterator.next(); + while (!leftMatch.done || !rightMatch.done) { + if ( + rightMatch.done || + (!leftMatch.done && leftMatch.value.start <= rightMatch.value.start) + ) { + if (leftMatch.done) break; + yield leftMatch.value; + leftMatch = leftIterator.next(); + } else { + yield rightMatch.value; + rightMatch = rightIterator.next(); + } + } + } finally { + try { + leftIterator.return?.(); + } finally { + rightIterator.return?.(); + } + } +} + +/** Redact complete or malformed JSON diagnostics without rewriting unrelated source. */ +export const redactIdaJsonDiagnostic = ( + text: string, + matches: Matches, + replacement: string = IDA_REDACTION_MARKER, +): string => { + let validDocument = false; + try { + JSON.parse(text); + validDocument = true; + } catch { + // Keep both raw and decoded interpretations when the document is incomplete. + } + return writeIdaDiagnostic( + text, + mergeMatches( + literalMatches(text, matches, validDocument), + decodedMatches(text, matches), + ), + replacement, + ); +}; + +/** Untyped producer text retains both literal and JSON character interpretations. */ +export const redactIdaUnclassifiedDiagnostic = ( + text: string, + matches: Matches, + replacement: string = IDA_REDACTION_MARKER, +): string => + writeIdaDiagnostic( + text, + mergeMatches(matches(text), decodedMatches(text, matches)), + replacement, + ); diff --git a/src/ida/IdaMcpConnection.test.ts b/src/ida/IdaMcpConnection.test.ts index a582ed902c..139c377e24 100644 --- a/src/ida/IdaMcpConnection.test.ts +++ b/src/ida/IdaMcpConnection.test.ts @@ -1,31 +1,68 @@ -import { describe, expect, it } from "vitest"; +import { inspect } from "node:util"; +import { SdkErrorCode, SdkHttpError } from "@modelcontextprotocol/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { + createIdaMcpConnection, decodeIdaToolResult, - redactIdaTransportFailure, } from "./IdaMcpConnection.js"; +import { createIdaDiagnosticRedactor } from "./IdaDiagnosticRedactor.js"; describe("upstream IDA tool result decoding", () => { - it("redacts configured transport authentication without redacting caller evidence", () => { - const error = redactIdaTransportFailure( - new Error( - "Invalid Authorization value Bearer fixture-secret at /caller/input", - ), - { - url: "http://127.0.0.1:12345/mcp", - headers: { Authorization: "Bearer fixture-secret" }, - mode: "attached", - timeoutMs: 1000, - }, - ); - expect(error.message).not.toContain("fixture-secret"); - expect(error.message).toContain("/caller/input"); + it("leaves successful legacy, structured, and native evidence unchanged", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Bearer fixture-secret" }, + mode: "attached", + timeoutMs: 1000, + }); + const evidence = "Caller-selected fixture-secret evidence"; expect( - decodeIdaToolResult({ - content: [ - { type: "text", text: "Caller-selected fixture-secret evidence" }, - ], - }), - ).toBe("Caller-selected fixture-secret evidence"); + decodeIdaToolResult( + { content: [{ type: "text", text: evidence }] }, + undefined, + redact, + ), + ).toBe(evidence); + expect( + decodeIdaToolResult( + { content: [], structuredContent: { evidence } }, + undefined, + redact, + ), + ).toEqual({ evidence }); + expect( + decodeIdaToolResult( + { content: [{ type: "text", text: JSON.stringify({ evidence }) }] }, + "native", + redact, + ), + ).toEqual({ evidence }); + }); + it("redacts credentials in truncated-output diagnostics before constructing the error", () => { + const redact = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Bearer fixture-secret" }, + mode: "attached", + timeoutMs: 1000, + }); + expect(() => + decodeIdaToolResult( + { + content: [], + _meta: { + ida_mcp: { + output_truncated: true, + download_hint: + "Download /caller/output?cursor=kept using Bearer fixture-secret", + }, + }, + }, + undefined, + redact, + ), + ).toThrow( + "Download /caller/output?cursor=kept using [redacted transport credential]", + ); }); it("decodes scalar wrappers and legacy text without dropping structured dictionaries", () => { expect( @@ -79,3 +116,208 @@ describe("upstream IDA tool result decoding", () => { ).toThrow("Download the full upstream output"); }); }); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +const failingHttpConnection = ( + credential: string, + response: Response | Error, +) => { + // Exercise the real SDK consumer with an explicitly simulated Fetch boundary. + vi.stubGlobal( + "fetch", + response instanceof Error + ? vi.fn().mockRejectedValue(response) + : vi.fn().mockResolvedValue(response), + ); + return createIdaMcpConnection( + { + url: "http://127.0.0.1:12345/mcp", + headers: { Authorization: "Custom " + credential }, + mode: "attached", + timeoutMs: 1000, + }, + {}, + ); +}; + +const inspectRetainedError = (error: unknown): string => + inspect(error, { depth: null, maxStringLength: null, breakLength: Infinity }); + +describe("retained IDA diagnostic errors", () => { + it.each([ + { + credential: "fixture-secret", + body: '{"reason":"fixture-secret at /caller/input",', + forbidden: "fixture-secret", + }, + { + credential: "", + body: String.raw`["\u003ctoken\u003e",x]`, + forbidden: "u003ctoken", + }, + ])( + "retains only sanitized malformed HTTP JSON: $credential", + async ({ credential, body, forbidden }) => { + const connection = failingHttpConnection( + credential, + new Response(body, { headers: { "Content-Type": "application/json" } }), + ); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected a malformed HTTP JSON failure"); + expect(failure.message).toContain("[redacted transport credential]"); + expect(inspectRetainedError(failure)).not.toContain(forbidden); + expect(inspectRetainedError(failure)).not.toContain(credential); + await connection.close(); + }, + ); + + it("does not rescan an owned malformed JSON replacement marker", async () => { + const connection = failingHttpConnection( + "redacted", + new Response("Rejected redacted", { + headers: { "Content-Type": "application/json" }, + }), + ); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected a malformed HTTP JSON failure"); + expect(failure.message).toBe( + "IDA MCP returned malformed HTTP JSON. Response body: \nRejected ␟", + ); + expect(inspectRetainedError(failure)).not.toContain("[[redacted"); + await connection.close(); + }); + + it("replaces an unknown network error graph with a safe owned reason", async () => { + const upstream = Object.assign( + new Error("fixture-secret refused at /caller/input", { + cause: new Error("nested fixture-secret"), + }), + { headers: { Authorization: "Custom fixture-secret" } }, + ); + const connection = failingHttpConnection("fixture-secret", upstream); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected a network failure"); + expect(failure.message).toContain("refused at /caller/input"); + expect(failure.cause).not.toBe(upstream); + expect(inspectRetainedError(failure)).not.toContain("fixture-secret"); + await connection.close(); + }); + + it("retains meaningful HTTP status without unsafe SDK metadata", async () => { + const connection = failingHttpConnection( + "fixture-secret", + new Response("unavailable fixture-secret at /caller/input", { + status: 500, + statusText: "fixture-secret", + headers: { "Content-Type": "text/plain" }, + }), + ); + const failure: unknown = await connection + .connect() + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) + throw new Error("Expected an HTTP failure"); + expect(failure.cause).toMatchObject({ data: { status: 500 } }); + expect(failure.message).toContain( + "unavailable [redacted transport credential] at /caller/input", + ); + expect(inspectRetainedError(failure)).not.toContain("fixture-secret"); + await connection.close(); + }); +}); + +it.each([ + { + code: SdkErrorCode.ClientHttpNotImplemented, + message: "Changed SDK context: fixture-secret", + }, + { + code: SdkErrorCode.ClientHttpFailedToOpenStream, + message: "Error POSTing to endpoint: fixture-secret", + }, +])( + "does not infer owned body spans from an unknown SDK shape: $code / $message", + async ({ code, message }) => { + const connection = failingHttpConnection( + "fixture-secret", + new SdkHttpError(code, message, { status: 500, text: "fixture-secret" }), + ); + try { + const failure = await connection + .connect() + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected SDK failure"); + expect(inspectRetainedError(failure)).not.toContain("fixture-secret"); + expect(failure.message).toContain("[redacted transport credential]"); + expect(failure.cause).toMatchObject({ code, data: { status: 500 } }); + } finally { + await connection.close(); + } + }, +); + +it("redacts JSON character candidates in unclassified SDK errors", async () => { + const connection = failingHttpConnection( + 'q"uote', + new Error(String.raw`network context {"error":"q\"uote"} at /caller/input`), + ); + try { + const failure = await connection.connect().catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected SDK failure"); + expect(failure.message).toContain( + '{"error":"[redacted transport credential]"} at /caller/input', + ); + expect(inspectRetainedError(failure)).not.toContain(String.raw`q\"uote`); + } finally { + await connection.close(); + } +}); + +it("owns unclassified truncated-output hints separately from their context", () => { + const diagnostic = createIdaDiagnosticRedactor({ + url: "http://127.0.0.1:12345/mcp", + mode: "attached", + timeoutMs: 1000, + headers: { Authorization: 'Custom q"uote' }, + }); + let failure: unknown; + try { + decodeIdaToolResult( + { + content: [], + _meta: { + ida_mcp: { + output_truncated: true, + download_hint: String.raw`{"error":"q\"uote","path":"/caller/output"}`, + }, + }, + }, + undefined, + diagnostic, + ); + } catch (cause: unknown) { + failure = cause; + } + if (!(failure instanceof Error)) + throw new Error("Expected truncation failure"); + expect(failure.message).toContain( + '{"error":"[redacted transport credential]","path":"/caller/output"}', + ); + expect(inspectRetainedError(failure)).not.toContain(String.raw`q\"uote`); +}); diff --git a/src/ida/IdaMcpConnection.ts b/src/ida/IdaMcpConnection.ts index 8137edcc96..1db07b8615 100644 --- a/src/ida/IdaMcpConnection.ts +++ b/src/ida/IdaMcpConnection.ts @@ -1,8 +1,16 @@ +import { + createIdaDiagnosticRedactor, + IDA_DIAGNOSTIC_SOURCE_BYTES, +} from "./IdaDiagnosticRedactor.js"; +import { IdaDiagnosticResourceError } from "./IdaDiagnosticWriter.js"; import { ChildProcess } from "node:child_process"; import { Socket } from "node:net"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { Client, + ProtocolError, + SdkErrorCode, + SdkHttpError, StreamableHTTPClientTransport, } from "@modelcontextprotocol/client"; import { @@ -12,7 +20,11 @@ import { import { z } from "zod"; import type { IdaConfiguration } from "./IdaConfiguration.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; -import { AnalysisProtocolError } from "../domain/analysisErrorCore.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { + AnalysisProtocolError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; /** Injectable MCP boundary used by the provider and transport conformance tests. */ @@ -53,14 +65,14 @@ const toolResultSchema = z.object({ .optional(), }); +const IDA_STDIO_MAX_MESSAGE_BYTES = 64 * 1024 * 1024; + const parseIdaJsonText = (text: string): JsonValue => { let value: unknown; try { value = JSON.parse(text); - } catch (cause: unknown) { - throw new AnalysisProtocolError("IDA MCP returned malformed JSON text.", { - cause, - }); + } catch { + throw new AnalysisProtocolError("IDA MCP returned malformed JSON text."); } return jsonValueSchema.parse(value); }; @@ -69,18 +81,30 @@ const parseIdaJsonText = (text: string): JsonValue => { export const decodeIdaToolResult = ( input: unknown, protocol?: "native", + diagnostic: IdaDiagnosticRedactor = createIdaDiagnosticRedactor(), ): JsonValue => { const result = toolResultSchema.parse(input); const blocks = result.content.flatMap((block) => block.type === "text" && block.text !== undefined ? [block.text] : [], ); - if (result.isError === true) + if (result.isError === true) { + const reason = diagnostic.blocks(blocks); throw new AnalysisProtocolError( - `IDA MCP tool failed: ${blocks.join("\n")}`, + diagnostic.message("IDA MCP tool failed: ", reason), + { + diagnostics: { reason }, + }, ); + } if (result._meta?.ida_mcp?.output_truncated === true) throw new AnalysisProtocolError( - `IDA MCP truncated its output; the preview cannot establish a complete observation. ${result._meta.ida_mcp.download_hint ?? "Use the upstream output download workflow to retrieve the full result."}`, + diagnostic.message( + "IDA MCP truncated its output; the preview cannot establish a complete observation. ", + diagnostic.blocks([ + result._meta.ida_mcp.download_hint ?? + "Use the upstream output download workflow to retrieve the full result.", + ]), + ), ); // Native MCP retains the complete JSON text while structuredContent may // contain a shortened preview. Parse that producer-owned representation once. @@ -113,59 +137,264 @@ export const decodeIdaToolResult = ( return values[0] ?? null; }; -/** Redact configured transport authentication values from SDK diagnostics only. */ -export const redactIdaTransportFailure = ( - cause: unknown, - config: IdaConfiguration, -): Error => { - let message = - cause instanceof Error - ? cause.message - : "Unknown IDA MCP transport failure"; - if ("headers" in config) { - for (const [name, value] of Object.entries(config.headers)) { - if (!/^(?:authorization|proxy-authorization|cookie)$/iu.test(name)) - continue; - const credential = value.match(/^(?:Bearer|Basic)\s+([\s\S]+)$/iu)?.[1]; - for (const secret of [value, credential]) - if (secret !== undefined && secret.length > 0) - message = message - .split(secret) - .join("[redacted transport credential]"); - } +// Retain only producer-sanitized JSON diagnostics, never a native parser cause. +class IdaHttpJsonError extends Error { + constructor(text: string, redactor: IdaDiagnosticRedactor) { + super( + redactor.message( + "IDA MCP returned malformed HTTP JSON. Response body: ", + text, + ), + ); } - return new Error(message, { cause }); +} + +type IdaDiagnosticRedactor = ReturnType; + +type IdaContentType = Readonly<{ + essence: string; + subtype: string; + charset: string | undefined; + parameters: ReadonlyMap; +}>; + +const httpQuotedOctet = (code: number): boolean => + code === 9 || + code === 32 || + (code >= 33 && code <= 126) || + (code >= 128 && code <= 255); + +/** Consume one complete RFC 9110 media type; never discard a malformed parameter. */ +const parseIdaContentType = ( + text: string, +): Result => { + let cursor = 0; + const token = new RegExp("[!#$%&'*+.^_`|~0-9A-Za-z-]+", "y"); + const readToken = (): string | undefined => { + token.lastIndex = cursor; + const match = token.exec(text); + if (match === null) return undefined; + cursor = token.lastIndex; + return match[0]; + }; + const readOws = (): void => { + while (text[cursor] === " " || text[cursor] === "\t") cursor++; + }; + const readQuoted = (): string | undefined => { + cursor++; + let value = ""; + while (cursor < text.length) { + let code = text.charCodeAt(cursor++); + if (code === 34) return value; + if (code === 92) { + code = text.charCodeAt(cursor++); + if (!httpQuotedOctet(code)) return undefined; + } else if (!httpQuotedOctet(code)) return undefined; + value += String.fromCharCode(code); + } + return undefined; + }; + readOws(); + const type = readToken()?.toLowerCase(); + if (type === undefined || text[cursor++] !== "/") return err("malformed"); + const subtype = readToken()?.toLowerCase(); + if (subtype === undefined) return err("malformed"); + const parameters = new Map(); + let ambiguous = false; + while (true) { + readOws(); + if (cursor === text.length) break; + if (text[cursor++] !== ";") return err("malformed"); + readOws(); + // RFC 9110 parameters admit an empty item after a semicolon. + if (cursor === text.length || text[cursor] === ";") continue; + const name = readToken()?.toLowerCase(); + if (name === undefined || text[cursor++] !== "=") return err("malformed"); + const value = text[cursor] === '"' ? readQuoted() : readToken(); + if (value === undefined) return err("malformed"); + // Repetition is ambiguous for this owner, even if both values agree. + // Keep parsing so syntax errors anywhere in the field cannot be discarded. + if (parameters.has(name)) ambiguous = true; + else parameters.set(name, value); + } + return ambiguous + ? err("ambiguous") + : ok({ + essence: type + "/" + subtype, + subtype, + charset: parameters.get("charset"), + parameters, + }); +}; +const httpErrorBody = ( + response: Response, + text: string, + redactor: IdaDiagnosticRedactor, +): string => { + const omitted = redactor.text( + "IDA MCP response body omitted: unsupported or unknown format or character encoding.", + ); + const contentType = response.headers.get("content-type"); + if (contentType === null) return omitted; + const mediaType = parseIdaContentType(contentType); + if (!mediaType.ok) return omitted; + const charset = mediaType.value.charset?.toLowerCase(); + if ( + charset !== undefined && + !["utf-8", "utf8", "us-ascii", "ascii"].includes(charset) + ) + return omitted; + if ( + mediaType.value.subtype === "json" || + mediaType.value.subtype.endsWith("+json") + ) + return redactor.json(text); + if (mediaType.value.essence === "text/plain") return redactor.text(text); + return omitted; }; +/** Keep error-body materialization within the diagnostic source-byte budget. */ +const readBoundedResponseText = async ( + response: Response, +): Promise => { + if (response.body === null) return ""; + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let total = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) return Buffer.concat(chunks, total).toString("utf8"); + total += value.byteLength; + if (total > IDA_DIAGNOSTIC_SOURCE_BYTES) { + await reader.cancel().catch(() => undefined); + return undefined; + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } +}; + +const diagnosticFetch = + (redactor: IdaDiagnosticRedactor): typeof fetch => + async (input, init) => { + const response = await fetch(input, init); + const readText = response.text.bind(response); + let safeBody: string | undefined; + if (!response.ok) { + // Normalize before the SDK can swallow a body-consumer failure or drop + // its format metadata. Both consumers use this one owned diagnostic. + const sourceBody = await readBoundedResponseText(response); + safeBody = + sourceBody === undefined + ? redactor.text( + `IDA MCP response body omitted: exceeds ${IDA_DIAGNOSTIC_SOURCE_BYTES} UTF-8 bytes.`, + ) + : httpErrorBody(response, sourceBody, redactor); + const body = safeBody; + Object.defineProperty(response, "text", { + value: async (): Promise => body, + }); + } + // Successful JSON remains lazy, and SSE keeps its original body stream. + Object.defineProperty(response, "json", { + value: async (): Promise => { + const text = safeBody ?? (await readText()); + try { + const value: unknown = JSON.parse(text); + return value; + } catch { + // Parsing failure owns diagnostics even when the HTTP status succeeded. + throw new IdaHttpJsonError( + safeBody ?? httpErrorBody(response, text, redactor), + redactor, + ); + } + }, + }); + return response; + }; + +const retainedTransportCause = (cause: unknown, message: string): Error => { + // Do not clone arbitrary upstream fields, stacks, data or nested causes. + if (cause instanceof SdkHttpError) + return new SdkHttpError(cause.code, message, { + status: cause.data.status, + statusText: "", + }); + if (cause instanceof ProtocolError) + return new ProtocolError(cause.code, message); + return new Error(message); +}; /** Create a standard SDK connection to an existing upstream registration. */ export const createIdaMcpConnection = ( config: IdaConfiguration, environment: Readonly, ): IdaMcpConnection => { const client = new Client({ name: "rea-ida-provider", version: "1" }); + const redactor = createIdaDiagnosticRedactor(config); let stderr = Buffer.alloc(0); let stderrBytes = 0; - const transportFailure = (cause: unknown): never => { - const failure = redactIdaTransportFailure(cause, config); - if (stderrBytes === 0) throw failure; + const transportFailure = ( + cause: unknown, + operation = "ida_mcp_response", + ): never => { + const bufferLimit = + cause instanceof Error + ? /ReadBuffer exceeded maximum size of (\d+) bytes/u.exec(cause.message) + : null; + if (cause instanceof IdaDiagnosticResourceError) throw cause; + let failure: Error; + if (cause instanceof IdaHttpJsonError) + failure = new Error(cause.message, { cause }); + else { + const message = + cause instanceof Error + ? cause.message + : "Unknown IDA MCP transport failure"; + const body = cause instanceof SdkHttpError ? cause.data.text : undefined; + // Only the pinned SDK POST producer places this owned body at a known + // span. Equal substrings elsewhere remain untrusted SDK context. + const prefix = "Error POSTing to endpoint: "; + const ownedPostBody = + cause instanceof SdkHttpError && + cause.code === SdkErrorCode.ClientHttpNotImplemented && + typeof body === "string" && + message === prefix + body; + // Normalized wire headers and supported decoded candidates have no LF. + // Separating owners prevents their concatenation from spelling a credential. + const safeMessage = ownedPostBody + ? redactor.message(prefix, body) + : redactor.blocks([message]); + failure = new Error(safeMessage, { + cause: retainedTransportCause(cause, safeMessage), + }); + } let start = 0; - // The byte tail may start inside a UTF-8 code point. Retain whole units. if (stderrBytes > stderr.length) while (start < stderr.length && (stderr[start]! & 0xc0) === 0x80) start++; - const diagnostic = redactIdaTransportFailure( - new Error(stderr.subarray(start).toString("utf8")), - config, - ).message; + const capturedOutput = { + stdout: "", + stderr: redactor.blocks([stderr.subarray(start).toString("utf8")]), + truncated: stderrBytes > stderr.length - start, + stdout_bytes: 0, + stderr_bytes: stderrBytes, + }; + if (bufferLimit !== null) + throw new AnalysisResourceConstraintError( + operation, + "transport", + `Upstream IDA MCP reply exceeded the ${bufferLimit[1]}-byte transport limit; no partial Evidence was returned.`, + { max_response_bytes: Number(bufferLimit[1]) }, + { cause, capturedOutput }, + ); + if (stderrBytes === 0) throw failure; throw new ProviderAdapterError("ida", "transport", { cause: failure, diagnostics: { reason: failure.message }, - capturedOutput: { - stdout: "", - stderr: diagnostic, - truncated: stderrBytes > stderr.length - start, - stdout_bytes: 0, - stderr_bytes: stderrBytes, - }, + capturedOutput, }); }; const transport = @@ -188,9 +417,11 @@ export const createIdaMcpConnection = ( ...snapshotEnvironment(config.env), }, stderr: "pipe", + maxBufferSize: IDA_STDIO_MAX_MESSAGE_BYTES, }) : new StreamableHTTPClientTransport(new URL(config.url), { requestInit: { headers: config.headers, redirect: "error" }, + fetch: diagnosticFetch(redactor), }); // Keep a bounded diagnostic tail for genuine failures; never persist ambient variables. if (transport instanceof StdioClientTransport) @@ -217,8 +448,9 @@ export const createIdaMcpConnection = ( return decodeIdaToolResult( await client .callTool({ name, arguments: args }, { timeout: config.timeoutMs }) - .catch(transportFailure), + .catch((cause: unknown) => transportFailure(cause, name)), config.protocol, + redactor, ); }, async readResource(uri) { diff --git a/src/ida/IdaMcpEnvironment.test.ts b/src/ida/IdaMcpEnvironment.test.ts index b744779ae8..5e7a746d0c 100644 --- a/src/ida/IdaMcpEnvironment.test.ts +++ b/src/ida/IdaMcpEnvironment.test.ts @@ -43,12 +43,23 @@ input.on('line', (line) => { HOME: "/selected-home", PATH: "/selected-path", }; + const registrationEnvironment: Record = { + REA_REGISTRATION_CONTEXT: "registration", + }; + if (process.platform === "win32") { + const systemRoot = process.env.SYSTEMROOT; + if (systemRoot === undefined || systemRoot.length === 0) + throw new Error( + "The Windows Node fixture requires its actual SYSTEMROOT.", + ); + registrationEnvironment.SYSTEMROOT = systemRoot; + } for (const selected of [environment, {}]) { const connection = createIdaMcpConnection( { command: process.execPath, args: [script], - env: { REA_REGISTRATION_CONTEXT: "registration" }, + env: registrationEnvironment, mode: "attached", timeoutMs: 5000, }, diff --git a/src/ida/IdaOperationFailures.test.ts b/src/ida/IdaOperationFailures.test.ts index 1f8a83c6b4..8886f68611 100644 --- a/src/ida/IdaOperationFailures.test.ts +++ b/src/ida/IdaOperationFailures.test.ts @@ -1,5 +1,7 @@ -import { rm } from "node:fs/promises"; +import { rm, stat } from "node:fs/promises"; +import { dirname } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { z } from "zod"; import { createIdaTarget, RecordingIdaMcp, @@ -7,6 +9,7 @@ import { import type { JsonValue } from "../domain/jsonValue.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { createIdaDiagnosticRedactor } from "./IdaDiagnosticRedactor.js"; import { decodeIdaToolResult } from "./IdaMcpConnection.js"; import type { AnalysisOperation } from "../application/AnalysisProvider.js"; import { IdaSessionClient } from "./IdaSessionClient.js"; @@ -17,6 +20,193 @@ afterEach(async () => { await rm(root, { recursive: true, force: true }); }); +it("preserves a rejected headless decompile diagnostic with completed lookup evidence and releases its owned worker", async () => { + const { root, target } = await createIdaTarget(); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + const reason = + "Invalid params: unexpected parameters: ['offset', 'max_lines']"; + const config = { + url: "http://127.0.0.1:12345/mcp", + headers: { + Authorization: "Token custom-fixture-token", + Cookie: "session=cookie-fixture-secret", + }, + mode: "headless" as const, + timeoutMs: 1000, + workspaceRoot: root, + }; + const redactor = createIdaDiagnosticRedactor(config); + producer.beforeCall = async (name) => { + if (name === "decompile") + decodeIdaToolResult( + { + isError: true, + content: [ + { + type: "text", + text: reason + " (custom-fixture-token / cookie-fixture-secret)", + }, + ], + }, + undefined, + redactor, + ); + }; + const safeReason = + reason + + " ([redacted transport credential] / [redacted transport credential])"; + const client = new IdaSessionClient(config, target, producer); + try { + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); + if (result.ok) throw new Error("Expected the upstream rejection"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "unreadable_output", + details: { + diagnostics: { reason: safeReason }, + partial_observation: { + provider: { id: "ida" }, + operation: "procedure_pseudo_code", + normalized_result: { + status: "partial", + observations: [expect.objectContaining({ tool: "lookup_funcs" })], + target_binding: { + status: "not_verified", + reason: "IDA MCP tool failed: \n" + safeReason, + }, + }, + }, + }, + }); + expect(JSON.stringify(projectAnalysisError(result.error))).not.toContain( + "custom-fixture-token", + ); + expect(JSON.stringify(projectAnalysisError(result.error))).not.toContain( + "cookie-fixture-secret", + ); + expect( + producer.calls.find(({ name }) => name === "decompile"), + ).toMatchObject({ + args: { addr: "0x1000", offset: 0, max_lines: 100 }, + }); + } finally { + const database = producer.database; + expect((await client.close()).ok).toBe(true); + expect(producer.calls.filter(({ name }) => name === "idb_close")).toEqual([ + { name: "idb_close", args: { database, save: false } }, + ]); + expect(producer.closes).toBe(1); + await expect(stat(dirname(producer.inputPath))).rejects.toMatchObject({ + code: "ENOENT", + }); + } +}); + +describe("headless IDA tool diagnostics in partial evidence", () => { + const callerPath = "/caller/input?token=visible"; + const marker = "[redacted transport credential]"; + it.each([ + { + name: "a decoded Basic password matching the tool failure prefix", + password: "failed", + upstream: "Rejected decompile request for " + callerPath, + safeReason: "Rejected decompile request for " + callerPath, + forbiddenInBinding: ["failed"], + }, + { + name: "a JSON-encoded tool explanation", + password: "", + upstream: String.raw`{"reason":"\u003csecret\u003e","user":"user","path":"/caller/input?token=visible"}`, + safeReason: + '{"reason":"' + + marker + + '","user":"user","path":"/caller/input?token=visible"}', + forbiddenInBinding: ["", String.raw`\u003csecret\u003e`], + }, + ])( + "redacts $name before retaining the public target-binding reason", + async ({ password, upstream, safeReason, forbiddenInBinding }) => { + const { root, target } = await createIdaTarget(); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + const config = { + url: "http://127.0.0.1:12345/mcp", + headers: { + Authorization: + "Basic " + + Buffer.from("user:" + password, "utf8").toString("base64"), + }, + mode: "headless" as const, + timeoutMs: 1000, + workspaceRoot: root, + }; + const redactor = createIdaDiagnosticRedactor(config); + producer.beforeCall = async (name) => { + if (name === "decompile") + decodeIdaToolResult( + { isError: true, content: [{ type: "text", text: upstream }] }, + undefined, + redactor, + ); + }; + const client = new IdaSessionClient(config, target, producer); + try { + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); + if (result.ok) throw new Error("Expected the producer's tool error"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "unreadable_output", + details: { + partial_observation: { + provider: { id: "ida" }, + operation: "procedure_pseudo_code", + normalized_result: { + status: "partial", + observations: [ + expect.objectContaining({ tool: "lookup_funcs" }), + ], + target_binding: { + status: "not_verified", + reason: expect.any(String), + }, + }, + }, + }, + }); + const details = z + .object({ + diagnostics: z.object({ reason: z.string() }), + partial_observation: z.object({ + normalized_result: z.object({ + target_binding: z.object({ reason: z.string() }), + }), + }), + }) + .parse(projected.details); + const bindingReason = + details.partial_observation.normalized_result.target_binding.reason; + for (const credential of forbiddenInBinding) + expect.soft(bindingReason).not.toContain(credential); + expect.soft(bindingReason).toContain(marker); + expect(bindingReason).toContain(callerPath); + expect.soft(details.diagnostics.reason).toBe(safeReason); + } finally { + expect((await client.close()).ok).toBe(true); + expect( + producer.calls.filter(({ name }) => name === "idb_close"), + ).toHaveLength(1); + await expect(stat(dirname(producer.inputPath))).rejects.toMatchObject({ + code: "ENOENT", + }); + } + }, + ); +}); + describe("headless IDA partial operation evidence", () => { it("retains completed rows when post-operation target identity changes", async () => { const { root, target } = await createIdaTarget(); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index 576dac1e71..252bdbe3a1 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -311,14 +311,12 @@ export class IdaOperationRunner { const current: number = offset; if (this.protocol === "legacy") { const page = legacyPageSchema.parse( - await this.call("list_strings", { offset, count: 100 }), - ); - result.push( - ...page.data.map((item) => { - const s = legacyStringSchema.parse(item); - return { address: s.address, value: s.string }; - }), + await this.call("list_strings", { offset, count: 0 }), ); + for (const item of page.data) { + const s = legacyStringSchema.parse(item); + result.push({ address: s.address, value: s.string }); + } offset = page.next_offset; } else { const page = ( diff --git a/src/ida/IdaProvider.ts b/src/ida/IdaProvider.ts index 5ce924fcd1..d2bf970600 100644 --- a/src/ida/IdaProvider.ts +++ b/src/ida/IdaProvider.ts @@ -68,13 +68,17 @@ export class IdaProvider implements AnalysisProviderCandidate { if (!privacy.available) return { status: "unavailable", - code: "unsupported_host", + code: privacy.code, reason: privacy.reason, diagnostics: { configured: true, mode: "headless", live_connection_probed: false, private_workspace_available: false, + windows_native_failure: { + stage: privacy.failureStage, + remediation: privacy.remediation, + }, }, }; } diff --git a/src/ida/IdaProviderCapabilities.ts b/src/ida/IdaProviderCapabilities.ts index 0ccc016d25..1394310d93 100644 --- a/src/ida/IdaProviderCapabilities.ts +++ b/src/ida/IdaProviderCapabilities.ts @@ -32,7 +32,6 @@ export type IdaOperation = (typeof IDA_OPERATIONS)[number]; /** Operations only the modern headless profile implements. */ const HEADLESS_ONLY_OPERATIONS: ReadonlySet = new Set([ "resolve_containing_procedure", - "list_documents", ]); /** Whether the configured profile admits an operation. */ diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index 1a35dd7c78..3d0feeed2e 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -1,6 +1,6 @@ import { readFile, readdir, rm } from "node:fs/promises"; import { sep } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { z } from "zod"; import { createIdaTarget, @@ -11,11 +11,34 @@ import { functionDossierSchema } from "../domain/hopperValues.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { WindowsPrivateRuntime } from "../windows/WindowsPrivateRuntime.js"; const roots: string[] = []; +const windowsRuntimes = new Set(); +beforeEach(() => { + if (process.platform !== "win32") return; + const create = WindowsPrivateRuntime.create; + vi.spyOn(WindowsPrivateRuntime, "create").mockImplementation( + (parent, prefix) => { + const runtime = create(parent, prefix); + windowsRuntimes.add(runtime); + return runtime; + }, + ); +}); afterEach(async () => { - for (const root of roots.splice(0)) - await rm(root, { recursive: true, force: true }); + try { + // These pure producer fixtures spawn no IDA worker. After the retention + // assertions, release only the actual native owners this test created. + for (const runtime of windowsRuntimes) { + await runtime.close(); + windowsRuntimes.delete(runtime); + } + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); + } finally { + vi.restoreAllMocks(); + } }); /** A failed close whose diagnostic names exactly these unverified sessions. */ const reportsSessions = (...sessions: readonly object[]) => ({ @@ -1495,6 +1518,38 @@ it("binds retained transport diagnostics to the requested operation and recovery await client.close(); }); +describe("attached IDA legacy string inventory", () => { + it("fetches legacy strings and literal searches in single upstream calls", async () => { + const { producer, client } = await fixture(); + const all = Array.from({ length: 250 }, (_, i) => ({ + address: `0x${(0x3000 + i).toString(16)}`, + length: 5, + string: `hello-${i}`, + })); + producer.overrideCall = (name, args) => { + if (name !== "list_strings") return undefined; + const offset = Number(args.offset ?? 0); + const count = Number(args.count ?? 100); + const data = count === 0 ? all : all.slice(offset, offset + count); + return { + data, + next_offset: + count === 0 || offset + count >= all.length ? null : offset + count, + }; + }; + const listed = await client.execute("list_strings", {}); + expect(listed.ok && listed.value.result).toHaveLength(250); + expect( + producer.calls.filter(({ name }) => name === "list_strings")[0]?.args + .count, + ).toBe(0); + expect( + producer.calls.filter(({ name }) => name === "list_strings"), + ).toHaveLength(1); + await client.close(); + }); +}); + it("retains transport diagnostics on a cancellation regardless of stderr", async () => { const { producer, client } = await fixture("attached"); await client.execute("health", {}); diff --git a/src/jeb/JebMcpEndpoint.test.ts b/src/jeb/JebMcpEndpoint.test.ts index 0252632d86..d07ad0d586 100644 --- a/src/jeb/JebMcpEndpoint.test.ts +++ b/src/jeb/JebMcpEndpoint.test.ts @@ -5,6 +5,7 @@ import { parseJebMcpEndpoint } from "./JebMcpEndpoint.js"; describe("JEB MCP endpoint", () => { it("rejects an explicit empty query or fragment instead of dropping it", () => { expect(parseJebMcpEndpoint("http://127.0.0.1:8425/mcp").ok).toBe(true); + expect(parseJebMcpEndpoint("http://127.0.0.1:8425/mcp/").ok).toBe(true); for (const value of [ "http://127.0.0.1:8425/mcp?", "http://127.0.0.1:8425/mcp#", @@ -12,4 +13,12 @@ describe("JEB MCP endpoint", () => { ]) expect(parseJebMcpEndpoint(value).ok, value).toBe(false); }); + + it("accepts only the declared MCP route", () => { + expect(parseJebMcpEndpoint("http://127.0.0.1:8425/mcp").ok).toBe(true); + for (const path of ["/mcp/extra", "/not-mcp", "/"]) + expect(parseJebMcpEndpoint(`http://127.0.0.1:8425${path}`).ok, path).toBe( + false, + ); + }); }); diff --git a/src/jeb/JebMcpEndpoint.ts b/src/jeb/JebMcpEndpoint.ts index bae4a00a9c..97bca88f0f 100644 --- a/src/jeb/JebMcpEndpoint.ts +++ b/src/jeb/JebMcpEndpoint.ts @@ -47,6 +47,12 @@ export const parseJebMcpEndpoint = ( "Embedding credentials in the JEB MCP endpoint is not supported; configure authentication with the engine", }), ); + if (url.pathname !== "/mcp" && url.pathname !== "/mcp/") + return err( + new AnalysisInputError("jeb_mcp_endpoint", { + cause: "JEB MCP endpoint must use the /mcp route", + }), + ); if (urlDeclaresQueryOrFragment(url)) return err( new AnalysisInputError("jeb_mcp_endpoint", { diff --git a/src/wasm/WabtArtifactProvider.test.ts b/src/wasm/WabtArtifactProvider.test.ts index 7d1eca0f7d..e7dc2a515d 100644 --- a/src/wasm/WabtArtifactProvider.test.ts +++ b/src/wasm/WabtArtifactProvider.test.ts @@ -544,6 +544,49 @@ it("reports WABT output beyond the budget as a resource constraint with a bounde signal: "SIGKILL", }); }); +it("does not attribute wasm-objdump output to a wasm2wat launch failure", async () => { + const selectedRoot = await createTestTempDirectory("rea-wabt-launch-"); + const path = join(selectedRoot, "selected.wasm"); + await writeFile(path, Buffer.alloc(76, 1)); + for (const tool of ["wasm-validate", "wasm-objdump", "wasm2wat"]) + await writeFile( + join(selectedRoot, tool + (process.platform === "win32" ? ".exe" : "")), + "source-owned executable seam", + { mode: 0o700 }, + ); + const dump = await readFile( + new URL("../../tests/fixtures/wasm/objdump.txt", import.meta.url), + "utf8", + ); + const provider = new WabtArtifactProvider( + { REA_WABT_BIN_DIRECTORY: selectedRoot }, + (input) => { + const version = input.arguments[0] === "--version"; + if (!version && input.command.includes("wasm2wat")) + throw Object.assign(new Error("spawn E2BIG"), { code: "E2BIG" }); + const script = version + ? 'process.stdout.write("1.0.42\\n")' + : input.command.includes("objdump") + ? `process.stdout.write(${JSON.stringify(dump)})` + : ""; + return spawnOwnedProviderProcess({ + ...input, + command: process.execPath, + arguments: ["-e", script], + expectedCommand: null, + }); + }, + ); + const result = await provider.inspect({ + path, + glue_paths: [], + candidate_paths: [], + }); + if (result.ok) throw new Error("Expected the wasm2wat launch to fail"); + expect(result.error.message).toContain("spawn E2BIG"); + expect(result.error.capturedOutput).toBeUndefined(); +}); + it("bounds an oversized stderr to an excerpt", () => { expect( projectAnalysisError( diff --git a/src/wasm/WabtArtifactProvider.ts b/src/wasm/WabtArtifactProvider.ts index d706c97e83..fae455c244 100644 --- a/src/wasm/WabtArtifactProvider.ts +++ b/src/wasm/WabtArtifactProvider.ts @@ -180,6 +180,7 @@ export class WabtArtifactProvider implements WasmArtifactPort { args: readonly string[], acceptNonZeroExit = false, ) => { + capturedOutput = undefined; await verifySnapshot(); const snapshot = await runOwnedCommand( { diff --git a/src/windows/WindowsNativeLoader.test.ts b/src/windows/WindowsNativeLoader.test.ts index c81e8e9a8d..9f2b5623d4 100644 --- a/src/windows/WindowsNativeLoader.test.ts +++ b/src/windows/WindowsNativeLoader.test.ts @@ -111,24 +111,28 @@ describe("packaged Windows native admission", () => { }); it.each([ - { platform: "linux" as const }, - { architecture: "arm64" }, - { nodeApiVersion: "7" }, - { nodeApiVersion: undefined }, - { packageVersion: "2.0.0" }, - ])("rejects incompatible host metadata before loading: %j", (override) => { - const value = fixture(); - expect( - loadWindowsNativeAuthority({ ...value.host, ...override }).available, - ).toBe(false); - expect(value.loads()).toBe(0); - }); + [{ platform: "linux" as const }, "host"], + [{ architecture: "arm64" }, "host"], + [{ nodeApiVersion: "7" }, "host"], + [{ nodeApiVersion: undefined }, "host"], + [{ packageVersion: "2.0.0" }, "artifact"], + ] as const)( + "rejects incompatible host metadata before loading: %j", + (override, failureStage) => { + const value = fixture(); + expect( + loadWindowsNativeAuthority({ ...value.host, ...override }), + ).toMatchObject({ available: false, failureStage }); + expect(value.loads()).toBe(0); + }, + ); it("rejects tampering even when the original file was a valid PE image", () => { const value = fixture(); value.bytes[127] = 1; expect(loadWindowsNativeAuthority(value.host)).toMatchObject({ available: false, + failureStage: "artifact", reason: expect.stringContaining("SHA-256"), }); expect(value.loads()).toBe(0); @@ -190,6 +194,7 @@ describe("packaged Windows native admission", () => { }), ).toMatchObject({ available: false, + failureStage: "self-test", retryable: true, reason: expect.stringContaining("private DACL: access denied"), }); @@ -202,6 +207,7 @@ describe("packaged Windows native admission", () => { loadArtifact: () => ({ call: () => ({ privateDacl: true }) }), }); expect(loaded.available).toBe(false); + expect(loaded).toMatchObject({ failureStage: "inspection" }); expect(loaded).not.toHaveProperty("retryable"); }); diff --git a/src/windows/WindowsNativeLoader.ts b/src/windows/WindowsNativeLoader.ts index 13bf4e9f3e..37620f53a2 100644 --- a/src/windows/WindowsNativeLoader.ts +++ b/src/windows/WindowsNativeLoader.ts @@ -71,6 +71,7 @@ export type WindowsNativeLoadResult = | { readonly available: false; readonly reason: string; + readonly failureStage: "host" | "artifact" | "self-test" | "inspection"; /** Only a failed native invocation can recover on a later query. */ readonly retryable?: true; }; @@ -93,15 +94,20 @@ export const loadWindowsNativeAuthority = ( if (host.platform !== "win32" || host.architecture !== "x64") return { available: false, + failureStage: "host", reason: "REA Windows native controls require a Windows x64 host.", }; + const nodeApiVersion = Number(host.nodeApiVersion); + if (!Number.isSafeInteger(nodeApiVersion) || nodeApiVersion < 8) + return { + available: false, + failureStage: "host", + reason: "REA Windows native controls require Node-API 8 or newer.", + }; try { const manifest = windowsNativeManifestSchema.parse(host.readManifest()); if (manifest.packageVersion !== host.packageVersion) throw new Error("Windows native artifact belongs to another REA version"); - const nodeApiVersion = Number(host.nodeApiVersion); - if (!Number.isSafeInteger(nodeApiVersion) || nodeApiVersion < 8) - throw new Error("Windows native artifact requires Node-API 8 or newer"); const artifact = host.readArtifact(); if ( createHash("sha256").update(artifact).digest("hex") !== @@ -118,15 +124,24 @@ export const loadWindowsNativeAuthority = ( } catch (cause: unknown) { return { available: false, + failureStage: "self-test", retryable: true, reason: `REA Windows native self-test failed: ${cause instanceof Error ? cause.message : String(cause)}`, }; } - const inspection = inspectionSchema.parse(observed); + const parsed = inspectionSchema.safeParse(observed); + if (!parsed.success) + return { + available: false, + failureStage: "inspection", + reason: `REA Windows native inspection is invalid: ${parsed.error.message}`, + }; + const inspection = parsed.data; return { available: true, authority: { call: binding.call, inspection } }; } catch (cause: unknown) { return { available: false, + failureStage: "artifact", reason: `REA Windows native controls are unavailable: ${cause instanceof Error ? cause.message : String(cause)}`, }; } diff --git a/tests/boundary/filesystem/ghidraJavaProbe.test.ts b/tests/boundary/filesystem/ghidraJavaProbe.test.ts index 415fbf3ddb..d48dc1ece9 100644 --- a/tests/boundary/filesystem/ghidraJavaProbe.test.ts +++ b/tests/boundary/filesystem/ghidraJavaProbe.test.ts @@ -88,6 +88,109 @@ it.each([ }, ); +it.each([ + { label: "omitted", property: "", shown: "bitness unknown" }, + { + label: "empty", + property: "sun.arch.data.model =\n", + shown: 'bitness unknown (sun.arch.data.model = "")', + }, + { + label: "malformed", + property: "sun.arch.data.model = 64junk\n", + shown: 'bitness unknown (sun.arch.data.model = "64junk")', + }, + { + label: "not exactly representable", + property: "sun.arch.data.model = 9007199254740993\n", + shown: 'bitness unknown (sun.arch.data.model = "9007199254740993")', + }, +])("keeps $label Java bitness unresolved", async ({ property, shown }) => { + const root = await createTestTempDirectory("rea-java-bitness-"); + const configuredHome = join(root, "selected-jdk"); + const reportedHome = join(root, "reported-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await mkdir(join(reportedHome, "bin"), { recursive: true }); + await writeFile( + join( + reportedHome, + "bin", + process.platform === "win32" ? "javac.exe" : "javac", + ), + "fixture", + { + mode: 0o755, + }, + ); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: ` java.version = 21.0.11\n java.home = ${reportedHome}\n${property}`, + status: 0, + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + expect(inspection.status).toBe("unavailable"); + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "version_unresolved", + detail: expect.stringContaining(shown), + }); + expect( + inspection.checks.find(({ name }) => name === "java")?.detail, + ).not.toMatch(/(?:0-bit|9007199254740992-bit)/); +}); + +it("keeps a known 32-bit Java report classified as unsupported", async () => { + const root = await createTestTempDirectory("rea-java-bitness-known-"); + const configuredHome = join(root, "selected-jdk"); + const reportedHome = join(root, "reported-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await mkdir(join(reportedHome, "bin"), { recursive: true }); + await writeFile( + join( + reportedHome, + "bin", + process.platform === "win32" ? "javac.exe" : "javac", + ), + "fixture", + { + mode: 0o755, + }, + ); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: ` java.version = 21.0.11\n java.home = ${reportedHome}\n sun.arch.data.model = 32\n`, + status: 0, + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "unsupported_version", + detail: expect.stringContaining("32-bit"), + }); +}); + it("bounds the Java version probe and reports an over-budget probe as a failed Java check", async () => { const root = await createTestTempDirectory("rea-java-probe-bound-"); const configuredHome = join(root, "selected-jdk"); diff --git a/tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts b/tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts new file mode 100644 index 0000000000..8555c1dbdc --- /dev/null +++ b/tests/boundary/filesystem/ghidraLanguageCatalogBound.test.ts @@ -0,0 +1,86 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; + +import { readGhidraLanguageCatalog } from "../../../src/ghidra/GhidraLanguageCatalog.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const LIMIT = 4 * 1024 * 1024; +const OVERSIZED = "oversized.ldefs"; + +// Count bytes delivered from the oversized definition, whichever read API is used. +const { readBytes } = vi.hoisted(() => ({ readBytes: { value: 0 } })); +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + const tracked = (path: unknown) => String(path).endsWith(OVERSIZED); + return { + ...actual, + readFile: async (...args: Parameters) => { + const bytes = await actual.readFile(...args); + if (tracked(args[0])) readBytes.value += bytes.length; + return bytes; + }, + open: async (...args: Parameters) => { + const handle = await actual.open(...args); + if (!tracked(args[0])) return handle; + const read = handle.read.bind(handle) as ( + ...readArgs: unknown[] + ) => Promise<{ bytesRead: number }>; + return Object.assign(handle, { + read: async (...readArgs: unknown[]) => { + const result = await read(...readArgs); + readBytes.value += result.bytesRead; + return result; + }, + }); + }, + }; +}); + +afterEach(() => { + readBytes.value = 0; +}); + +it("skips an oversized definition without buffering it beyond the catalog limit", async () => { + const root = await createTestTempDirectory("rea-ghidra-ldefs-bound-"); + const module = (name: string) => + join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(module("huge"), { recursive: true }); + await mkdir(module("x86"), { recursive: true }); + await writeFile( + join(module("huge"), OVERSIZED), + Buffer.concat([ + Buffer.from( + '', + ), + Buffer.alloc(2 * LIMIT, 0x20), + ]), + ); + await writeFile( + join(module("x86"), "x86.ldefs"), + '', + ); + + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "x86:LE:32:default", compilerSpecIds: [] }, + ]); + expect(readBytes.value).toBeGreaterThan(0); + expect(readBytes.value).toBeLessThanOrEqual(LIMIT + 1); +}); + +it("keeps a definition at exactly the catalog limit", async () => { + const root = await createTestTempDirectory("rea-ghidra-ldefs-limit-"); + const languages = join(root, "Ghidra/Processors/edge/data/languages"); + await mkdir(languages, { recursive: true }); + const document = Buffer.from( + '', + ); + await writeFile( + join(languages, "edge.ldefs"), + Buffer.concat([document, Buffer.alloc(LIMIT - document.length, 0x20)]), + ); + + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "edge:LE:32:default", compilerSpecIds: [] }, + ]); +}); diff --git a/tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts b/tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts new file mode 100644 index 0000000000..c009837658 --- /dev/null +++ b/tests/boundary/filesystem/ghidraLanguageCatalogCleanup.test.ts @@ -0,0 +1,203 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; + +import { + GhidraLanguageCatalogCleanupError, + readGhidraLanguageCatalog, +} from "../../../src/ghidra/GhidraLanguageCatalog.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +interface FileControl { + closeFails: boolean; + /** Hold close() until every synchronized handle has reached it. */ + synchronizedClose: boolean; + invalidateOnFailure: boolean; + readFails: boolean; + invalidated: boolean; + closeCalls: number; +} + +// Per-file failure controls, keyed by definition file name, and a barrier +// that releases synchronized closes only after all of them are pending. +const { controls, barrier } = vi.hoisted(() => { + const state = { expected: 0, arrived: 0, release: () => {} }; + const gate = new Promise((resolve) => { + state.release = resolve; + }); + return { + controls: new Map(), + barrier: { + state, + arrive: async () => { + state.arrived += 1; + if (state.arrived >= state.expected) state.release(); + await gate; + }, + }, + }; +}); +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + const real = await actual.open(...args); + const control = controls.get(basename(String(args[0]))); + if (control === undefined) return real; + let released = false; + const release = async () => { + if (released) return; + released = true; + await real.close(); + }; + return { + get fd() { + return control.invalidated ? -1 : real.fd; + }, + read: (...readArgs: Parameters) => + control.readFails + ? Promise.reject(new Error("definition read failed")) + : real.read(...readArgs), + close: async () => { + control.closeCalls += 1; + if (control.synchronizedClose && control.closeCalls === 1) + await barrier.arrive(); + if (control.closeFails) { + if (control.invalidateOnFailure) { + await release(); + control.invalidated = true; + } + throw new Error(`definition close failed: ${String(args[0])}`); + } + await release(); + }, + }; + }, + }; +}); + +let preserveUnconfirmedOwner = false; + +const control = (name: string, overrides: Partial = {}) => { + const value: FileControl = { + closeFails: false, + synchronizedClose: false, + invalidateOnFailure: false, + readFails: false, + invalidated: false, + closeCalls: 0, + ...overrides, + }; + controls.set(name, value); + return value; +}; + +const installation = async (name: string): Promise => { + const root = await createTestTempDirectory("rea-ghidra-ldefs-cleanup-"); + const languages = join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(languages, { recursive: true }); + await writeFile( + join(languages, `${name}.ldefs`), + ``, + ); + return root; +}; + +// Let every retained owner close, then drain them through a healthy read. +afterEach(async () => { + if (preserveUnconfirmedOwner) { + preserveUnconfirmedOwner = false; + controls.clear(); + return; + } + for (const value of controls.values()) value.closeFails = false; + await readGhidraLanguageCatalog(await installation("drain")); + controls.clear(); +}); + +it("retries a retained close before the next read", async () => { + const retained = control("flaky.ldefs", { closeFails: true }); + await expect( + readGhidraLanguageCatalog(await installation("flaky")), + ).rejects.toMatchObject({ paths: [expect.stringContaining("flaky.ldefs")] }); + retained.closeFails = false; + expect(await readGhidraLanguageCatalog(await installation("x86"))).toEqual([ + { id: "x86:LE:32:default", compilerSpecIds: [] }, + ]); + expect(retained.closeCalls).toBe(2); +}); + +it("attempts every retained close and reports the remaining failures together", async () => { + const first = control("first.ldefs", { + closeFails: true, + synchronizedClose: true, + }); + const second = control("second.ldefs", { + closeFails: true, + synchronizedClose: true, + }); + barrier.state.expected = 2; + // Concurrent resolutions can each retain an owner. + const roots = [await installation("first"), await installation("second")]; + const results = await Promise.allSettled( + roots.map((root) => readGhidraLanguageCatalog(root)), + ); + expect(results.map(({ status }) => status)).toEqual(["rejected", "rejected"]); + second.closeFails = false; + const failure = await readGhidraLanguageCatalog( + await installation("x86"), + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(failure).toMatchObject({ + paths: [expect.stringContaining("first.ldefs")], + }); + expect(second.closeCalls).toBe(2); + expect(first.closeCalls).toBe(2); +}); + +it("keeps the read failure when the close also fails", async () => { + control("damaged.ldefs", { readFails: true, closeFails: true }); + const failure = await readGhidraLanguageCatalog( + await installation("damaged"), + ).catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(failure).toMatchObject({ + readFailure: expect.objectContaining({ message: "definition read failed" }), + }); +}); + +it("reports retained cleanup for a catalog without definitions", async () => { + control("pending.ldefs", { closeFails: true }); + await expect( + readGhidraLanguageCatalog(await installation("pending")), + ).rejects.toBeInstanceOf(GhidraLanguageCatalogCleanupError); + const empty = await createTestTempDirectory("rea-ghidra-ldefs-empty-"); + await expect(readGhidraLanguageCatalog(empty)).rejects.toMatchObject({ + paths: [expect.stringContaining("pending.ldefs")], + }); +}); + +it("retains an invalidated descriptor owner and keeps later reads blocked", async () => { + const broken = control("broken.ldefs", { + closeFails: true, + invalidateOnFailure: true, + }); + const first = await readGhidraLanguageCatalog( + await installation("broken"), + ).catch((cause: unknown) => cause); + expect(first).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(first).toMatchObject({ + paths: [expect.stringContaining("broken.ldefs")], + }); + + const later = await readGhidraLanguageCatalog( + await installation("x86"), + ).catch((cause: unknown) => cause); + expect(later).toBeInstanceOf(GhidraLanguageCatalogCleanupError); + expect(later).toMatchObject({ + paths: [expect.stringContaining("broken.ldefs")], + }); + expect(broken.closeCalls).toBe(1); + preserveUnconfirmedOwner = true; +}); diff --git a/tests/boundary/filesystem/mipsAbiFlags.test.ts b/tests/boundary/filesystem/mipsAbiFlags.test.ts index 1b724d31bd..e89f5cdbff 100644 --- a/tests/boundary/filesystem/mipsAbiFlags.test.ts +++ b/tests/boundary/filesystem/mipsAbiFlags.test.ts @@ -156,7 +156,10 @@ for (const little of [true, false]) { else bytes.writeUInt32BE(sectionOffset, 32); const resolved = await resolveFixture(bytes); assert.equal(resolved.mips.abiFlags?.fpAbi, 5); - assert.equal(ghidraMipsUnsupportedReason(resolved), null); + assert.match( + ghidraMipsUnsupportedReason(resolved) ?? "", + /ABI declaration inspection is incomplete/u, + ); assert.match( resolved.mips.limitations?.join(" ") ?? "", /section header table/u, @@ -172,7 +175,7 @@ it("keeps declarations in uncaptured section headers unknown when program ABI da assert.equal(resolved.mips.abiFlags, null); assert.match( ghidraMipsUnsupportedReason(resolved) ?? "", - /inspected ABI.*unavailable headers remain unknown/u, + /ABI declaration inspection is incomplete.*unavailable headers remain unknown/u, ); }); for (const failure of ["conflicting", "outside", "duplicate"]) { @@ -370,6 +373,26 @@ it("projects a malformed out-of-probe ABI record as a target error", async () => assert.deepEqual(await readFile(path), bytes); }); +for (const little of [true, false]) { + it(`refuses GNU ABI attributes by section type without reading attacker-controlled names or sizes (${little ? "LE" : "BE"})`, async () => { + const bytes = fixture(little); + const write16 = little + ? (value: number, offset: number) => bytes.writeUInt16LE(value, offset) + : (value: number, offset: number) => bytes.writeUInt16BE(value, offset); + const write32 = little + ? (value: number, offset: number) => bytes.writeUInt32LE(value, offset) + : (value: number, offset: number) => bytes.writeUInt32BE(value, offset); + write32(0xffffffff, 168); // Unknown name; the section type is authoritative. + write32(0x6ffffff5, 172); // SHT_GNU_ATTRIBUTES. + write32(0xfffffff0, 184); // Must not be read as a payload offset. + write32(0xffffffff, 188); // Must not control allocation. + write16(0, 50); // No section-name string table is required for detection. + const result = await inspect(bytes, little); + assert.ok(!result.ok); + assert.match(result.error, /GNU ABI attributes that REA does not inspect/u); + }); +} + it("preserves cancellation instead of turning it into malformed target data", async () => { const directory = await createTestTempDirectory("rea-mips-resolution-"); const path = join(directory, "fixture.elf"); diff --git a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts index e786fc3ccc..f7643f3377 100644 --- a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts +++ b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts @@ -175,7 +175,8 @@ it("retains terminal-call qualifications across function views and snapshot repl }); expect(evidence.limitations, name).toContain(limitation); expect(evidence.analysis_profile?.parameters).toMatchObject({ - function_boundary_observations: "ghidra-terminal-call-limitations-v2", + function_boundary_observations: + "ghidra-function-boundary-limitations-v3", }); const replay = await harness.session.execute(name, evidence.parameters); if (!replay.ok) throw replay.error; diff --git a/tests/boundary/mcp/idaTransport.test.ts b/tests/boundary/mcp/idaTransport.test.ts index 4a43bdb2e1..2844d8eca2 100644 --- a/tests/boundary/mcp/idaTransport.test.ts +++ b/tests/boundary/mcp/idaTransport.test.ts @@ -1,9 +1,46 @@ import { createServer } from "node:http"; -import { afterEach, expect, it } from "vitest"; +import { inspect } from "node:util"; +import { ProtocolError } from "@modelcontextprotocol/client"; +import { afterEach, expect, it, vi } from "vitest"; +import { constants } from "node:buffer"; +import { IdaDiagnosticResourceError } from "../../../src/ida/IdaDiagnosticWriter.js"; import { z } from "zod"; +import { AnalysisProtocolError } from "../../../src/domain/analysisErrorCore.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; import { createIdaMcpConnection } from "../../../src/ida/IdaMcpConnection.js"; import type { IdaMcpConnection } from "../../../src/ida/IdaMcpConnection.js"; +// Inject an owned capacity failure at the producer while retaining the real +// SDK. Small writer-limit tests separately exercise actual expansion checks. +vi.mock("../../../src/ida/IdaDiagnosticRedactor.js", async (importOriginal) => { + const actual = + await importOriginal< + typeof import("../../../src/ida/IdaDiagnosticRedactor.js") + >(); + return { + ...actual, + createIdaDiagnosticRedactor: ( + config: Parameters[0], + ) => { + const redactor = actual.createIdaDiagnosticRedactor(config); + return { + ...redactor, + text: redactor.text, + json: (message: string) => { + if (message === "fixture-resource-boundary") + throw new IdaDiagnosticResourceError( + message.length, + constants.MAX_STRING_LENGTH + 1, + ); + return redactor.json(message); + }, + }; + }, + }; +}); + const cleanups: (() => Promise)[] = []; afterEach(async () => { for (const cleanup of cleanups.splice(0).reverse()) await cleanup(); @@ -19,7 +56,36 @@ const requestSchema = z.object({ .passthrough() .optional(), }); -const fixture = async (redirectUrl?: string, cycle = false, native = false) => { +type IdaTransportFixtureOptions = { + readonly redirectUrl?: string; + readonly cycle?: boolean; + readonly native?: boolean; + readonly toolResult?: JsonValue; + readonly resourceResponseText?: string; + readonly toolRpcError?: { + readonly code: number; + readonly message: string; + readonly data?: JsonValue; + }; + readonly toolStatus?: number; + readonly toolStatusText?: string; + readonly toolContentType?: string; + readonly toolResponseBody?: string | Buffer; + readonly configurationHeaders?: Readonly>; +}; +const fixture = async ({ + redirectUrl, + cycle = false, + native = false, + toolResult, + resourceResponseText, + toolRpcError, + toolStatus = 200, + toolStatusText, + toolContentType = "application/json", + toolResponseBody, + configurationHeaders = { Authorization: "Bearer transport-fixture-secret" }, +}: IdaTransportFixtureOptions = {}) => { const methods: string[] = []; const headers: (string | undefined)[] = []; const server = createServer(async (request, response) => { @@ -78,28 +144,47 @@ const fixture = async (redirectUrl?: string, cycle = false, native = false) => { { uri: "ida://idb/metadata", mimeType: "application/json", - text: JSON.stringify({ - path: "/fixture/input.i64", - sha256: "a".repeat(64), - base: "0x1000", - module: "input", - }), + text: + resourceResponseText ?? + JSON.stringify({ + path: "/fixture/input.i64", + sha256: "a".repeat(64), + base: "0x1000", + module: "input", + }), }, ], } - : native - ? { - content: [ - { - type: "text", - text: JSON.stringify([{ observed: true }]), - }, - ], - structuredContent: { result: [], _output_truncated: true }, - } - : { content: [], structuredContent: { observed: true } }; - response.writeHead(200, { "Content-Type": "application/json" }); - response.end(JSON.stringify({ jsonrpc: "2.0", id: input.id, result })); + : toolResult !== undefined + ? toolResult + : native + ? { + content: [ + { + type: "text", + text: JSON.stringify([{ observed: true }]), + }, + ], + structuredContent: { result: [], _output_truncated: true }, + } + : { content: [], structuredContent: { observed: true } }; + if (input.method === "tools/call" && toolStatusText !== undefined) + response.statusMessage = toolStatusText; + response.writeHead(input.method === "tools/call" ? toolStatus : 200, { + "Content-Type": + input.method === "tools/call" ? toolContentType : "application/json", + }); + response.end( + input.method === "tools/call" && toolResponseBody !== undefined + ? toolResponseBody + : JSON.stringify({ + jsonrpc: "2.0", + id: input.id, + ...(input.method === "tools/call" && toolRpcError !== undefined + ? { error: toolRpcError } + : { result }), + }), + ); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); cleanups.push( @@ -117,7 +202,7 @@ const fixture = async (redirectUrl?: string, cycle = false, native = false) => { const connection: IdaMcpConnection = createIdaMcpConnection( { url: `http://127.0.0.1:${address.port}/mcp`, - headers: { Authorization: "Bearer transport-fixture-secret" }, + headers: configurationHeaders, mode: native ? "attached" : "headless", ...(native ? { protocol: "native" as const } : {}), timeoutMs: 2000, @@ -148,15 +233,30 @@ it("uses the pinned SDK for a local Streamable HTTP handshake, paged discovery, ).toBe(true); }); +it("omits an oversized HTTP error body before retaining or redacting it", async () => { + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + toolResponseBody: "owned-diagnostic-" + "x".repeat(70 * 1024), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("response body omitted"); + expect(failure.message).not.toContain("owned-diagnostic-"); +}); + it("rejects redirects instead of forwarding transport credentials to another endpoint", async () => { const destination = await fixture(); - const { connection } = await fixture(destination.url); + const { connection } = await fixture({ redirectUrl: destination.url }); await expect(connection.connect()).rejects.toThrow(); expect(destination.headers).toEqual([]); }); it("rejects nonterminating SDK pagination instead of returning a partial tool catalog", async () => { - const { connection, methods } = await fixture(undefined, true); + const { connection, methods } = await fixture({ cycle: true }); await expect(connection.connect()).rejects.toThrow( "pagination did not terminate", ); @@ -166,7 +266,7 @@ it("rejects nonterminating SDK pagination instead of returning a partial tool ca }); it("reads native metadata resources and complete JSON text through the real SDK transport", async () => { - const { connection, methods } = await fixture(undefined, false, true); + const { connection, methods } = await fixture({ native: true }); expect(await connection.connect()).toEqual(["first", "second"]); expect(await connection.call("first", {})).toEqual([{ observed: true }]); expect(await connection.readResource?.("ida://idb/metadata")).toMatchObject({ @@ -176,3 +276,1182 @@ it("reads native metadata resources and complete JSON text through the real SDK await expect(connection.readResource?.("ida://other")).rejects.toThrow(); expect(methods).toContain("resources/read"); }); + +it("accepts a legacy stdio reply over the SDK default while staying bounded", async () => { + const script = String.raw` +let input = ""; +const send = (message) => process.stdout.write(JSON.stringify(message) + "\n"); +process.stdin.setEncoding("utf8"); +process.stdin.on("data", (chunk) => { + input += chunk; + for (;;) { + const newline = input.indexOf("\n"); + if (newline < 0) break; + const request = JSON.parse(input.slice(0, newline)); + input = input.slice(newline + 1); + if (request.id === undefined) continue; + const result = request.method === "initialize" + ? { protocolVersion: request.params.protocolVersion, capabilities: { tools: {} }, serverInfo: { name: "large-reply-fixture", version: "1" } } + : request.method === "tools/list" + ? { tools: [{ name: "list_strings", inputSchema: { type: "object", properties: {} } }] } + : { content: [{ type: "text", text: "fixture" }], structuredContent: { result: "x".repeat(11 * 1024 * 1024) } }; + send({ jsonrpc: "2.0", id: request.id, result }); + } +}); +`; + const connection = createIdaMcpConnection( + { + command: process.execPath, + args: ["-e", script], + env: {}, + mode: "attached", + timeoutMs: 10_000, + }, + {}, + ); + cleanups.push(() => connection.close()); + expect(await connection.connect()).toEqual(["list_strings"]); + expect(await connection.call("list_strings", {})).toEqual( + "x".repeat(11 * 1024 * 1024), + ); +}); + +it.each([false, true])( + "preserves an upstream tool rejection through SDK transport and public error projection (native: %s)", + async (native) => { + const reason = + "Invalid params: unexpected parameters: ['offset', 'max_lines']"; + const { connection } = await fixture({ + native, + toolResult: { + isError: true, + content: [{ type: "text", text: reason }], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", { + addr: "0x1000", + offset: 0, + max_lines: 100, + }) + .catch((cause: unknown) => cause); + + expect(failure).toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error( + "Expected the upstream rejection to remain a protocol error", + ); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); + +it.each([false, true])( + "redacts configured credentials from SDK tool errors while retaining caller context (native: %s)", + async (native) => { + const configurationHeaders = { + aUtHoRiZaTiOn: "\r\n \tBearer transport-fixture-secret \r\n", + "Proxy-Authorization": "Basic proxy-fixture-secret", + CoOkIe: "session=cookie-fixture-secret", + "X-Evidence": "selected-header-evidence", + }; + const context = + "/caller/input?cursor=visible API_KEY=caller-evidence selected-header-evidence"; + const { connection } = await fixture({ + native, + configurationHeaders, + toolResult: { + isError: true, + content: [ + { + type: "text", + text: + "Rejected Bearer transport-fixture-secret / transport-fixture-secret / Basic proxy-fixture-secret / proxy-fixture-secret / session=cookie-fixture-secret at " + + context, + }, + ], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error( + "Expected the upstream rejection to remain a protocol error", + ); + const reason = + "Rejected [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] / [redacted transport credential] at " + + context; + expect(failure.message).toBe("IDA MCP tool failed: \n" + reason); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); + +type ParsedCredentialCase = { + readonly label: string; + readonly headers: Readonly>; + readonly payloads: readonly string[]; +}; +const basicCredentialCases: readonly ParsedCredentialCase[] = [ + { + label: "decoded Basic ASCII credentials", + headers: { + Authorization: + "Basic " + + Buffer.from("basic-fixture-user:basic-fixture:password").toString( + "base64", + ), + }, + payloads: [ + "basic-fixture-user:basic-fixture:password", + "basic-fixture:password", + ], + }, + { + label: "decoded Proxy Basic UTF-8 credentials", + headers: { + "Proxy-Authorization": + "bAsIc " + + Buffer.from("utf8-fixture-user:páss-fixture:☃", "utf8").toString( + "base64", + ), + }, + payloads: ["utf8-fixture-user:páss-fixture:☃", "páss-fixture:☃"], + }, + { + label: "decoded Basic Latin-1 credentials", + headers: { + Authorization: + "Basic " + + Buffer.from( + "latin-fixture-user:café-fixture:password", + "latin1", + ).toString("base64"), + }, + payloads: [ + "latin-fixture-user:café-fixture:password", + "café-fixture:password", + ], + }, +]; +const parsedCredentialCases: readonly ParsedCredentialCase[] = [ + ...basicCredentialCases, + { + label: "Cookie values", + headers: { + Cookie: + 'session=cookie-fixture-secret; quoted="quoted-fixture=value=="; empty=', + }, + payloads: ["cookie-fixture-secret", "quoted-fixture=value=="], + }, + { + label: "Authorization Token payload", + headers: { Authorization: "Token custom-fixture-token+/==" }, + payloads: ["custom-fixture-token+/=="], + }, + { + label: "Proxy-Authorization custom token68", + headers: { "Proxy-Authorization": "Custom proxy-fixture-token._~+/-==" }, + payloads: ["proxy-fixture-token._~+/-=="], + }, + { + label: "Digest auth-param values", + headers: { + Authorization: + 'Digest username="fixture-user", nonce="nonce,fixture", response="response\\\"secret", opaque="back\\\\slash"', + }, + payloads: [ + "fixture-user", + "nonce,fixture", + 'response"secret', + "back\\slash", + ], + }, +]; + +it.each( + parsedCredentialCases.flatMap((testCase) => + [false, true].map((native) => ({ ...testCase, native })), + ), +)( + "redacts parsed $label from tool errors (native: $native)", + async ({ headers, payloads, native }) => { + const context = "/caller/input?cursor=visible API_KEY=caller-evidence"; + const { connection } = await fixture({ + native, + configurationHeaders: headers, + toolResult: { + isError: true, + content: [ + { + type: "text", + text: "Rejected " + payloads.join(" / ") + " at " + context, + }, + ], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error( + "Expected the upstream rejection to remain a protocol error", + ); + const reason = + "Rejected " + + payloads.map(() => "[redacted transport credential]").join(" / ") + + " at " + + context; + expect(failure.message).toBe("IDA MCP tool failed: \n" + reason); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); + +it.each([false, true])( + "retains successful credential-like evidence through SDK transport (native: %s)", + async (native) => { + const evidence = { + observed: "custom-fixture-token+/==", + cookie: "quoted-fixture=value==", + digest: 'response"secret', + }; + const { connection } = await fixture({ + native, + configurationHeaders: { + Authorization: "Token custom-fixture-token+/==", + Cookie: 'session="quoted-fixture=value=="', + "Proxy-Authorization": 'Digest response="response\\\"secret"', + }, + toolResult: native + ? { content: [{ type: "text", text: JSON.stringify(evidence) }] } + : { content: [], structuredContent: evidence }, + }); + await connection.connect(); + expect(await connection.call("first", {})).toEqual(evidence); + }, +); + +it.each<{ + label: string; + headers: Readonly>; + payloads: readonly string[]; +}>([ + { + label: "Cookie and custom token payloads", + headers: { + Authorization: "Token custom-fixture-token", + Cookie: "session=cookie-fixture-secret", + }, + payloads: ["custom-fixture-token", "cookie-fixture-secret"], + }, + { + label: "JSON-escaped Digest payloads and header", + headers: { + Authorization: 'Digest response="q\\\"uote", opaque="back\\\\slash"', + }, + payloads: [ + 'q"uote', + "back\\slash", + 'Digest response="q\\\"uote", opaque="back\\\\slash"', + ], + }, + ...basicCredentialCases, +])( + "redacts $label echoed in an HTTP failure through the SDK rejection path", + async ({ headers, payloads }) => { + const context = "/caller/input?cursor=visible API_KEY=caller-evidence"; + const { connection } = await fixture({ + toolStatus: 500, + configurationHeaders: headers, + toolResult: { + isError: true, + content: [ + { + type: "text", + text: + "Transport rejected " + payloads.join(" / ") + " at " + context, + }, + ], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + "Transport rejected " + + payloads.map(() => "[redacted transport credential]").join(" / ") + + " at " + + context, + ); + for (const payload of payloads) + expect(failure.message).not.toContain( + JSON.stringify(payload).slice(1, -1), + ); + }, +); + +it.each<{ + label: string; + headers: Readonly>; + credential: string; + wire: string; + toolContentType: string; +}>([ + { + label: "Go-style HTML escapes", + toolContentType: "application/problem+json; charset=utf-8", + headers: { Cookie: "session=&/fixture" }, + credential: "&/fixture", + wire: String.raw`\u003csecret\u003e\u0026/fixture`, + }, + { + label: "fully escaped ASCII with mixed hex case and solidus", + toolContentType: "application/json", + headers: { Authorization: "Token aBz/secret" }, + credential: "aBz/secret", + wire: String.raw`\u0061\u0042\u007A\/\u0073\u0065\u0063\u0072\u0065\u0074`, + }, + { + label: "Latin-1 auth-param Unicode and solidus escapes", + toolContentType: "application/json", + headers: { Authorization: 'Digest response="café/secret"' }, + credential: "café/secret", + wire: String.raw`caf\u00E9\/secret`, + }, +])( + "redacts $label in SDK HTTP failures without rewriting unrelated wire evidence", + async ({ headers, credential, wire, toolContentType }) => { + const prefix = '{\n "reason": "Rejected '; + const suffix = + String.raw`", "numeric": 9007199254740993, "duplicate": "first", "duplicate": "second", "context": "\u003cpublic\u003e\/asset", "symbol": "\ud83d\uDE00"` + + "\n}\n"; + const toolResponseBody = prefix + wire + suffix; + expect(JSON.parse(toolResponseBody)).toMatchObject({ + reason: "Rejected " + credential, + }); + const { connection } = await fixture({ + toolStatus: 500, + toolContentType, + configurationHeaders: headers, + toolResponseBody, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + prefix + "[redacted transport credential]" + suffix, + ); + expect(failure.message).not.toContain(wire); + }, +); + +const jsonFailureResponses = [ + { toolStatus: 200, toolContentType: "application/json" }, + { toolStatus: 500, toolContentType: "application/json" }, + { + toolStatus: 500, + toolContentType: "application/problem+json; charset=utf-8", + }, +] as const; + +it.each( + jsonFailureResponses.flatMap((response) => + [ + { + label: "unquoted escaped body", + credential: "", + wire: String.raw`\u003ctoken\u003e`, + toolResponseBody: String.raw`\u003ctoken\u003e`, + safeBody: "[redacted transport credential]", + }, + { + label: "quoted escaped array entry before invalid JSON", + credential: "", + wire: String.raw`\u003cs\u003e`, + toolResponseBody: String.raw`["\u003cs\u003e",x]`, + safeBody: '["[redacted transport credential]",x]', + }, + ].map((body) => ({ ...response, ...body })), + ), +)( + "redacts $label from malformed HTTP $toolStatus $toolContentType responses", + async ({ + credential, + wire, + toolResponseBody, + safeBody, + toolStatus, + toolContentType, + }) => { + const { connection } = await fixture({ + configurationHeaders: { Cookie: "token=" + credential }, + toolResponseBody, + toolStatus, + toolContentType, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject malformed JSON"); + expect(failure.message).toContain(safeBody); + if (toolStatus === 200) + expect(failure.message).toMatch(/malformed.*HTTP.*JSON/iu); + else expect(failure.cause).toMatchObject({ data: { status: 500 } }); + expect(failure.message).not.toContain(toolResponseBody); + expect(failure.message).not.toContain(wire); + expect(failure.message).not.toContain(credential); + }, +); + +it.each( + jsonFailureResponses.flatMap((response) => + [ + { + credentialLabel: "literal short escape", + credential: String.raw`folder\tmp`, + }, + { + credentialLabel: "literal Unicode escape", + credential: String.raw`folder\u0061`, + }, + ].flatMap((credentialCase) => + [ + { + bodyLabel: "unquoted body", + prefix: "Rejected ", + suffix: " at /caller/input?cursor=visible", + }, + { + bodyLabel: "quoted value in an invalid document", + prefix: 'Rejected "', + suffix: '" at /caller/input?cursor=visible', + }, + { + bodyLabel: "unfinished JSON string", + prefix: '{"reason":"Rejected ', + suffix: " at /caller/input?cursor=visible", + }, + ].map((body) => ({ ...response, ...credentialCase, ...body })), + ), + ), +)( + "redacts $credentialLabel in $bodyLabel from HTTP $toolStatus $toolContentType diagnostics and their public projection", + async ({ credential, toolStatus, toolContentType, prefix, suffix }) => { + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Custom " + credential }, + toolStatus, + toolContentType, + toolResponseBody: prefix + credential + suffix, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject malformed JSON"); + expect(failure.message).toContain( + prefix + "[redacted transport credential]" + suffix, + ); + expect(failure.message).not.toContain(credential); + if (toolStatus === 200) + expect(failure.message).toMatch(/malformed.*HTTP.*JSON/iu); + else expect(failure.cause).toMatchObject({ data: { status: 500 } }); + const projected = projectAnalysisError( + new ProviderAdapterError("ida", "decompile", { + cause: failure, + diagnostics: { reason: failure.message }, + }), + ); + expect(projected).toMatchObject({ + code: "execution_failure", + details: { diagnostics: { reason: failure.message } }, + }); + expect(JSON.stringify(projected)).not.toContain( + JSON.stringify(credential).slice(1, -1), + ); + }, +); + +it("preserves JSON escape syntax that spells a configured credential without containing its value", async () => { + const toolResponseBody = String.raw`{"reason":"\u003cpublic\u003e","numeric":9007199254740993}`; + const { connection } = await fixture({ + toolStatus: 500, + configurationHeaders: { Cookie: "token=u003c" }, + toolResponseBody, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain(toolResponseBody); + expect(failure.message).not.toContain("[redacted transport credential]"); +}); + +it("redacts complete opaque authentication values containing quotes from plain HTTP error bodies", async () => { + const credential = 'prefix"quoted"suffix'; + const header = "Custom " + credential; + const context = "/caller/input?cursor=visible API_KEY=caller-evidence"; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: header }, + toolResponseBody: + "Rejected " + header + " / " + credential + " at " + context, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + "Rejected [redacted transport credential] / [redacted transport credential] at " + + context, + ); + expect(failure.message).not.toContain(credential); +}); + +it.each(["Rejected ", ""])( + "redacts quoted literal credentials from text/plain HTTP failures with prefix %j", + async (prefix) => { + const credential = String.raw`folder\tmp`; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody: prefix + '"' + credential + '"', + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).not.toBeInstanceOf(AnalysisProtocolError); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the HTTP failure"); + expect(failure.message).toContain( + prefix + '"[redacted transport credential]"', + ); + expect(failure.message).not.toContain(credential); + }, +); + +it("redacts a JSON-RPC error exactly once after successful HTTP delivery", async () => { + const { connection } = await fixture({ + configurationHeaders: { Cookie: "token=redacted" }, + toolRpcError: { code: -32602, message: "redacted at /caller/input" }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected the SDK to reject the JSON-RPC error"); + expect(failure.cause).toBeInstanceOf(ProtocolError); + expect(failure.message.endsWith("␟ at /caller/input")).toBe(true); + expect(failure.message.match(/␟/gu)).toHaveLength(1); + expect(failure.message).not.toContain("redacted"); +}); + +it.each([ + "", + String.raw`\u003ccause-fixture-secret\u003e`, +])( + "does not retain sensitive malformed HTTP JSON in inspected causes: %s", + async (wire) => { + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" }, + toolResponseBody: '{"reason":"Rejected ' + wire + " at /caller/input", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected malformed JSON rejection"); + expect(failure.message).toContain( + "[redacted transport credential] at /caller/input", + ); + const inspected = inspect(failure, { + depth: null, + showHidden: true, + maxStringLength: null, + }); + for (const value of [ + "", + wire, + JSON.stringify(wire).slice(1, -1), + ]) + expect(inspected).not.toContain(value); + }, +); + +it("retains safe JSON-RPC code and message without unknown remote cause data", async () => { + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Bearer rpc-cause-fixture-secret" }, + toolRpcError: { + code: -32602, + message: "Rejected request at /caller/input", + data: { + echo: "rpc-cause-fixture-secret", + nested: { caller: "/caller/input" }, + }, + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected JSON-RPC rejection"); + expect(failure.message).toContain("Rejected request at /caller/input"); + expect(failure.cause).toBeInstanceOf(ProtocolError); + expect(failure.cause).toMatchObject({ code: -32602 }); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "rpc-cause-fixture-secret", + ); +}); + +it.each([ + "text/html", + "application/problem+xml", + "application/octet-stream", + "text/plain; charset=utf-16", + "not a media type", +])( + "omits HTTP error bodies whose declared encoding is unsupported: %s", + async (toolContentType) => { + const body = + "<format-fixture-secret>&/fixture at /caller/input"; + const { connection } = await fixture({ + configurationHeaders: { + Cookie: "session=&/fixture", + }, + toolStatus: 500, + toolContentType, + toolResponseBody: body, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP error"); + expect(failure.message).toMatch(/body.*omitted|omitted.*body/iu); + expect(failure.cause).toMatchObject({ data: { status: 500 } }); + const inspected = inspect(failure, { + depth: null, + showHidden: true, + maxStringLength: null, + }); + expect(inspected).not.toContain(body); + expect(inspected).not.toContain("format-fixture-secret"); + }, +); + +it.each(["native tool", "resource"] as const)( + "does not retain the producer JSON parse cause for %s text", + async (source) => { + const malformedText = '["token",x]'; + const { connection } = await fixture({ + native: true, + configurationHeaders: { Authorization: "Bearer token" }, + ...(source === "native tool" + ? { toolResult: { content: [{ type: "text", text: malformedText }] } } + : { resourceResponseText: malformedText }), + }); + await connection.connect(); + const failure = await ( + source === "native tool" + ? connection.call("first", {}) + : connection.readResource?.("ida://idb/metadata") + )?.catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(AnalysisProtocolError); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "token", + ); + }, +); + +it.each([200, 500])( + "preserves owned diagnostic capacity failures through HTTP %s SDK handling", + async (toolStatus) => { + const { connection } = await fixture({ + toolStatus, + toolResponseBody: "fixture-resource-boundary", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + expect(failure).toBeInstanceOf(IdaDiagnosticResourceError); + if (!(failure instanceof IdaDiagnosticResourceError)) + throw new Error("Expected an owned diagnostic capacity failure"); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "resource_constraint", + details: { + resource: "memory", + reported_limits: { boundary: "ida-diagnostic-redaction" }, + }, + }); + }, +); + +it.each( + [200, 500].flatMap((toolStatus) => + ["utf-16le", "utf-16", " = utf-16", "=utf-8; charset=utf-16"].map( + (charset) => ({ toolStatus, charset }), + ), + ), +)( + "omits unsupported $charset diagnostic bytes from HTTP $toolStatus JSON failures", + async ({ toolStatus, charset }) => { + const credential = "charset-fixture-secret"; + const utf16 = Buffer.from('{"reason":"' + credential + '"}', "utf16le"); + const encoded = + charset === "utf-16" + ? Buffer.concat([Buffer.from([0xff, 0xfe]), utf16]) + : utf16; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + credential }, + toolStatus, + toolContentType: + "application/json; charset" + + (charset.startsWith("=") || charset.startsWith(" ") + ? charset + : "=" + charset), + toolResponseBody: encoded, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected unsupported encoded JSON error"); + expect(failure.message).toMatch(/body.*omitted|omitted.*body/iu); + expect(failure.message).not.toContain(encoded.toString("utf8")); + const inspected = inspect(failure, { + depth: null, + showHidden: true, + maxStringLength: null, + }); + expect( + inspected + .replaceAll("\\x00", "") + .replaceAll("\\u0000", "") + .replaceAll("\0", ""), + ).not.toContain(credential); + }, +); + +it("retains successfully parsed analysis values despite unsupported declared charset metadata", async () => { + const observed = "charset-fixture-secret"; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + observed }, + toolContentType: "application/json; charset=utf-16le", + toolResult: { content: [], structuredContent: { observed } }, + }); + await connection.connect(); + expect(await connection.call("first", {})).toEqual({ observed }); +}); + +it.each( + [200, 500].flatMap((toolStatus) => + [ + "application/json; charset = utf-16", + "application/json; charset\t=utf-16le", + "application/json; charset=utf-8; CHARSET=utf-16le", + 'application/json; note="unterminated; charset=utf-16le', + ].map((toolContentType) => ({ toolStatus, toolContentType })), + ), +)( + "omits encoded diagnostic bytes when Content-Type is incomplete or ambiguous: HTTP $toolStatus $toolContentType", + async ({ toolStatus, toolContentType }) => { + const credential = "mime-fixture-secret"; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + credential }, + toolStatus, + toolContentType, + toolResponseBody: Buffer.from( + '{"reason":"' + credential + '"}', + "utf16le", + ), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected malformed Content-Type failure"); + expect(failure.message).toMatch(/body.*omitted|omitted.*body/iu); + expect( + inspect(failure, { depth: null, showHidden: true }).replaceAll( + "\\x00", + "", + ), + ).not.toContain(credential); + }, +); + +it.each([ + "application/json; charset=", + 'application/json; charset="utf-8', + 'application/json; charset="utf-8"junk', + "application/json; note=a,b", + "application/json; note=a; NOTE=b", + "text/plain; charset = utf-16", +])( + "omits error bodies whose whole Content-Type is invalid or conflicting: %s", + async (toolContentType) => { + const { connection } = await fixture({ + toolStatus: 500, + toolContentType, + toolResponseBody: "untrusted /caller/body", + }); + await connection.connect(); + await expect(connection.call("first", {})).rejects.toThrow( + /body.*omitted|omitted.*body/iu, + ); + }, +); + +it.each([ + "application/json", + 'Application/Problem+JSON\t; Note="a; charset=utf-16"; ChArSeT="UTF-8"', + String.raw`application/json; charset="utf\-8"; note="escaped\"quote; slash\\"`, + 'application/json; note=""; charset=utf-8;', + "application/json;; ;charset=utf-8;", + "text/plain; note=caller; charset=utf8", +])( + "retains safe producer diagnostics for complete legal Content-Type: %s", + async (toolContentType) => { + const credential = "mime-fixture-secret"; + const reason = "Rejected " + credential + " at /caller/input"; + const { connection } = await fixture({ + configurationHeaders: { Cookie: "session=" + credential }, + toolStatus: 500, + toolContentType, + toolResponseBody: toolContentType.startsWith("text/plain") + ? reason + : JSON.stringify({ reason }), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected upstream error"); + expect(failure.message).toContain( + "Rejected [redacted transport credential] at /caller/input", + ); + expect(failure.message).not.toContain("omitted"); + expect(failure.message).not.toContain(credential); + }, +); + +it.each(["endpoint", ""])( + "redacts the complete SDK context when the trusted body repeats its text: %s", + async (toolResponseBody) => { + const credential = "Error POSTing to endpoint"; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toBe( + "[redacted transport credential]: \n" + toolResponseBody, + ); + expect(failure.cause).toMatchObject({ data: { status: 500 } }); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); + }, +); + +it("keeps a sanitized plain HTTP body intact without exempting matching status text", async () => { + const { connection } = await fixture({ + toolStatus: 500, + toolStatusText: "abcXdef", + toolContentType: "text/plain", + configurationHeaders: { + Authorization: "Custom abcXdef", + Cookie: "token=redacted", + }, + toolResponseBody: "X redacted at /caller/input", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("X ␟ at /caller/input"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "abcXdef", + ); +}); + +it("keeps SDK context and the owned body separate when joining them would form a credential", async () => { + const credential = "endpoint: public"; + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody: "public", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("endpoint: \npublic"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); +}); + +it("keeps owned HTTP JSON failure context separate from the sanitized body", async () => { + const credential = "Response body: public"; + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Custom " + credential }, + toolResponseBody: "public", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected malformed JSON failure"); + expect(failure.message).toContain("Response body: \npublic"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); +}); + +it.each(["tool rejection", "HTTP rejection", "malformed HTTP JSON"] as const)( + "redacts overlapping credential spans through the SDK: %s", + async (source) => { + const reason = "Rejected X]SECRET at /caller/input"; + const { connection } = await fixture({ + configurationHeaders: { Authorization: "Custom X]", Cookie: "a=]SECRET" }, + ...(source === "tool rejection" + ? { + toolResult: { + isError: true, + content: [{ type: "text", text: reason }], + }, + } + : { + toolStatus: source === "HTTP rejection" ? 500 : 200, + toolResponseBody: + String.raw`{"reason":"Rejected X\u005dSECRET at /caller/input"` + + (source === "HTTP rejection" ? "}" : ",x"), + }), + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) + throw new Error("Expected producer failure"); + expect(failure.message).toContain("at /caller/input"); + const retained = inspect(failure, { depth: null, showHidden: true }); + expect(retained).not.toContain("SECRET"); + expect(retained).not.toContain("X]"); + }, +); + +it("does not introduce a credential at a redaction-marker boundary in a real HTTP failure", async () => { + const { connection } = await fixture({ + toolStatus: 500, + toolContentType: "text/plain", + configurationHeaders: { Authorization: "Custom X", Cookie: "a=]SECRET" }, + toolResponseBody: "Rejected XSECRET at /caller/input", + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof Error)) throw new Error("Expected HTTP failure"); + expect(failure.message).toContain("SECRET at /caller/input"); + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + "]SECRET", + ); +}); + +it.each( + [ + { + label: "Basic pair and leading-space password", + authorization: "Basic ZmFpbGVkOiBwdWJsaWM=", + forbidden: ["failed: public", " public"], + }, + { + label: "opaque payload spanning the tool prefix and body", + authorization: "Custom failed: public", + forbidden: ["failed: public"], + }, + { + label: "credential equal to the fixed tool prefix", + authorization: "Custom IDA MCP tool failed", + forbidden: ["IDA MCP tool failed"], + }, + ].flatMap((testCase) => + [false, true].map((native) => ({ ...testCase, native })), + ), +)( + "keeps tool-error owners separate for $label (native: $native)", + async ({ authorization, forbidden, native }) => { + const { connection } = await fixture({ + native, + configurationHeaders: { Authorization: authorization }, + toolResult: { + isError: true, + content: [{ type: "text", text: "public" }], + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error("Expected the tool rejection to remain a protocol error"); + expect(failure.message).toContain("public"); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason: "public" } }, + }); + for (const credential of forbidden) + expect(inspect(failure, { depth: null, showHidden: true })).not.toContain( + credential, + ); + }, +); + +const toolDiagnosticBlockCases: readonly { + label: string; + headers: Readonly>; + blocks: readonly string[]; + safeBlocks: readonly string[]; +}[] = [ + { + label: "JSON-escaped quoted credential without rewriting other source", + headers: { Authorization: 'Custom q"uote' }, + blocks: [ + '{"error":"Rejected q\\"uote at /caller/input","amount":9007199254740993,"duplicate":1,"duplicate":2}', + ], + safeBlocks: [ + '{"error":"Rejected [redacted transport credential] at /caller/input","amount":9007199254740993,"duplicate":1,"duplicate":2}', + ], + }, + { + label: "independent JSON arrays, escaped keys and surrogate pairs", + headers: { + Authorization: + "Basic " + Buffer.from("user:😀", "utf8").toString("base64"), + Cookie: "token=; syntax=u003c", + }, + blocks: [ + "Plain status at /caller/plain", + '[{"path":"/caller/array"},"\\u003cjson-secret\\u003e"]', + '{"\\u003cjson-secret\\u003e":"key context","emoji":"\\uD83d\\uDE00 at /caller/emoji"}', + '{"public":"\\u003cpublic\\u003e","amount":9007199254740993}', + ], + safeBlocks: [ + "Plain status at /caller/plain", + '[{"path":"/caller/array"},"[redacted transport credential]"]', + '{"[redacted transport credential]":"key context","emoji":"[redacted transport credential] at /caller/emoji"}', + '{"public":"\\[redacted transport credential]public\\u003e","amount":9007199254740993}', + ], + }, + { + label: + "unclassified text retains both raw and decoded credential candidates", + headers: { + Authorization: "Custom folder\\tmp", + Cookie: "token=", + }, + blocks: [ + "Rejected folder\\tmp at /caller/plain; literal \\u003cpublic\\u003e", + ], + safeBlocks: [ + "Rejected [redacted transport credential] at /caller/plain; literal [redacted transport credential]", + ], + }, + { + label: "quoted plain backslash credential despite valid JSON appearance", + headers: { Authorization: "Custom folder\\tmp" }, + blocks: [String.raw`"folder\tmp at /caller/input"`], + safeBlocks: ['"[redacted transport credential] at /caller/input"'], + }, + { + label: "malformed JSON quote escapes and independent context", + headers: { Authorization: 'Custom q"uote' }, + blocks: [ + String.raw`{"error":"Rejected q\"uote at /caller/input`, + "Second context at /caller/second", + ], + safeBlocks: [ + '{"error":"Rejected [redacted transport credential] at /caller/input', + "Second context at /caller/second", + ], + }, +]; + +it.each( + toolDiagnosticBlockCases.flatMap((testCase) => + [false, true].map((native) => ({ ...testCase, native })), + ), +)( + "owns each SDK tool diagnostic block as $label (native: $native)", + async ({ headers, blocks, safeBlocks, native }) => { + const { connection } = await fixture({ + native, + configurationHeaders: headers, + toolResult: { + isError: true, + content: blocks.map((text) => ({ type: "text", text })), + }, + }); + await connection.connect(); + const failure = await connection + .call("first", {}) + .catch((cause: unknown) => cause); + if (!(failure instanceof AnalysisProtocolError)) + throw new Error("Expected the tool rejection to remain a protocol error"); + const reason = safeBlocks.join("\n"); + expect(failure.message.endsWith(reason)).toBe(true); + expect(projectAnalysisError(failure)).toMatchObject({ + code: "unreadable_output", + details: { diagnostics: { reason } }, + }); + }, +); diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index 9ce3f77ced..4bd24f5f76 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -31,6 +31,7 @@ import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js const fixturePath = fileURLToPath( new URL("../../../fixtures/fakeGhidra.mjs", import.meta.url), ); + const PROFILE_DIGEST = "a".repeat(64); const PROVIDER_VERSION = "12.1.4"; const TARGET_SHA256 = createHash("sha256") @@ -40,6 +41,8 @@ const HOST_TRANSPORT: GhidraTransportKind = process.platform === "win32" ? "authenticated-loopback-tcp" : "unix-socket"; type FixtureMode = + | "reject_first_connection" + | "reject_all_connections" | "success" | "fragmented" | "oversized_result" @@ -1131,3 +1134,30 @@ describe.skipIf(process.platform === "win32")( ); }, ); + +it("reconnects a dropped preauthentication loopback socket within the original startup deadline", async () => { + const launcher = new FixtureLauncher("reject_first_connection"); + const client = clientFor(launcher, { + transport: "authenticated-loopback-tcp", + }); + const started = await client.start(); + expect(started.ok).toBe(true); + expect(launcher.processes).toHaveLength(1); + const result = await client.callToolIfReady("list_strings", {}); + expect(result).toMatchObject({ ok: true }); +}); + +it("bounds retries when loopback preauthentication keeps failing", async () => { + const launcher = new FixtureLauncher("reject_all_connections"); + const client = clientFor(launcher, { + transport: "authenticated-loopback-tcp", + }); + expect(await client.start()).toMatchObject({ + ok: false, + error: { kind: "process" }, + }); + expect(launcher.processes).toHaveLength(1); + await expect(waitForChildExit(launcher.processes[0]!, 5_000)).resolves.toBe( + true, + ); +}); diff --git a/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts b/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts index 62310facf0..6188f0a39b 100644 --- a/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.cleanupRetry.test.ts @@ -184,6 +184,7 @@ it.each([false, true])( processSupervisor: supervisor, runtimeRoot, shutdownConfirmed: false, + externalApplication: undefined, }; const socket = new Socket(); onTestFinished(() => { @@ -393,6 +394,7 @@ it("preserves confirmed document shutdown while retrying owned process cleanup", processSupervisor: supervisor, runtimeRoot, shutdownConfirmed: false, + externalApplication: undefined, } satisfies HopperOwnedResources, activeRequest: null, retainDocument: false, @@ -457,6 +459,7 @@ it.each([ processSupervisor: supervisor, runtimeRoot, shutdownConfirmed: false, + externalApplication: undefined, }; const result = await cleanupHopperSession({ socket: new Socket(), diff --git a/tests/boundary/providers/hopper/hopperClient.restart.test.ts b/tests/boundary/providers/hopper/hopperClient.restart.test.ts index 5acf382f63..ef22cf0f4a 100644 --- a/tests/boundary/providers/hopper/hopperClient.restart.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.restart.test.ts @@ -30,7 +30,11 @@ describe("HopperClient restart", () => { await expect(client.start()).resolves.toEqual({ ok: true, - value: { name: "REA Hopper bridge", version: "1.0.0" }, + value: { + name: "REA Hopper bridge", + version: "1.0.0", + processId: expect.any(Number), + }, }); }); @@ -42,7 +46,11 @@ describe("HopperClient restart", () => { await closing; await expect(restarting).resolves.toEqual({ ok: true, - value: { name: "REA Hopper bridge", version: "1.0.0" }, + value: { + name: "REA Hopper bridge", + version: "1.0.0", + processId: expect.any(Number), + }, }); await expect(client.callTool("echo", { value: "fresh" })).resolves.toEqual({ ok: true, diff --git a/tests/composition/idaProvider.test.ts b/tests/composition/idaProvider.test.ts index 538c9d30ee..58f3b78da4 100644 --- a/tests/composition/idaProvider.test.ts +++ b/tests/composition/idaProvider.test.ts @@ -186,7 +186,7 @@ describe("IDA provider composition", () => { }); describe("IDA binary_overview execution", () => { - it("reports document selection unavailable when binary_overview executes against an IDA session", async () => { + it("uses the bound IDA document and omits unsupported segment coverage", async () => { const { root, target } = await createIdaTarget(); roots.push(root); const path = join(root, "registration.json"); @@ -200,17 +200,16 @@ describe("IDA binary_overview execution", () => { expect((await session.open(target.path)).ok).toBe(true); const tools = new EnhancedTools(session); const overview = await tools.execute("binary_overview", {}); - expect(overview).toMatchObject({ - ok: false, - error: { - _tag: "AnalysisCapabilityUnavailableError", - providerId: "ida", - operation: "list_documents", - message: expect.stringContaining( - "operation is not declared by this provider", - ), - }, - }); + expect(overview.ok).toBe(true); + if (overview.ok) { + expect(overview.value).toMatchObject({ + document: target.path, + procedure_count: 1, + string_count: 1, + }); + expect(overview.value).not.toHaveProperty("segments"); + expect(overview.value).not.toHaveProperty("segment_count"); + } const procedures = await session.execute("list_procedures", {}); expect(procedures.ok).toBe(true); expect((await session.close()).ok).toBe(true); diff --git a/tests/conformance/ghidra/ReaNoReturnProbe.java b/tests/conformance/ghidra/ReaNoReturnProbe.java index b13c2b5e9d..19f98f346a 100644 --- a/tests/conformance/ghidra/ReaNoReturnProbe.java +++ b/tests/conformance/ghidra/ReaNoReturnProbe.java @@ -12,6 +12,8 @@ import ghidra.program.model.listing.FlowOverride; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.Instruction; import ghidra.program.model.symbol.SourceType; +import ghidra.program.model.symbol.Reference; +import ghidra.program.model.symbol.RefType; import java.io.File; import java.lang.reflect.Method; import java.nio.file.Files; @@ -20,9 +22,12 @@ import java.util.HexFormat; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.ArrayList; /** Injects reported analysis flags into imported ELF/Mach-O functions; never executes target code. */ public class ReaNoReturnProbe extends GhidraScript { + private record MemoryCallReference(Address source, Address target, RefType type, + SourceType sourceType, int operandIndex, boolean primary) {} private final JsonArray checks = new JsonArray(); private final Map originalEnds = new LinkedHashMap<>(); private final Map originalSignatures = new LinkedHashMap<>(); @@ -85,6 +90,114 @@ public class ReaNoReturnProbe extends GhidraScript { return instance; } + private void knownInteriorCall(GhidraScript bridge) throws Exception { + Function caller = function("rea_tls_chain"); + Function target = function("rea_tls"); + Address interior = target.getEntryPoint().add(1); + require("interior_target_is_owned", currentProgram.getFunctionManager().getFunctionAt(interior) == null && + target.equals(currentProgram.getFunctionManager().getFunctionContaining(interior))); + Instruction call = null; + for (Instruction instruction : currentProgram.getListing().getInstructions(caller.getBody(), true)) { + if (instruction.getFlowType().isCall()) { call = instruction; break; } + } + if (call == null) throw new IllegalStateException("Missing real call instruction"); + var references = currentProgram.getReferenceManager(); + List originals = new ArrayList<>(); + List saved = new ArrayList<>(); + for (Reference reference : call.getReferencesFrom()) { + if (!reference.getReferenceType().isCall()) continue; + originals.add(reference); + saved.add(new MemoryCallReference(reference.getFromAddress(), reference.getToAddress(), + reference.getReferenceType(), reference.getSource(), reference.getOperandIndex(), reference.isPrimary())); + } + require("original_call_reference_present", !originals.isEmpty()); + Reference changed = null; + try { + for (Reference reference : originals) references.delete(reference); + changed = references.addMemoryReference(call.getAddress(), interior, + RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, 0); + require("interior_target_omitted_by_callee_getter", !caller.getCalledFunctions(monitor).contains(target)); + JsonObject query = new JsonObject(); + query.addProperty("document", currentProgram.getName()); + query.addProperty("procedure", caller.getName()); + query.addProperty("direction", "outgoing"); + Method read = bridge.getClass().getDeclaredMethod("procedureReferences", JsonObject.class); + read.setAccessible(true); + JsonObject output = (JsonObject) read.invoke(bridge, query); + require("interior_reference_keeps_known_procedure", output.getAsJsonArray("references").asList().stream().anyMatch( + edge -> !edge.getAsJsonObject().get("target_procedure").isJsonNull() && + edge.getAsJsonObject().getAsJsonObject("target_procedure").get("name").getAsString().equals(target.getName()))); + require("known_reference_is_not_also_unresolved", output.getAsJsonArray("unresolved_calls").isEmpty()); + query.remove("direction"); + Method callees = bridge.getClass().getDeclaredMethod("procedureCalls", JsonObject.class, boolean.class); + callees.setAccessible(true); + JsonObject inventory = (JsonObject) callees.invoke(bridge, query, false); + require("omitted_callee_inventory_stays_partial", inventory.getAsJsonObject("relationship_coverage").get("status").getAsString().equals("partial")); + require("omitted_callee_direction_is_explicit", + inventory.getAsJsonObject("relationship_coverage").getAsJsonArray("limited_directions").toString().equals("[\"callees\"]")); + require("interior_caller_omitted_by_entry_getter", !target.getCallingFunctions(monitor).contains(caller)); + query.addProperty("procedure", target.getName()); + JsonObject incoming = (JsonObject) callees.invoke(bridge, query, true); + require("off_entry_caller_inventory_stays_partial", + incoming.getAsJsonObject("relationship_coverage").get("status").getAsString().equals("partial")); + require("off_entry_caller_direction_is_explicit", + incoming.getAsJsonObject("relationship_coverage").getAsJsonArray("limited_directions").toString().equals("[\"callers\"]")); + JsonObject dossierQuery = new JsonObject(); + dossierQuery.addProperty("procedure", target.getName()); + Method analyze = bridge.getClass().getDeclaredMethod("analyzeFunctionObservation", JsonObject.class); + analyze.setAccessible(true); + Object observation = analyze.invoke(bridge, dossierQuery); + Method coverage = observation.getClass().getDeclaredMethod("relationshipCoverage"); + coverage.setAccessible(true); + require("dossier_preserves_caller_only_gap", + ((JsonObject) coverage.invoke(observation)).getAsJsonArray("limited_directions").toString().equals("[\"callers\"]")); + } finally { + if (changed != null) references.delete(changed); + for (MemoryCallReference reference : saved) { + Reference restored = references.addMemoryReference(reference.source(), reference.target(), + reference.type(), reference.sourceType(), reference.operandIndex()); + references.setPrimary(restored, reference.primary()); + } + } + } + + private void nonCallInstructionReference(GhidraScript bridge) throws Exception { + Function caller = function("rea_tls_chain"); + Function target = function("rea_memcpy"); + Instruction source = null; + for (Instruction instruction : currentProgram.getListing().getInstructions(caller.getBody(), true)) { + if (!instruction.getFlowType().isCall()) { source = instruction; break; } + } + if (source == null) throw new IllegalStateException("Missing non-call instruction"); + var references = currentProgram.getReferenceManager(); + Reference added = references.addMemoryReference(source.getAddress(), target.getEntryPoint().add(1), + RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, -1); + try { + require("non_call_reference_target_omitted", !caller.getCalledFunctions(monitor).contains(target)); + JsonObject query = new JsonObject(); + query.addProperty("document", currentProgram.getName()); + query.addProperty("procedure", caller.getName()); + Method read = bridge.getClass().getDeclaredMethod("procedureCalls", JsonObject.class, boolean.class); + read.setAccessible(true); + JsonObject output = (JsonObject) read.invoke(bridge, query, false); + require("non_call_instruction_reference_stays_partial", + output.getAsJsonObject("relationship_coverage").get("status").getAsString().equals("partial")); + query.addProperty("direction", "outgoing"); + Method edges = bridge.getClass().getDeclaredMethod("procedureReferences", JsonObject.class); + edges.setAccessible(true); + JsonObject known = (JsonObject) edges.invoke(bridge, query); + require("non_call_known_reference_is_not_unresolved", known.getAsJsonArray("unresolved_calls").isEmpty()); + references.delete(added); + Address unknown = currentProgram.getMemory().getMaxAddress().add(0x1000); + added = references.addMemoryReference(source.getAddress(), unknown, + RefType.UNCONDITIONAL_CALL, SourceType.USER_DEFINED, -1); + JsonObject unresolved = (JsonObject) edges.invoke(bridge, query); + require("non_call_unknown_reference_keeps_site", unresolved.getAsJsonArray("unresolved_calls").size() == 1); + } finally { + references.delete(added); + } + } + @Override public void run() throws Exception { String[] args = getScriptArgs(); @@ -164,6 +277,8 @@ public class ReaNoReturnProbe extends GhidraScript { Object output = pseudocode.invoke(bridge, query); require("recovered_pseudocode", output instanceof JsonObject value && value.get("value").getAsString().contains("rea_after_call")); + nonCallInstructionReference(bridge); + knownInteriorCall(bridge); } finally { ((ghidra.app.decompiler.DecompInterface) field.get(bridge)).dispose(); } diff --git a/tests/conformance/providers/ghidra/windowsCapabilities.test.ts b/tests/conformance/providers/ghidra/windowsCapabilities.test.ts index 86fe3ec3b7..abc96587d8 100644 --- a/tests/conformance/providers/ghidra/windowsCapabilities.test.ts +++ b/tests/conformance/providers/ghidra/windowsCapabilities.test.ts @@ -17,16 +17,25 @@ const dependencies = ( available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, private_runtime_dacl: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, reparse_safe_path_admission: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, }), probeSymlinkCreation: () => @@ -91,16 +100,25 @@ describe("Windows host capability report", () => { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, private_runtime_dacl: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, reparse_safe_path_admission: { available: false, reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, proof: "not-proven", + code: "runtime_missing", + failureStage: "artifact", + remediation: "Install the matching Windows x64 REA package.", }, }, }); diff --git a/tests/fixtures/fakeGhidra.mjs b/tests/fixtures/fakeGhidra.mjs index 0a130c744b..a7f29ba126 100644 --- a/tests/fixtures/fakeGhidra.mjs +++ b/tests/fixtures/fakeGhidra.mjs @@ -32,7 +32,15 @@ if (mode === "silent") setInterval(() => undefined, 1_000); else { await rm(endpointPath, { force: true }); const sockets = new Set(); + let rejectedConnections = 0; const server = createServer((socket) => { + if ( + mode === "reject_all_connections" || + (mode === "reject_first_connection" && rejectedConnections++ === 0) + ) { + socket.destroy(); + return; + } sockets.add(socket); socket.once("close", () => sockets.delete(socket)); socket.setEncoding("utf8"); diff --git a/tests/fixtures/fakeHopper.mjs b/tests/fixtures/fakeHopper.mjs index 192d69851a..2491988a42 100644 --- a/tests/fixtures/fakeHopper.mjs +++ b/tests/fixtures/fakeHopper.mjs @@ -217,6 +217,7 @@ const server = createServer((socket) => { name: "REA Hopper bridge", version: "1.0.0", run_id: runId, + process_id: process.pid, }, }, true, diff --git a/tests/fixtures/ghidraHandshakeProbe.py b/tests/fixtures/ghidraHandshakeProbe.py new file mode 100644 index 0000000000..1c363e51a8 --- /dev/null +++ b/tests/fixtures/ghidraHandshakeProbe.py @@ -0,0 +1,107 @@ +"""JDK-only TCP transport regression using the production admission methods. + +Run: python3 tests/fixtures/ghidraHandshakeProbe.py /path/to/java +Ghidra request parsing/dispatch are fixture seams; no engine is required. +""" +import pathlib +import subprocess +import sys +import tempfile + +root = pathlib.Path(__file__).resolve().parents[2] +bridge = (root / "bridge/ghidra/ReaGhidraBridge.java").read_text() +start = bridge.index(" private static String readLoopbackHandshake(") +end = bridge.index(" private void acceptClient(", start) +methods = bridge[start:end] +harness = r''' +import java.io.*; +import java.net.*; +import java.nio.channels.*; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.*; + +public class ReaHandshakeProbe { + record SessionDescriptor(String token) {} + record Request(String method) {} + private Request parseRequest(String line, String token) { + if (!line.equals(token + ":ping")) throw new IllegalArgumentException(); + return new Request("ping"); + } + private void serveClient(SessionDescriptor descriptor, BufferedReader reader, + BufferedWriter writer, Request first) throws Exception { + if (!first.method().equals("ping")) throw new AssertionError("first request lost"); + String next = reader.readLine(); + if (next == null || next.length() != 200_000) + throw new AssertionError("pipelined authenticated request lost or capped"); + writer.write("accepted\n"); + writer.flush(); + } + static Socket connect(int port) throws Exception { + Socket peer = new Socket("127.0.0.1", port); + peer.setSoTimeout(3_000); + return peer; + } + static void closed(Socket peer) throws Exception { + if (peer.getInputStream().read() != -1) throw new AssertionError("peer not rejected"); + } + public static void main(String[] args) throws Exception { + try (ServerSocketChannel server = ServerSocketChannel.open(); + ExecutorService worker = Executors.newSingleThreadExecutor()) { + server.bind(new InetSocketAddress("127.0.0.1", 0)); + int port = ((InetSocketAddress) server.getLocalAddress()).getPort(); + Future serving = worker.submit(() -> { + try { new ReaHandshakeProbe().acceptAuthenticatedLoopbackClient( + server, new SessionDescriptor("secret")); } + catch (Exception error) { throw new RuntimeException(error); } + }); + try (Socket invalid = connect(port)) { + invalid.getOutputStream().write("bad-token:ping\n".getBytes(StandardCharsets.UTF_8)); + closed(invalid); + } + // EOF before a newline is not an authenticated request. + try (Socket eof = connect(port)) { + eof.getOutputStream().write("secret:ping".getBytes(StandardCharsets.UTF_8)); + eof.shutdownOutput(); + closed(eof); + } + try (Socket oversized = connect(port)) { + oversized.getOutputStream().write(new byte[64 * 1024 + 1]); + closed(oversized); + } + try (Socket idle = connect(port)) { + long start = System.nanoTime(); + closed(idle); + long elapsed = (System.nanoTime() - start) / 1_000_000; + if (elapsed < 700 || elapsed > 2_500) throw new AssertionError("idle deadline " + elapsed); + } + try (Socket trickle = connect(port)) { + long start = System.nanoTime(); + for (int i = 0; i < 4; i++) { + trickle.getOutputStream().write('x'); + Thread.sleep(250); + } + closed(trickle); + if ((System.nanoTime() - start) / 1_000_000 > 2_000) + throw new AssertionError("slow input extended deadline"); + } + // Same write contains the first and the following frame. The first + // reader must consume exactly one line without swallowing lookahead. + try (Socket valid = connect(port)) { + valid.getOutputStream().write(("secret:ping\n" + "x".repeat(200_000) + "\n") + .getBytes(StandardCharsets.UTF_8)); + String reply = new BufferedReader(new InputStreamReader(valid.getInputStream(), + StandardCharsets.UTF_8)).readLine(); + if (!"accepted".equals(reply)) throw new AssertionError("legitimate client failed"); + } + serving.get(3, TimeUnit.SECONDS); + } + System.out.println("PASS: rejection, EOF, frame ceiling, idle/trickle deadlines, retry, pipeline, authenticated large request"); + } +__METHODS__ +} +'''.replace("__METHODS__", methods) +with tempfile.TemporaryDirectory(prefix="rea-ghidra-handshake-") as directory: + source = pathlib.Path(directory) / "ReaHandshakeProbe.java" + source.write_text(harness) + subprocess.run([sys.argv[1] if len(sys.argv) > 1 else "java", str(source)], + check=True, timeout=30)