reverse-engineering-anything/scripts/verify-qwen-client.mjs

import assert from "node:assert/strict";
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
import {
  mkdir,
  writeFile,
  readFile,
  mkdtemp,
  realpath,
  symlink,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import { createVerifierCommands } from "./lib/verify-commands.mjs";
import { saveVerifierCommandOutput } from "./lib/verifier-command-output.mjs";
import { withVerifierCleanup } from "./lib/verifier-cleanup.mjs";
import {
  CLIENT_JAVASCRIPT_FIXTURE_SOURCE,
  assertClientJavaScriptFixtureEvidence,
} from "./verify/clients/javascript-fixture.mjs";
import {
  findClientResult,
  isClientJavaScriptEvidence,
} from "./verify/clients/result-discovery.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";

const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
const client = "qwen";
const mode = process.argv[2] ?? "call";
assert(
  ["chat", "call"].includes(mode),
  "Usage: node scripts/verify-qwen-client.mjs [chat|call]",
);
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
// npm exposes only qwen.cmd on Windows. Resolve a selected JavaScript entry
// before the verifier switches to its disposable workspace.
const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined;
const reducedPath = process.env.REA_VERIFY_QWEN_REDUCED_PATH === "1";
assert(
  !reducedPath || (process.platform !== "win32" && entryPoint !== undefined),
  "REA_VERIFY_QWEN_REDUCED_PATH=1 requires POSIX and a JavaScript REA_VERIFY_QWEN_COMMAND entry point",
);
const launch = (args) =>
  entryPoint === undefined
    ? [command, args]
    : [process.execPath, [entryPoint, ...args]];
const verifier = createVerifierRun();
const profile = join(lab, "selected profile");
const account = join(lab, "account");
const runtime = join(profile, "runtime");
const workspace = join(lab, "workspace");
const sharedSkills = join(account, ".agents", "skills");
let clientVersion = null;
let fixture;
let passed = false;
let evidenceSeen = false;
let skillSeen = false;
let finalSeen = false;
let skillRequested = false;
let searchCallId;
let analysisCallId;
let artifactReadByAgent = false;
let artifactPath;
let selectedSkillLoaded = false;
let artifactDigest;
let catalogSize = null;
let registrationCommand = null;
let clientPath = null;
const commands = await createVerifierCommands();
await withVerifierCleanup(async () => {
  await withVerifierCleanup(async () => {
    await Promise.all([
      mkdir(profile, { recursive: true }),
      mkdir(account),
      mkdir(workspace),
    ]);
    const baseEnvironment = {
      PATH: process.env.PATH ?? "",
      HOME: account,
      USERPROFILE: account,
      QWEN_HOME: profile,
      QWEN_RUNTIME_DIR: runtime,
      REA_PROCESS_RUN_ID: verifier.run_id,
    };
    try {
      clientVersion = (
        await commands.exec(...launch(["--version"]), {
          env: baseEnvironment,
          timeout: 30_000,
          stdin: "pipe",
        })
      ).stdout.trim();
    } catch (cause) {
      const windowsHint =
        process.platform === "win32"
          ? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js."
          : "";
      throw new Error(
        `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`,
        { cause },
      );
    }
    const original = JSON.stringify(
      {
        security: {
          auth: { selectedType: "openai" },
          folderTrust: { enabled: false },
        },
        telemetry: { enabled: false },
        model: { name: "rea-client-fixture" },
        mcpServers: { other: { command: "unrelated-server", disabled: true } },
      },
      null,
      2,
    );
    await writeFile(join(profile, "settings.json"), original);
    for (const step of [
      ["setup", "--client", "qwen_code", "--dry-run"],
      ["setup", "--client", "qwen_code", "--yes"],
      ["doctor", "--client", "qwen_code", "--skill"],
    ]) {
      const response = await commands.exec(
        process.execPath,
        [join(runtimeRoot, "scripts/rea.mjs"), ...step, "--json"],
        { env: baseEnvironment, cwd: repo, timeout: 60_000, stdin: "pipe" },
      );
      const result = JSON.parse(response.stdout);
      if (step.includes("--dry-run"))
        assert(
          result.plannedActions.every(
            (a) =>
              a.target.startsWith(profile + sep) ||
              a.target.startsWith(sharedSkills + sep),
          ),
        );
      await writeFile(
        join(
          lab,
          `${step[0]}${step.includes("--dry-run") ? "-plan" : ""}.json`,
        ),
        response.stdout,
      );
    }
    assert.equal(
      await readFile(join(profile, "settings.json.rea.backup"), "utf8"),
      original,
    );
    const configured = JSON.parse(
      await readFile(join(profile, "settings.json"), "utf8"),
    );
    registrationCommand = [
      configured.mcpServers.rea.command,
      ...configured.mcpServers.rea.args,
    ];
    assert.deepEqual(configured.model, { name: "rea-client-fixture" });
    assert.equal(configured.skills, undefined);
    assert.deepEqual(configured.mcpServers.other, {
      command: "unrelated-server",
      disabled: true,
    });
    clientPath = baseEnvironment.PATH;
    if (reducedPath) {
      assert.equal(
        configured.mcpServers.rea.env?.PATH,
        undefined,
        "Setup must not persist its ambient PATH into the registration",
      );
      clientPath = join(lab, "client-bin");
      await mkdir(clientPath);
      // Qwen's Evidence readback uses its shell tool, which launches bare bash.
      // Keep that fixture dependency available without exposing node/npm/npx.
      const bash = await commands.exec("bash", ["-c", "command -v bash"], {
        env: baseEnvironment,
        timeout: 5_000,
      });
      await symlink(
        await realpath(bash.stdout.trim()),
        join(clientPath, "bash"),
      );
      await commands.exec(
        process.execPath,
        [
          "-e",
          "const {spawnSync}=require('node:child_process');const assert=require('node:assert/strict');for(const command of ['node','npm','npx'])assert.equal(spawnSync(command,['--version'],{timeout:1000}).error?.code,'ENOENT',`${command} unexpectedly available on reduced PATH`);",
        ],
        {
          env: { ...baseEnvironment, PATH: clientPath },
          timeout: 5_000,
        },
      );
    }
    const target = join(lab, "fixture α测试");
    await mkdir(target, { recursive: true });
    await writeFile(`${target}/app.js`, CLIENT_JAVASCRIPT_FIXTURE_SOURCE);
    fixture = await createOpenAiModelFixture({
      directory: lab,
      prefix: `${client}-${mode}`,
      onRequest: async ({ body, tools, requestIndex }) => {
        skillSeen ||= JSON.stringify(body.messages ?? []).includes(
          "reverse-engineer-anything",
        );
        const messages = (body.messages ?? []).map((message) => ({
          role: message.role,
          callId: message.tool_call_id,
          text:
            typeof message.content === "string"
              ? message.content
              : (message.content ?? [])
                  .map((part) => part.text ?? "")
                  .join("\n"),
        }));
        const messageText = messages.map((message) => message.text).join("\n");
        const toolResponseText = (callId) =>
          messages.find(
            (message) => message.role === "tool" && message.callId === callId,
          )?.text ?? "";
        selectedSkillLoaded ||= messageText.includes(
          join(sharedSkills, "reverse-engineer-anything"),
        );
        const toolText = JSON.stringify(
          (body.messages ?? []).filter((m) => m.role === "tool"),
        );
        if (
          toolText.includes("REA_CLIENT_FIXTURE") &&
          /(?:\\?"REA_CLIENT_FIXTURE\\?"\s*:\s*true)/u.test(toolText)
        ) {
          artifactReadByAgent = true;
          assert(artifactPath);
          const artifact = await readFile(artifactPath);
          artifactDigest = createHash("sha256").update(artifact).digest("hex");
          const { evidence } = await assertClientJavaScriptFixtureEvidence(
            JSON.parse(artifact.toString("utf8")),
            target,
          );
          evidenceSeen = true;
          await writeFile(
            `${lab}/${client}-${mode}-evidence.json`,
            JSON.stringify(evidence),
          );
        }
        for (const message of body.messages ?? []) {
          if (message.role !== "tool") continue;
          const candidate = findClientResult(
            message.content,
            isClientJavaScriptEvidence,
          );
          if (candidate) {
            const { evidence } = await assertClientJavaScriptFixtureEvidence(
              candidate,
              target,
            );
            evidenceSeen = true;
            await writeFile(
              `${lab}/${client}-${mode}-evidence.json`,
              JSON.stringify(evidence),
            );
          }
        }
        const selected = tools.find((t) =>
          t.name.includes("analyze_javascript_application"),
        );
        const reviewedNames = [
          ...toolResponseText(searchCallId).matchAll(
            /(?:\\?"name\\?"\s*:\s*\\?")([^"\\]*analyze_javascript_application)(?:\\?")/g,
          ),
        ]
          .map((m) => m[1])
          .sort((a, b) => b.length - a.length);
        const search = tools.find((t) => t.name === "tool_search");
        const invoke = tools.find((t) => t.name === "tool_call");
        const skill = tools.find((t) => t.name === "skill");
        const shell = tools.find((t) => t.name === "run_shell_command");
        const persisted = toolResponseText(analysisCallId).match(
          /Full output saved to: ([^\r\n]+)/u,
        );
        const call = (name, args) => ({
          role: "assistant",
          content: null,
          tool_calls: [
            {
              id: `call_rea_fixture_${requestIndex}`,
              type: "function",
              function: { name, arguments: JSON.stringify(args) },
            },
          ],
        });
        let message;
        if (mode === "chat" || evidenceSeen) {
          message = {
            role: "assistant",
            content: "REA_CLIENT_COMPATIBILITY_OK",
          };
          finalSeen = true;
        } else if (skill && !skillRequested) {
          skillRequested = true;
          message = call(skill.name, { skill: "reverse-engineer-anything" });
        } else if (persisted && shell) {
          artifactPath = persisted[1];
          assert(
            (await realpath(artifactPath)).startsWith(
              (await realpath(runtime)) + sep,
            ),
          );
          const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
          // Keep paths and JSON quotes out of shell syntax on Windows and POSIX.
          const encodedScript = Buffer.from(script).toString("base64");
          const nodeCommand =
            process.platform === "win32"
              ? "node"
              : `'${process.execPath.replaceAll("'", "'\"'\"'")}'`;
          message = call(shell.name, {
            command: `${nodeCommand} -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`,
            description:
              "Read the complete saved REA Evidence and verify the fixture facts.",
          });
        } else if (selected) {
          message = call(selected.name, {
            input_path: target,
            format: "directory",
          });
          analysisCallId = message.tool_calls[0].id;
        } else if (reviewedNames.length && invoke) {
          message = call(invoke.name, {
            name: reviewedNames[0],
            arguments: { input_path: target, format: "directory" },
          });
          analysisCallId = message.tool_calls[0].id;
        } else if (search && searchCallId === undefined) {
          message = call(search.name, {
            query: "analyze_javascript_application",
          });
          searchCallId = message.tool_calls[0].id;
        } else
          throw new Error(
            `REA analysis tool not exposed; tools: ${tools.map((t) => t.name).join(", ")}`,
          );
        return message;
      },
    });
    const endpoint = fixture.baseUrl;
    const prompt =
      mode === "chat"
        ? "Say hello briefly."
        : `Use REA to analyze the JavaScript directory ${target}. Report the result.`;
    const env = {
      ...baseEnvironment,
      PATH: clientPath,
      OPENAI_API_KEY: "local-fixture-key",
      OPENAI_BASE_URL: endpoint,
    };
    const args = [
      "--auth-type",
      "openai",
      "--openai-base-url",
      endpoint,
      "--model",
      "rea-client-fixture",
      "--approval-mode",
      mode === "chat" ? "default" : "yolo",
      "--allowed-mcp-server-names",
      "rea",
      "--output-format",
      "stream-json",
      "--max-session-turns",
      "8",
      "-p",
      prompt,
    ];
    const response = await saveVerifierCommandOutput(
      () =>
        commands.exec(...launch(args), {
          env,
          cwd: workspace,
          timeout: 120_000,
          maxBuffer: 10 * 1024 * 1024,
          stdin: "pipe",
        }),
      {
        stdout: join(lab, `${mode}.stdout`),
        stderr: join(lab, `${mode}.stderr`),
      },
    );
    const initialized = response.stdout
      .split("\n")
      .filter(Boolean)
      .map((line) => JSON.parse(line))
      .find((v) => v.type === "system" && v.subtype === "init");
    assert(initialized);
    const observed = initialized.tools
      .filter((n) => n.startsWith("mcp__rea__"))
      .sort();
    catalogSize = observed.length;
    assert.deepEqual(
      observed,
      TOOL_CONTRACTS.map((c) => `mcp__rea__${c.name}`).sort(),
      "Qwen must discover the complete REA catalog",
    );
    assert.equal(fixture.failure, undefined);
    assert(finalSeen);
    if (mode === "call") {
      assert(evidenceSeen);
      assert(selectedSkillLoaded);
      assert(artifactReadByAgent);
    }
  }, [
    () => commands.close(),
    async () => {
      await fixture?.close();
    },
  ]);
  passed = true;
}, [
  async () => {
    const lineage = await completeVerifierRun(verifier);
    const receipt = {
      client: "Qwen Code",
      clientVersion,
      mode,
      reducedPath,
      ...(reducedPath ? { clientPath } : {}),
      registrationCommand,
      passed,
      artifacts: lab,
      runtimeRoot,
      host: {
        platform: process.platform,
        architecture: process.arch,
        node: process.versions.node,
      },
      modelFixture:
        "loopback deterministic OpenAI-compatible endpoint; no live model-provider claim",
      skillDiscovery:
        "native HOME shared personal directory; no explicit skill setting",
      catalogSize,
      requests: fixture?.requests,
      probes: fixture?.probes,
      evidenceSeen,
      selectedSkillLoaded,
      artifactReadByAgent,
      artifactPath,
      artifactDigest,
      finalSeen,
      fixtureFailure: fixture?.failure,
      verifier: lineage,
    };
    await withVerifierCleanup(
      () =>
        writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2)),
      [
        () => {
          console.log(JSON.stringify(receipt));
        },
      ],
    );
  },
]);