From 6060de329de9f3636f63e24a2419847fd9d3f889 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 14 Oct 2026 21:51:19 +0800 Subject: [PATCH] fix(release): rehearse qualified packages before publication --- .../prepare-release-package/action.yml | 26 +++ .github/workflows/release-recovery.yml | 4 + .github/workflows/release.yml | 99 ++++++----- docs/releasing.md | 20 +++ scripts/ci/npm-release.d.mts | 27 +++ scripts/ci/npm-release.mjs | 126 +++++++++++++ scripts/ci/release-recovery.mjs | 28 +-- scripts/ci/scopes.mjs | 6 + scripts/ci/verified-package.mjs | 25 ++- scripts/lib/release-package-canary.mjs | 120 +++++++++++++ scripts/release-npm-tag.mjs | 18 +- scripts/verify-published-package.mjs | 150 ++++------------ scripts/verify-release-artifact.mjs | 47 +++++ .../filesystem/ciVerifiedPackage.test.ts | 3 +- tests/boundary/filesystem/npmRelease.test.ts | 132 ++++++++++++++ .../filesystem/releaseRecovery.test.ts | 19 +- .../filesystem/releaseWorkflow.test.ts | 122 ++++++++++--- .../process/releaseArtifactCanary.test.ts | 166 ++++++++++++++++++ 18 files changed, 925 insertions(+), 213 deletions(-) create mode 100644 .github/actions/prepare-release-package/action.yml create mode 100644 scripts/ci/npm-release.d.mts create mode 100644 scripts/ci/npm-release.mjs create mode 100644 scripts/lib/release-package-canary.mjs create mode 100644 scripts/verify-release-artifact.mjs create mode 100644 tests/boundary/filesystem/npmRelease.test.ts create mode 100644 tests/boundary/process/releaseArtifactCanary.test.ts diff --git a/.github/actions/prepare-release-package/action.yml b/.github/actions/prepare-release-package/action.yml new file mode 100644 index 0000000000..7f0bc99383 --- /dev/null +++ b/.github/actions/prepare-release-package/action.yml @@ -0,0 +1,26 @@ +name: Prepare qualified release inputs +description: Restore and validate the exact qualified tarball and its verifier runtime +outputs: + version: + description: Package version validated against the source and qualified manifest + value: ${{ steps.package.outputs.version }} + tarball: + description: Absolute path to the validated qualified npm tarball + value: ${{ steps.package.outputs.tarball }} +runs: + using: composite + steps: + - name: Download the exact qualified package + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: verified-release-package + path: .cache/verified-package/ + - name: Verify qualified package provenance and digest + id: package + shell: bash + run: node scripts/ci/verified-package.mjs validate .cache/verified-package + - name: Restore qualified verifier runtime + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: rea-test-runtime + path: . diff --git a/.github/workflows/release-recovery.yml b/.github/workflows/release-recovery.yml index befb77310d..07a6a2164c 100644 --- a/.github/workflows/release-recovery.yml +++ b/.github/workflows/release-recovery.yml @@ -74,6 +74,10 @@ jobs: RECOVERY_SOURCE_DIRECTORY: .cache/release-source RECOVERY_ARTIFACT_DIRECTORY: .cache/verified-package run: node scripts/ci/release-recovery.mjs verify + - name: Rehearse the original qualified tarball + env: + RECOVERY_VERSION: ${{ steps.release.outputs.version }} + run: node scripts/verify-release-artifact.mjs .cache/verified-package/package.tgz "$RECOVERY_VERSION" - name: Verify published CLI, MCP and update env: RECOVERY_VERSION: ${{ steps.release.outputs.version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ddfbbd00bd..a1ae284e64 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -304,12 +304,41 @@ jobs: with: source_sha: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }} - publish: - name: Publish to npm + rehearsal: + name: Rehearse qualified release on a clean runner needs: [merged-release, release-please, qualification] if: >- always() && !cancelled() && needs.qualification.result == 'success' runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + actions: read + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + ref: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }} + persist-credentials: false + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: .nvmrc + package-manager-cache: false + - run: npm ci + - uses: ./.github/actions/prepare-release-package + id: package + - name: Exercise the exact qualified tarball + env: + VERIFIED_PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + VERIFIED_PACKAGE_VERSION: ${{ steps.package.outputs.version }} + run: node scripts/verify-release-artifact.mjs "$VERIFIED_PACKAGE_TARBALL" "$VERIFIED_PACKAGE_VERSION" + + publish: + name: Publish to npm + needs: [merged-release, release-please, qualification, rehearsal] + if: >- + always() && !cancelled() && + needs.qualification.result == 'success' && needs.rehearsal.result == 'success' + runs-on: ubuntu-latest permissions: contents: read id-token: write @@ -327,62 +356,36 @@ jobs: - run: npm ci - - name: Download the exact qualified package - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: verified-release-package - path: .cache/verified-package/ - - - name: Verify qualified package provenance and digest + - uses: ./.github/actions/prepare-release-package id: package - run: node scripts/ci/verified-package.mjs validate .cache/verified-package - - name: Restore qualified verifier runtime - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: rea-test-runtime - path: . + - name: Inspect existing npm version + id: registry + env: + VERIFIED_PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + VERIFIED_PACKAGE_VERSION: ${{ steps.package.outputs.version }} + run: node scripts/ci/npm-release.mjs inspect "$VERIFIED_PACKAGE_TARBALL" "$VERIFIED_PACKAGE_VERSION" - name: Publish + if: steps.registry.outputs.published == 'false' shell: bash env: VERIFIED_PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + VERIFIED_PACKAGE_VERSION: ${{ steps.package.outputs.version }} run: | - version="$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)')" - published_version="$(npm view "rea-agents@${version}" version --json 2>/dev/null || true)" - if [ "${published_version}" = "\"${version}\"" ]; then - echo "rea-agents@${version} is already published; verifying the existing package." - else - tag="$(node scripts/release-npm-tag.mjs "${version}")" - npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}" - fi + tag="$(node scripts/release-npm-tag.mjs "$VERIFIED_PACKAGE_VERSION")" + npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}" + + - name: Verify published npm bytes and dist-tag + env: + VERIFIED_PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} + VERIFIED_PACKAGE_VERSION: ${{ steps.package.outputs.version }} + run: node scripts/ci/npm-release.mjs verify "$VERIFIED_PACKAGE_TARBALL" "$VERIFIED_PACKAGE_VERSION" - name: Verify published CLI from npm - shell: bash - run: | - version="$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)')" - verification_root="$(mktemp -d)" - trap 'rm -rf -- "${verification_root}"' EXIT - cd "${verification_root}" - max_attempts=120 - sleep_seconds=15 - total_window_minutes=$((max_attempts * sleep_seconds / 60)) - for attempt in $(seq 1 "${max_attempts}"); do - published_version="$(npm view --prefer-online "rea-agents@${version}" version --json 2>/dev/null || true)" - if [ "${published_version}" = "\"${version}\"" ]; then - echo "rea-agents@${version} is visible on npm after attempt ${attempt}/${max_attempts}; verifying the published CLI." - node "${GITHUB_WORKSPACE}/scripts/verify-published-package.mjs" "${version}" - exit 0 - else - echo "rea-agents@${version} is not visible on npm yet (attempt ${attempt}/${max_attempts}; up to ${total_window_minutes} minutes)" >&2 - fi - if [ "${attempt}" -lt "${max_attempts}" ]; then - sleep "${sleep_seconds}" - fi - done - echo "rea-agents@${version} did not become visible on npm within ${total_window_minutes} minutes (${max_attempts} attempts × ${sleep_seconds}s)" >&2 - echo "This indicates npm registry propagation delay, not a CLI verification failure." >&2 - exit 1 + env: + VERIFIED_PACKAGE_VERSION: ${{ steps.package.outputs.version }} + run: node scripts/verify-published-package.mjs "$VERIFIED_PACKAGE_VERSION" publish-mcp: name: Publish to MCP Registry diff --git a/docs/releasing.md b/docs/releasing.md index 644edbe7ae..5b11f79ae4 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -10,6 +10,26 @@ SHA returned by Release Please. Before either registry is published, the release run calls full CI qualification at that exact revision, including coverage, portable providers, and every native package host. Later main commits cannot change or cancel that proof. +After qualification, a separate runner with read-only repository/actions +permissions restores the retained tarball and verifier runtime. It installs +that exact archive into an empty prefix, then exercises CLI help/discovery and +the real stdio MCP catalog/session contract. Installation finishes before the +protocol deadline begins, with dependency lifecycle scripts disabled. Only a +successful rehearsal enables the job with npm publishing authority. Both jobs +use the same release-input action; neither rebuilds or repacks the archive. +This rehearsal checks package consumption on the Linux release runner; +qualification retains its broader native-host and real-provider checks. + +Before publishing, npm lookup failures and malformed metadata stop the job. +An existing version must match the qualified tarball's SHA-512 and intended +`latest`/`next` dist-tag before it can be reused. Publication never overwrites +an existing version or silently changes a dist-tag. After publication, only +confirmed registry absence is retried within the propagation window. Content +or tag mismatches fail immediately. The post-publication verifier installs +from npm and repeats the same CLI/MCP checks; it also exercises a real update +from an older stable release. For a prerelease on `next`, that updater still +targets stable `latest`, as the public update command does. + The required reverse-tools lane runs fresh coverage and both mutation preflights. Complete mutation sweeps run separately on a schedule or manual dispatch and do not block publication; see [the testing guide](testing.md). diff --git a/scripts/ci/npm-release.d.mts b/scripts/ci/npm-release.d.mts new file mode 100644 index 0000000000..1d70d3c33b --- /dev/null +++ b/scripts/ci/npm-release.d.mts @@ -0,0 +1,27 @@ +export type NpmReleaseState = + | { status: "absent" } + | { + status: "published"; + integrity: string; + tag: "latest" | "next"; + taggedVersion: string | null; + stableVersion: string | null; + }; +export function readNpmRelease( + version: string, + request?: (url: string, init: RequestInit) => Promise, +): Promise; +export function requireMatchingNpmRelease( + tarball: string, + version: string, + published: NpmReleaseState, +): Promise; +export function stableUpdateVersion( + version: string, + published: NpmReleaseState, +): string; +export function waitForNpmRelease( + version: string, + lookup?: (version: string) => Promise, + pause?: (milliseconds: number) => Promise, +): Promise>; diff --git a/scripts/ci/npm-release.mjs b/scripts/ci/npm-release.mjs new file mode 100644 index 0000000000..8bd0f1464b --- /dev/null +++ b/scripts/ci/npm-release.mjs @@ -0,0 +1,126 @@ +import { appendFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { setTimeout } from "node:timers/promises"; +import { z } from "zod"; +import { prerelease, valid } from "semver"; +import { releaseNpmTag } from "../release-npm-tag.mjs"; +import { verifyPublishedPackageIntegrity } from "./verified-package.mjs"; + +/** Parse one npm metadata snapshot; errors and malformed records are never absence. */ +export async function readNpmRelease(version, request = fetch) { + const tag = releaseNpmTag(version); + const response = await request("https://registry.npmjs.org/rea-agents", { + headers: { + Accept: "application/vnd.npm.install-v1+json", + "Cache-Control": "no-cache", + }, + signal: AbortSignal.timeout(30_000), + }); + if (response.status === 404) return { status: "absent" }; + if (!response.ok) + throw new Error(`npm registry lookup failed: HTTP ${response.status}`); + const metadata = z + .object({ + name: z.literal("rea-agents"), + versions: z.record(z.string(), z.unknown()), + "dist-tags": z.record(z.string(), z.string()), + }) + .parse(await response.json()); + if (!Object.hasOwn(metadata.versions, version)) return { status: "absent" }; + const published = z + .object({ + name: z.literal("rea-agents"), + version: z.literal(version), + dist: z.object({ integrity: z.string().min(1) }), + }) + .parse(metadata.versions[version]); + return { + status: "published", + integrity: published.dist.integrity, + tag, + taggedVersion: metadata["dist-tags"][tag] ?? null, + stableVersion: metadata["dist-tags"].latest ?? null, + }; +} + +/** REA update follows stable latest even when the candidate is published on next. */ +export function stableUpdateVersion(version, published) { + if (published.status !== "published" || published.taggedVersion !== version) + throw new Error(`npm does not select ${version} on its release channel`); + const stable = published.stableVersion; + if ( + typeof stable !== "string" || + valid(stable) !== stable || + prerelease(stable) !== null + ) + throw new Error( + "npm latest does not select a stable release for the update canary", + ); + return stable; +} + +/** Validate content before accepting a previously published or newly visible version. */ +export async function requireMatchingNpmRelease(tarball, version, published) { + if (published.status !== "published") + throw new Error(`rea-agents@${version} is not published`); + await verifyPublishedPackageIntegrity(tarball, published.integrity); + if (published.taggedVersion !== version) + throw new Error( + `Expected npm ${published.tag} to select ${version}, observed ${published.taggedVersion ?? "missing"}; refusing to change an existing dist-tag`, + ); +} + +/** Retry only confirmed absence during the existing registry propagation window. */ +export async function waitForNpmRelease( + version, + lookup = readNpmRelease, + pause = setTimeout, +) { + const attempts = 120; + for (let attempt = 1; attempt <= attempts; attempt++) { + const published = await lookup(version); + if (published.status === "published") return published; + if (attempt < attempts) await pause(15_000); + } + throw new Error( + `rea-agents@${version} was not visible on npm after 120 attempts over 30 minutes`, + ); +} + +async function main() { + const [operation, tarball, version, ...extra] = process.argv.slice(2); + if ( + !["inspect", "verify"].includes(operation) || + tarball === undefined || + version === undefined || + extra.length + ) + throw new Error( + "Usage: node scripts/ci/npm-release.mjs inspect|verify ", + ); + const published = + operation === "inspect" + ? await readNpmRelease(version) + : await waitForNpmRelease(version); + if (published.status === "published") + await requireMatchingNpmRelease(tarball, version, published); + if (operation === "inspect") { + await appendFile( + z.string().min(1).parse(process.env.GITHUB_OUTPUT), + `published=${published.status === "published"}\n`, + ); + } else if (process.env.GITHUB_STEP_SUMMARY !== undefined) { + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + `npm: rea-agents@${version} is published, matches the qualified tarball, and is selected by ${published.tag}.\n`, + ); + } + process.stdout.write(`${JSON.stringify({ version, ...published })}\n`); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) + await main(); diff --git a/scripts/ci/release-recovery.mjs b/scripts/ci/release-recovery.mjs index cb5046ba00..fefaa25d35 100644 --- a/scripts/ci/release-recovery.mjs +++ b/scripts/ci/release-recovery.mjs @@ -1,6 +1,4 @@ import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { createReadStream } from "node:fs"; import { appendFile, readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -8,7 +6,10 @@ import { promisify } from "node:util"; import { valid } from "semver"; import { z } from "zod"; -import { validateVerifiedPackage } from "./verified-package.mjs"; +import { + validateVerifiedPackage, + verifyPublishedPackageIntegrity, +} from "./verified-package.mjs"; const execute = promisify(execFile); const sha = z.string().regex(/^[a-f0-9]{40}$/u); @@ -81,8 +82,18 @@ export function selectRecoveryRelease({ throw new Error(`Original release did not pass ${name}`); const publish = requireJob("Publish to npm"); const writes = publish.steps.filter((step) => step.name === "Publish"); - if (writes.length !== 1 || writes[0].conclusion !== "success") - throw new Error("Original npm publication did not succeed"); + const verified = publish.steps.filter( + (step) => step.name === "Verify published npm bytes and dist-tag", + ); + if ( + !( + (writes.length === 1 && writes[0].conclusion === "success") || + (verified.length === 1 && verified[0].conclusion === "success") + ) + ) + throw new Error( + "Original npm publication did not succeed or verify existing bytes", + ); return { version, source_sha: tagSha, @@ -108,12 +119,7 @@ export async function verifyRecoveryPackage({ packageVersion: version, nodeVersion: process.version, }); - const hash = createHash("sha512"); - for await (const chunk of createReadStream(tarball)) hash.update(chunk); - if (integrity !== `sha512-${hash.digest("base64")}`) - throw new Error( - "Published npm integrity differs from the original qualified package", - ); + await verifyPublishedPackageIntegrity(tarball, integrity); return { version, sourceSha, runId, runAttempt, integrity }; } diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index 17a0c8e0ff..3b237a7bf0 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -91,12 +91,18 @@ export const SCOPES = { // Four-host installed-package matrix and the Windows curated lane. package: [ ".github/actions/publish-mcp/**", + ".github/actions/prepare-release-package/**", ".github/workflows/release-recovery.yml", "scripts/ci/release-recovery.mjs", "scripts/ci/verified-package.mjs", + "scripts/ci/npm-release.*", + "scripts/verify-release-artifact.mjs", + "scripts/lib/release-package-canary.mjs", "scripts/verify-published-package.mjs", "tests/boundary/filesystem/releaseRecovery.test.ts", "tests/boundary/filesystem/mcpRegistryPublication.test.ts", + "tests/boundary/filesystem/npmRelease.test.ts", + "tests/boundary/process/releaseArtifactCanary.test.ts", "src/windows/**", "native/windows/**", "src/process/capture/FilesystemSnapshot.ts", diff --git a/scripts/ci/verified-package.mjs b/scripts/ci/verified-package.mjs index 87d74bab84..444289bab2 100644 --- a/scripts/ci/verified-package.mjs +++ b/scripts/ci/verified-package.mjs @@ -5,6 +5,7 @@ import { copyFile, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { valid } from "semver"; const MANIFEST_NAME = "manifest.json"; const TARBALL_NAME = "package.tgz"; @@ -155,6 +156,8 @@ function validateManifestShape(manifest) { ]) if (typeof manifest[key] !== "string" || manifest[key].length === 0) throw new Error(`Verified package manifest has an invalid ${key}`); + if (valid(manifest.packageVersion) !== manifest.packageVersion) + throw new Error("Verified package manifest has an invalid package version"); if ( typeof manifest.sha256 !== "string" || !/^[a-f0-9]{64}$/u.test(manifest.sha256) @@ -173,6 +176,16 @@ async function sha256File(path) { return hash.digest("hex"); } +/** Require the registry to serve the exact archive selected for publication. */ +export async function verifyPublishedPackageIntegrity(tarball, integrity) { + const hash = createHash("sha512"); + for await (const chunk of createReadStream(tarball)) hash.update(chunk); + if (integrity !== `sha512-${hash.digest("base64")}`) + throw new Error( + "Published npm integrity differs from the original qualified package", + ); +} + async function validateFromCli(directory) { const sourceSha = process.env.GITHUB_SHA; const runId = parsePositiveInteger( @@ -202,10 +215,14 @@ async function validateFromCli(directory) { throw new Error( "GITHUB_OUTPUT is required to publish the validated tarball path", ); - await writeFile(process.env.GITHUB_OUTPUT, `tarball=${tarball}\n`, { - encoding: "utf8", - flag: "a", - }); + await writeFile( + process.env.GITHUB_OUTPUT, + `tarball=${tarball}\nversion=${packageJson.version}\n`, + { + encoding: "utf8", + flag: "a", + }, + ); process.stdout.write(`${tarball}\n`); } diff --git a/scripts/lib/release-package-canary.mjs b/scripts/lib/release-package-canary.mjs new file mode 100644 index 0000000000..1270f3282e --- /dev/null +++ b/scripts/lib/release-package-canary.mjs @@ -0,0 +1,120 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { execFile } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { CATALOG_IDENTITY } from "../../dist/catalogIdentity.js"; +import { TOOL_CONTRACTS } from "../../dist/contracts/toolContracts.js"; +import { + requireMcpLifecycleResult, + sortVerifierNames, +} from "./mcp-verifier-results.mjs"; + +const execute = promisify(execFile); + +/** Install the selected archive or registry specifier before any protocol deadline. */ +export async function installCanaryPackage(specifier, workspace, environment) { + const prefix = join(workspace, "installed"); + await execute( + "npm", + [ + "install", + "--ignore-scripts", + "--no-save", + "--no-package-lock", + "--no-audit", + "--no-fund", + "--prefix", + prefix, + specifier, + ], + { cwd: workspace, env: environment, maxBuffer: 16 * 1024 * 1024 }, + ); + return join(prefix, "node_modules", "rea-agents"); +} + +/** Exercise the same installed CLI/MCP contract before and after publication. */ +export async function verifyInstalledCanaryPackage( + packageRoot, + version, + workspace, + environment, +) { + const installed = JSON.parse( + await readFile(join(packageRoot, "package.json"), "utf8"), + ); + if (installed.name !== "rea-agents" || installed.version !== version) + throw new Error( + `Installed package identity is ${installed.name}@${installed.version}, expected rea-agents@${version}`, + ); + const launcher = join(packageRoot, "scripts", "rea.mjs"); + for (const option of ["--help", "--llms"]) + await execute(process.execPath, [launcher, option], { + cwd: workspace, + env: environment, + maxBuffer: 16 * 1024 * 1024, + }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [launcher, "mcp"], + cwd: workspace, + env: environment, + stderr: "pipe", + }); + const client = new Client({ name: "release-package-canary", version: "1" }); + let stderr = ""; + transport.stderr?.on("data", (chunk) => { + if (stderr.length < 16_384) + stderr += chunk.toString("utf8").slice(0, 16_384 - stderr.length); + }); + try { + await client.connect(transport); + const tools = await client.listTools(); + const toolNames = tools.tools.map(({ name }) => name); + const canonicalToolNames = new Set(TOOL_CONTRACTS.map(({ name }) => name)); + const publishedToolNames = new Set(toolNames); + const unknown = toolNames.filter((name) => !canonicalToolNames.has(name)); + const missing = [...canonicalToolNames].filter( + (name) => !publishedToolNames.has(name), + ); + if ( + toolNames.length === 0 || + publishedToolNames.size !== toolNames.length || + unknown.length || + missing.length + ) + throw new Error( + `Installed MCP exposed an invalid canonical tool catalog (${toolNames.length} tools, unknown: ${unknown.join(", ") || "none"}, missing: ${missing.join(", ") || "none"})`, + ); + const status = requireMcpLifecycleResult( + await client.callTool({ name: "binary_session", arguments: {} }), + "binary_session", + ); + const availability = sortVerifierNames( + status.tool_availability.map(({ name }) => name), + ); + if ( + JSON.stringify(availability) !== + JSON.stringify(sortVerifierNames(canonicalToolNames)) + ) + throw new Error( + "Installed binary_session omitted canonical tool availability", + ); + return { + mcpTools: toolNames.length, + canonicalMcpTools: CATALOG_IDENTITY.counts.mcp_tools, + }; + } catch (cause) { + throw new Error(`Installed package MCP verification failed: ${stderr}`, { + cause, + }); + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } +} diff --git a/scripts/release-npm-tag.mjs b/scripts/release-npm-tag.mjs index 37766a4ecf..ad85b9798c 100644 --- a/scripts/release-npm-tag.mjs +++ b/scripts/release-npm-tag.mjs @@ -1,9 +1,15 @@ import semver from "semver"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; -const version = process.argv[2]; -if (version === undefined || semver.valid(version) !== version) { - console.error("Expected an exact SemVer version."); - process.exitCode = 1; -} else { - process.stdout.write(semver.prerelease(version) === null ? "latest" : "next"); +export function releaseNpmTag(version) { + if (version === undefined || semver.valid(version) !== version) + throw new Error("Expected an exact SemVer version."); + return semver.prerelease(version) === null ? "latest" : "next"; } + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) + process.stdout.write(releaseNpmTag(process.argv[2])); diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs index c1ae6e2087..646b62e25c 100644 --- a/scripts/verify-published-package.mjs +++ b/scripts/verify-published-package.mjs @@ -1,5 +1,3 @@ -import { Client } from "@modelcontextprotocol/client"; -import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { execFile } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -7,20 +5,23 @@ import { delimiter, join } from "node:path"; import { promisify } from "node:util"; import { pathToFileURL } from "node:url"; -import { lt, rcompare } from "semver"; +import { lt, prerelease, rcompare, valid } from "semver"; +import { readNpmRelease, stableUpdateVersion } from "./ci/npm-release.mjs"; -import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; -import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { - requireMcpLifecycleResult, - sortVerifierNames, -} from "./lib/mcp-verifier-results.mjs"; + installCanaryPackage, + verifyInstalledCanaryPackage, +} from "./lib/release-package-canary.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; const verifierRun = createVerifierRun(); const execFileAsync = promisify(execFile); const version = process.argv[2]; -if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u.test(version ?? "")) +if ( + typeof version !== "string" || + valid(version) !== version || + process.argv.length !== 3 +) throw new Error("Usage: node scripts/verify-published-package.mjs "); const canaryRoot = await mkdtemp(join(tmpdir(), "rea-published-canary-")); @@ -29,114 +30,30 @@ const serverEnvironment = { npm_config_cache: join(canaryRoot, "npm-cache"), }; delete serverEnvironment.HOPPER_TARGET_PATH; -const client = new Client({ - name: "published-package-canary", - version: "1", -}); -let stderr = ""; -let transport; - -let publishedToolCount = 0; +let surface; let update; - try { - // Registry installation has its own lifetime; it must finish before the - // MCP handshake deadline starts. Reuse this exact installation for update. - const publishedPrefix = join(canaryRoot, "published"); - await execFileAsync( - "npm", - [ - "install", - "--ignore-scripts", - "--no-save", - "--no-package-lock", - "--prefix", - publishedPrefix, - `rea-agents@${version}`, - ], - { cwd: canaryRoot, env: serverEnvironment, maxBuffer: 16 * 1024 * 1024 }, + const publishedRoot = await installCanaryPackage( + `rea-agents@${version}`, + canaryRoot, + serverEnvironment, ); - const publishedRoot = join(publishedPrefix, "node_modules", "rea-agents"); - const installed = JSON.parse( - await readFile(join(publishedRoot, "package.json"), "utf8"), + surface = await verifyInstalledCanaryPackage( + publishedRoot, + version, + canaryRoot, + serverEnvironment, ); - if (installed.version !== version) - throw new Error( - `Published installation is ${installed.version}, expected ${version}`, - ); - const launcher = join(publishedRoot, "scripts", "rea.mjs"); - for (const option of ["--help", "--llms"]) - await execFileAsync(process.execPath, [launcher, option], { - cwd: canaryRoot, - env: serverEnvironment, - maxBuffer: 16 * 1024 * 1024, - }); - transport = new StdioClientTransport({ - command: process.execPath, - args: [launcher, "mcp"], - cwd: canaryRoot, - env: serverEnvironment, - stderr: "pipe", - }); - transport.stderr?.on("data", (chunk) => { - if (stderr.length < 16_384) stderr += chunk.toString("utf8"); - }); - await client.connect(transport); - const tools = await client.listTools(); - const toolNames = tools.tools.map(({ name }) => name); - const canonicalToolNames = new Set(TOOL_CONTRACTS.map(({ name }) => name)); - const unknownToolNames = toolNames.filter( - (name) => !canonicalToolNames.has(name), - ); - const publishedToolNames = new Set(toolNames); - const missingToolNames = [...canonicalToolNames].filter( - (name) => !publishedToolNames.has(name), - ); - publishedToolCount = toolNames.length; - if ( - publishedToolCount === 0 || - publishedToolNames.size !== publishedToolCount || - unknownToolNames.length > 0 || - missingToolNames.length > 0 - ) - throw new Error( - `Published MCP exposed an invalid canonical tool catalog (${String(publishedToolCount)} tools, unknown: ${unknownToolNames.join(", ") || "none"}, missing: ${missingToolNames.join(", ") || "none"})`, - ); - const statusReply = await client.callTool({ - name: "binary_session", - arguments: {}, - }); - const status = requireMcpLifecycleResult(statusReply, "binary_session"); - const availabilityNames = sortVerifierNames( - status.tool_availability.map(({ name }) => name), - ); - if ( - JSON.stringify(availabilityNames) !== - JSON.stringify(sortVerifierNames(canonicalToolNames)) - ) - throw new Error( - "Published binary_session omitted canonical tool availability", - ); update = await verifyPublishedUpdate({ version, canaryRoot, updaterRoot: publishedRoot, }); -} catch (cause) { - throw new Error(`Published package verification failed: ${stderr}`, { - cause, - }); } finally { - try { - await client.close(); - await transport?.close(); - } finally { - await rm(canaryRoot, { recursive: true, force: true }); - } + await rm(canaryRoot, { recursive: true, force: true }); } - process.stdout.write( - `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), package: "rea-agents", version, mcpTools: publishedToolCount, canonicalMcpTools: CATALOG_IDENTITY.counts.mcp_tools, update })}\n`, + `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), package: "rea-agents", version, ...surface, update })}\n`, ); async function verifyPublishedUpdate({ @@ -144,6 +61,10 @@ async function verifyPublishedUpdate({ canaryRoot, updaterRoot, }) { + const updateVersion = stableUpdateVersion( + targetVersion, + await readNpmRelease(targetVersion), + ); const prefix = join(canaryRoot, "update-prefix"); const home = join(canaryRoot, "update-home"); const environment = { @@ -168,7 +89,7 @@ async function verifyPublishedUpdate({ PATH: `${join(prefix, "bin")}${delimiter}${process.env.PATH ?? ""}`, }; const previousVersion = await findPreviousPublishedVersion( - targetVersion, + updateVersion, canaryRoot, environment, ); @@ -214,20 +135,20 @@ if (result.status === "failed") process.exitCode = 1; if ( result.status !== "updated" || result.previousVersion !== previousVersion || - result.latestVersion !== targetVersion || - result.installedVersion !== targetVersion || - result.command.at(-1) !== `rea-agents@${targetVersion}` + result.latestVersion !== updateVersion || + result.installedVersion !== updateVersion || + result.command.at(-1) !== `rea-agents@${updateVersion}` ) throw new Error( - `Published REA updater did not update ${previousVersion} to ${targetVersion}: ${stdout}`, + `Published REA updater did not update ${previousVersion} to stable ${updateVersion}: ${stdout}`, ); const installed = JSON.parse( await readFile(join(npmRoot, "rea-agents", "package.json"), "utf8"), ); - if (installed.version !== targetVersion) + if (installed.version !== updateVersion) throw new Error( - `REA update installed ${installed.version}, expected ${targetVersion}`, + `REA update installed ${installed.version}, expected stable ${updateVersion}`, ); const help = ( @@ -259,7 +180,10 @@ async function findPreviousPublishedVersion(targetVersion, cwd, environment) { const previous = versions .filter( (candidate) => - typeof candidate === "string" && lt(candidate, targetVersion), + typeof candidate === "string" && + valid(candidate) === candidate && + prerelease(candidate) === null && + lt(candidate, targetVersion), ) .sort(rcompare)[0]; if (previous === undefined) diff --git a/scripts/verify-release-artifact.mjs b/scripts/verify-release-artifact.mjs new file mode 100644 index 0000000000..b04f1d0a4f --- /dev/null +++ b/scripts/verify-release-artifact.mjs @@ -0,0 +1,47 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { valid } from "semver"; +import { + installCanaryPackage, + verifyInstalledCanaryPackage, +} from "./lib/release-package-canary.mjs"; +import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; + +const [tarball, version, ...extra] = process.argv.slice(2); +if ( + tarball === undefined || + typeof version !== "string" || + valid(version) !== version || + extra.length +) + throw new Error( + "Usage: node scripts/verify-release-artifact.mjs ", + ); +const source = resolve(tarball); +const verifierRun = createVerifierRun(); +const workspace = await mkdtemp(join(tmpdir(), "rea-release-rehearsal-")); +const environment = { + ...process.env, + npm_config_cache: join(workspace, "npm-cache"), +}; +delete environment.HOPPER_TARGET_PATH; +let result; +try { + const packageRoot = await installCanaryPackage( + source, + workspace, + environment, + ); + result = await verifyInstalledCanaryPackage( + packageRoot, + version, + workspace, + environment, + ); +} finally { + await rm(workspace, { recursive: true, force: true }); +} +process.stdout.write( + `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), package: "rea-agents", version, source, ...result })}\n`, +); diff --git a/tests/boundary/filesystem/ciVerifiedPackage.test.ts b/tests/boundary/filesystem/ciVerifiedPackage.test.ts index ff50450242..c2ee285249 100644 --- a/tests/boundary/filesystem/ciVerifiedPackage.test.ts +++ b/tests/boundary/filesystem/ciVerifiedPackage.test.ts @@ -29,7 +29,7 @@ it("validates a tarball against the exact checkout and current workflow attempt" join(fixture.artifactDirectory, "package.tgz"), ); expect(await readFile(fixture.output, "utf8")).toBe( - `tarball=${join(fixture.artifactDirectory, "package.tgz")}\n`, + `tarball=${join(fixture.artifactDirectory, "package.tgz")}\nversion=1.2.3\n`, ); }); @@ -46,6 +46,7 @@ it.each([ ["wrong verifier platform", { platform: "darwin" }], ["wrong verifier architecture", { architecture: "arm64" }], ["wrong package digest", { sha256: "c".repeat(64) }], + ["invalid package version", { packageVersion: "invalid\noutput=injection" }], ])("rejects %s", async (_label, override) => { const fixture = await createFixture({ override }); diff --git a/tests/boundary/filesystem/npmRelease.test.ts b/tests/boundary/filesystem/npmRelease.test.ts new file mode 100644 index 0000000000..c355e21b7d --- /dev/null +++ b/tests/boundary/filesystem/npmRelease.test.ts @@ -0,0 +1,132 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { + readNpmRelease, + requireMatchingNpmRelease, + stableUpdateVersion, + waitForNpmRelease, +} from "../../../scripts/ci/npm-release.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const metadata = ( + version: string, + integrity: string, + tags: Record, +) => ({ + name: "rea-agents", + versions: { [version]: { name: "rea-agents", version, dist: { integrity } } }, + "dist-tags": tags, +}); + +it("accepts existing registry bytes only when content and selected channel match", async () => { + const root = await createTestTempDirectory("rea-npm-release-"); + const tarball = join(root, "package.tgz"); + const archive = "qualified archive"; + await writeFile(tarball, archive); + const integrity = `sha512-${createHash("sha512").update(archive).digest("base64")}`; + const published = await readNpmRelease("6.5.0", async () => + Response.json(metadata("6.5.0", integrity, { latest: "6.5.0" })), + ); + await expect( + requireMatchingNpmRelease(tarball, "6.5.0", published), + ).resolves.toBeUndefined(); + expect(stableUpdateVersion("6.5.0", published)).toBe("6.5.0"); + if (published.status !== "published") + throw new Error("Fixture release is absent"); + await expect( + requireMatchingNpmRelease(tarball, "6.5.0", { + ...published, + integrity: "sha512-wrong", + }), + ).rejects.toThrow("npm integrity differs"); + await expect( + requireMatchingNpmRelease(tarball, "6.5.0", { + ...published, + taggedVersion: "6.6.0", + }), + ).rejects.toThrow("refusing to change"); +}); + +it("keeps prerelease verification on next and the updater on stable latest", async () => { + const published = await readNpmRelease("6.6.0-rc.1", async () => + Response.json( + metadata("6.6.0-rc.1", "sha512-candidate", { + latest: "6.5.0", + next: "6.6.0-rc.1", + }), + ), + ); + expect(published).toMatchObject({ tag: "next", taggedVersion: "6.6.0-rc.1" }); + expect(stableUpdateVersion("6.6.0-rc.1", published)).toBe("6.5.0"); + if (published.status !== "published") + throw new Error("Fixture release is absent"); + expect(() => + stableUpdateVersion("6.6.0-rc.1", { + ...published, + stableVersion: "6.6.0-rc.1", + }), + ).toThrow("stable release"); +}); + +it("distinguishes absence from failed or malformed registry responses", async () => { + await expect( + readNpmRelease( + "6.5.0", + async () => new Response("missing", { status: 404 }), + ), + ).resolves.toEqual({ status: "absent" }); + await expect( + readNpmRelease("6.5.0", async () => + Response.json({ name: "rea-agents", versions: {}, "dist-tags": {} }), + ), + ).resolves.toEqual({ status: "absent" }); + await expect( + readNpmRelease( + "6.5.0", + async () => new Response("failed", { status: 503 }), + ), + ).rejects.toThrow("HTTP 503"); + await expect( + readNpmRelease("6.5.0", async () => Response.json({})), + ).rejects.toThrow(); + await expect( + readNpmRelease("6.5.0", async () => + Response.json(metadata("6.5.0", "", { latest: "6.5.0" })), + ), + ).rejects.toThrow(); +}); + +it("waits only for confirmed propagation absence and does not retry errors", async () => { + const published = await readNpmRelease("6.5.0", async () => + Response.json(metadata("6.5.0", "sha512-target", { latest: "6.5.0" })), + ); + let calls = 0; + const pauses: number[] = []; + await expect( + waitForNpmRelease( + "6.5.0", + async () => (++calls === 1 ? { status: "absent" } : published), + async (milliseconds) => { + pauses.push(milliseconds); + }, + ), + ).resolves.toEqual(published); + expect(calls).toBe(2); + expect(pauses).toEqual([15_000]); + let failures = 0; + await expect( + waitForNpmRelease( + "6.5.0", + async () => { + failures++; + throw new Error("unavailable"); + }, + async () => { + throw new Error("must not retry"); + }, + ), + ).rejects.toThrow("unavailable"); + expect(failures).toBe(1); +}); diff --git a/tests/boundary/filesystem/releaseRecovery.test.ts b/tests/boundary/filesystem/releaseRecovery.test.ts index 05689d5024..ee1772d13a 100644 --- a/tests/boundary/filesystem/releaseRecovery.test.ts +++ b/tests/boundary/filesystem/releaseRecovery.test.ts @@ -55,6 +55,19 @@ it("recovers a failed post-publication run using its successful original qualifi }); }); +it("recovers a byte-verified existing version when the npm write was skipped", () => { + const input = selection(); + input.jobs[2]!.steps[0]!.conclusion = "skipped"; + input.jobs[2]!.steps.push({ + name: "Verify published npm bytes and dist-tag", + conclusion: "success", + }); + expect(selectRecoveryRelease(input)).toMatchObject({ + version: "6.5.0", + run_id: 123, + }); +}); + it.each(["failure", "skipped", "cancelled"])( "refuses recovery when original qualification was %s", (conclusion) => { @@ -184,8 +197,12 @@ it("limits recovery to verification and dependent MCP publication from the relea const canary = steps.findIndex((candidate) => candidate.run?.includes("verify-published-package.mjs"), ); + const rehearsal = steps.findIndex((candidate) => + candidate.run?.includes("verify-release-artifact.mjs"), + ); expect(digest).toBeGreaterThanOrEqual(0); - expect(canary).toBeGreaterThan(digest); + expect(rehearsal).toBeGreaterThan(digest); + expect(canary).toBeGreaterThan(rehearsal); const publisher = workflow.jobs["publish-mcp"]; expect(publisher.needs).toBe("verify"); expect( diff --git a/tests/boundary/filesystem/releaseWorkflow.test.ts b/tests/boundary/filesystem/releaseWorkflow.test.ts index 3e51acd657..712ed8945e 100644 --- a/tests/boundary/filesystem/releaseWorkflow.test.ts +++ b/tests/boundary/filesystem/releaseWorkflow.test.ts @@ -15,10 +15,12 @@ const stepSchema = z.object({ uses: z.string().optional(), run: z.string().optional(), if: z.string().optional(), + "continue-on-error": z.union([z.boolean(), z.string()]).optional(), with: z.record(z.string(), z.unknown()).optional(), env: z.record(z.string(), z.unknown()).optional(), }); const jobSchema = z.object({ + "continue-on-error": z.union([z.boolean(), z.string()]).optional(), if: z.string().optional(), needs: z.union([z.string(), z.array(z.string())]).optional(), outputs: z.record(z.string(), z.string()).optional(), @@ -41,6 +43,9 @@ async function readReleaseWorkflow() { with: z.record(z.string(), z.string()), }), publish: jobSchema, + rehearsal: jobSchema.extend({ + permissions: z.record(z.string(), z.string()), + }), "publish-mcp": jobSchema, }), }) @@ -287,6 +292,10 @@ it("binds npm and MCP publication to the same immutable release SHA", async () = "${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }}", }); expect(workflow.jobs.publish.needs).toContain("qualification"); + expect(workflow.jobs.publish.needs).toContain("rehearsal"); + expect(workflow.jobs.publish.if).toContain( + "needs.rehearsal.result == 'success'", + ); expect(workflow.jobs.publish.if).toContain( "needs.qualification.result == 'success'", ); @@ -295,7 +304,11 @@ it("binds npm and MCP publication to the same immutable release SHA", async () = ); expect(workflow.jobs.publish.needs).toContain("merged-release"); expect(workflow.jobs["publish-mcp"].needs).toContain("merged-release"); - for (const job of [workflow.jobs.publish, workflow.jobs["publish-mcp"]]) { + for (const job of [ + workflow.jobs.rehearsal, + workflow.jobs.publish, + workflow.jobs["publish-mcp"], + ]) { const checkout = job.steps.find((step) => step.uses?.startsWith("actions/checkout@"), ); @@ -657,47 +670,98 @@ it("routes website verification through the central PR policy", async () => { expect(ci.jobs.website.if).toBe("needs.changes.outputs.website == 'true'"); }); -// Publishing is irreversible. Keep the release authority invariant as a static -// boundary check; it is not proof that package installation or publishing works. -it("requires qualified package validation before publishing and retains the registry canary", async () => { - const workflow = z +// The rehearsal executes the retained archive on a new read-only runner; the +// process boundary suite exercises its real installation and protocol checks. +it("rehearses the qualified archive before npm authority and retains registry verification", async () => { + const workflow = await readReleaseWorkflow(); + const rehearsal = workflow.jobs.rehearsal; + expect(rehearsal.permissions).toEqual({ contents: "read", actions: "read" }); + expect(rehearsal["continue-on-error"]).toBeUndefined(); + expect(rehearsal.needs).toContain("qualification"); + expect(rehearsal.if).toContain("needs.qualification.result == 'success'"); + const exerciseIndex = rehearsal.steps.findIndex((step) => + step.run?.includes("verify-release-artifact.mjs"), + ); + const exercise = rehearsal.steps[exerciseIndex]; + expect(exercise?.env).toEqual({ + VERIFIED_PACKAGE_TARBALL: "${{ steps.package.outputs.tarball }}", + VERIFIED_PACKAGE_VERSION: "${{ steps.package.outputs.version }}", + }); + expect(exercise?.if).toBeUndefined(); + expect(exercise?.["continue-on-error"]).toBeUndefined(); + const publish = workflow.jobs.publish; + expect(publish.needs).toContain("rehearsal"); + expect(publish.if).toContain("needs.rehearsal.result == 'success'"); + for (const job of [rehearsal, publish]) { + const inputs = job.steps.findIndex( + (step) => step.uses === "./.github/actions/prepare-release-package", + ); + expect(inputs).toBeGreaterThanOrEqual(0); + expect(job.steps[inputs]?.id).toBe("package"); + expect(job.steps[inputs]?.if).toBeUndefined(); + expect(job.steps[inputs]?.["continue-on-error"]).toBeUndefined(); + if (job === rehearsal) expect(exerciseIndex).toBeGreaterThan(inputs); + } + const action = z .object({ - jobs: z.object({ - publish: z.object({ - steps: z.array(stepSchema), - }), + outputs: z.object({ + tarball: z.object({ value: z.string() }), + version: z.object({ value: z.string() }), }), + runs: z.object({ steps: z.array(stepSchema) }), }) .parse( parse( await readFile( - new URL("../../../.github/workflows/release.yml", import.meta.url), + ".github/actions/prepare-release-package/action.yml", "utf8", ), ), ); - const commands = workflow.jobs.publish.steps.map(({ run }) => run ?? ""); - const verify = commands.findIndex((command) => - command.includes("scripts/ci/verified-package.mjs validate"), + expect(action.outputs.tarball.value).toBe( + "${{ steps.package.outputs.tarball }}", ); - const publish = commands.findIndex((command) => - command.includes("npm publish"), + expect(action.outputs.version.value).toBe( + "${{ steps.package.outputs.version }}", ); - expect(verify).toBeGreaterThanOrEqual(0); - expect(publish).toBeGreaterThan(verify); - const steps = workflow.jobs.publish.steps; - const runtime = steps.findIndex( + const archive = action.runs.steps.findIndex( + (step) => step.with?.name === "verified-release-package", + ); + const validation = action.runs.steps.findIndex((step) => + step.run?.includes("scripts/ci/verified-package.mjs validate"), + ); + const runtime = action.runs.steps.findIndex( (step) => step.with?.name === "rea-test-runtime", ); - expect(runtime).toBeGreaterThan(verify); - expect(runtime).toBeLessThan(publish); - expect(steps[runtime]?.uses).toBe( - "actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093", + expect(archive).toBeGreaterThanOrEqual(0); + expect(validation).toBeGreaterThan(archive); + expect(runtime).toBeGreaterThan(validation); + for (const step of action.runs.steps) { + expect(step.if).toBeUndefined(); + expect(step["continue-on-error"]).toBeUndefined(); + } + const commands = publish.steps.map((step) => step.run ?? ""); + const inspect = commands.findIndex((command) => + command.includes("npm-release.mjs inspect"), ); - expect(steps[runtime]?.with).toEqual({ name: "rea-test-runtime", path: "." }); - expect( - commands - .slice(publish + 1) - .some((command) => command.includes("verify-published-package.mjs")), - ).toBe(true); + const write = commands.findIndex((command) => + command.includes("npm publish"), + ); + const bytes = commands.findIndex((command) => + command.includes("npm-release.mjs verify"), + ); + const canary = commands.findIndex((command) => + command.includes("verify-published-package.mjs"), + ); + expect(inspect).toBeGreaterThanOrEqual(0); + expect(write).toBeGreaterThan(inspect); + expect(bytes).toBeGreaterThan(write); + expect(canary).toBeGreaterThan(bytes); + expect(publish.steps[write]?.if).toBe( + "steps.registry.outputs.published == 'false'", + ); + for (const index of [inspect, bytes, canary]) { + expect(publish.steps[index]?.if).toBeUndefined(); + expect(publish.steps[index]?.["continue-on-error"]).toBeUndefined(); + } }); diff --git a/tests/boundary/process/releaseArtifactCanary.test.ts b/tests/boundary/process/releaseArtifactCanary.test.ts new file mode 100644 index 0000000000..690e58a120 --- /dev/null +++ b/tests/boundary/process/releaseArtifactCanary.test.ts @@ -0,0 +1,166 @@ +import { execFile } from "node:child_process"; +import { + copyFile, + mkdir, + readFile, + symlink, + writeFile, +} from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +import { z } from "zod"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); + +// Release rehearsal runs on Linux; the npm process adapter is also exercised on macOS. +it.skipIf(process.platform === "win32")( + "fails without declared runtime, then rehearses an installed tarball and rejects catalog drift", + async () => { + const root = await createTestTempDirectory("rea-release-canary-"); + const candidate = join(root, "candidate"); + const controller = join(root, "controller"); + const { version } = z + .object({ version: z.string() }) + .parse(JSON.parse(await readFile("package.json", "utf8"))); + const harness = await createApplicationMcpHarness(); + let snapshot; + try { + snapshot = { + serverInfo: harness.client.getServerVersion(), + protocolVersion: harness.client.getNegotiatedProtocolVersion(), + tools: await harness.client.listTools(), + reply: await harness.client.callTool({ + name: "binary_session", + arguments: {}, + }), + }; + } finally { + await harness.close(); + } + await mkdir(join(candidate, "scripts"), { recursive: true }); + await writeFile( + join(candidate, "package.json"), + JSON.stringify({ + name: "rea-agents", + version, + type: "module", + files: ["scripts"], + }), + ); + await copyFile( + "tests/fixtures/mcpDoctorReply.mjs", + join(candidate, "scripts/replay.mjs"), + ); + await writeFile( + join(candidate, "scripts/reply.json"), + JSON.stringify(snapshot), + ); + await writeFile( + join(candidate, "scripts/rea.mjs"), + ` +import { fileURLToPath } from "node:url"; +if (process.argv[2] === "mcp") { + process.argv[2] = fileURLToPath(new URL("./reply.json", import.meta.url)); + await import("./replay.mjs"); +} else if (["--help", "--llms"].includes(process.argv[2])) { + console.log("REA fixture CLI"); +} else { process.exitCode = 1; } +`, + ); + const packed = await execute( + "npm", + ["pack", "--ignore-scripts", "--json"], + { + cwd: candidate, + }, + ); + const [{ filename }] = z + .tuple([z.object({ filename: z.string() })]) + .parse(JSON.parse(packed.stdout)); + const tarball = join(candidate, filename); + for (const file of [ + "scripts/verify-release-artifact.mjs", + "scripts/lib/release-package-canary.mjs", + "scripts/lib/mcp-verifier-results.mjs", + "scripts/lib/verifier-run.mjs", + ]) { + const target = join(controller, file); + await mkdir(dirname(target), { recursive: true }); + await copyFile(file, target); + } + await writeFile( + join(controller, "package.json"), + JSON.stringify({ name: "rea-agents", version, type: "module" }), + ); + await symlink( + resolve("node_modules"), + join(controller, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + const invoke = (selected = tarball) => + execute( + process.execPath, + [ + join(controller, "scripts/verify-release-artifact.mjs"), + selected, + version, + ], + { cwd: controller, maxBuffer: 16 * 1024 * 1024 }, + ); + await expect(invoke()).rejects.toMatchObject({ + stderr: expect.stringContaining("ERR_MODULE_NOT_FOUND"), + }); + await symlink( + resolve("dist"), + join(controller, "dist"), + process.platform === "win32" ? "junction" : "dir", + ); + const result = JSON.parse((await invoke()).stdout); + expect(result).toMatchObject({ + package: "rea-agents", + version, + source: tarball, + mcpTools: snapshot.tools.tools.length, + }); + if (process.platform !== "win32") + expect(result.verifier_run.process_lineage).toMatchObject({ + status: "verified", + descendants: [], + }); + // Repack with one omitted canonical tool; the real protocol still succeeds, + // so only the independent release catalog assertion can reject this artifact. + snapshot.tools.tools.pop(); + await writeFile( + join(candidate, "scripts/reply.json"), + JSON.stringify(snapshot), + ); + await execute("npm", ["pack", "--ignore-scripts", "--json"], { + cwd: candidate, + }); + await expect(invoke()).rejects.toMatchObject({ + stderr: expect.stringContaining("invalid canonical tool catalog"), + }); + await writeFile( + join(candidate, "package.json"), + JSON.stringify({ + name: "rea-agents", + version: "0.0.0", + type: "module", + files: ["scripts"], + }), + ); + const wrong = await execute("npm", ["pack", "--ignore-scripts", "--json"], { + cwd: candidate, + }); + const [{ filename: wrongFilename }] = z + .tuple([z.object({ filename: z.string() })]) + .parse(JSON.parse(wrong.stdout)); + await expect(invoke(join(candidate, wrongFilename))).rejects.toMatchObject({ + stderr: expect.stringContaining("Installed package identity"), + }); + }, + 30_000, +);