fix(scripts): keep Python runtime deployment isolated and deduplicated

This commit is contained in:
Tianyi Cui committed 2026-10-07 13:51:48 +08:00
1 parent 44dd60de91
commit a46412bd53
11 files changed
+119 -22

No files matched your search

@@ -21,8 +21,8 @@
en: c35ea4b858448b49
zh: 1f8ff0a9f82ca881
/agent-note-single-file-executable-sdk-runtime-distribution-single-exe/decision/build-pipeline-and-artifacts:
en: 6356c325e36dc2a8
zh: 63da32299cb356c9
en: d3bbd24ead02af4f
zh: 64623d88bee4314b
/agent-note-single-file-executable-sdk-runtime-distribution-single-exe/decision/python-sdk-distribution-two-carriers-exe-for-production-node-for-development:
en: 9bd60549d4a52ec0
zh: e2343b7210212ee0
@@ -48,7 +48,7 @@ The [Office kit](2026-09-14-independent-libreoffice-kit.md) and its installed de
### Build pipeline and artifacts
[`scripts/build-exe-for-python-sdk.ts`](../../../../scripts/build-exe-for-python-sdk.ts): runtime closure verification → `pnpm run build` → (after clearing) `pnpm --filter dsh-python-runtime-closure deploy --legacy --prod --config.allow-unused-patches=true --config.node-linker=hoisted --config.auto-install-peers=false --config.link-workspace-packages=true` **directly into** `python/sdk-runtime/src/deepseek_harness_runtime/runtime/node/`, including [`python/sdk-runtime/runtime-bootstrap.mjs`](../../../../python/sdk-runtime/runtime-bootstrap.mjs) as the carrier-root `runtime-bootstrap.mjs` → restore direct workspace packages omitted by legacy deploy and reject any remaining manifest gap → replace staged dependency symlinks with their target bytes, remove package-manager `.bin` links, and fail if any symlink remains → verify the deployed bootstrap and inject pkg configuration with that bin plus assets covering dynamic profile, bundle, frontend, preset, native-library, and configuration reads → stage the target `node-pty` addon → invoke `pkg --sea` once per target → write `deepseek-harness-sdk-runtime-<platform>-<arch>` under `dist-exe/` and copy it with its target-specific Office directory into the runtime directory. The Python runtime owns that bootstrap. It calls the public CLI export for ordinary launches and dispatches a provider-private selection to the same `@deepseek-ai/dsh-subprocess-local/runner` core without changing CLI grammar or adding another executable; the [native-containment reference](../../../../packages/subprocess/subprocess-local/README.md) owns that private path. Linux CI rebuilds `pty.node` inside the matching manylinux 2.28 container because legacy deploy omits that install side effect. Every target copies its native `@vscode/ripgrep` binary beside the executable as the required `-rg` sidecar; pkg runtimes select that sidecar through `process.pkg`, while ordinary Node execution uses `@vscode/ripgrep` directly. macOS uses its target prebuild and also emits the required `-spawn-helper`. The deploy flags preserve the measured runtime layout: `--legacy` is the mandatory path with inject-workspace-packages off; hoisted gives pkg a stable single-instance layout that the explicit materialization pass makes symlink-free; disabling automatic peer installation prevents undeclared peers from expanding the closure; link-workspace-packages selects direct workspace dependencies. [`pnpm-workspace.yaml`](../../../../pnpm-workspace.yaml) overrides the transitive `@deepseek-ai/cosmokit` and `@deepseek-ai/schemastery` semver requests to the pinned vendor sources so legacy deploy never resolves those unpublished names from a registry. Production deployment permits unused workspace patches because development-only tools can be absent from its closure; included-package patch failures still fail deployment. Workspace installs retain strict unused-patch validation.
[`scripts/build-exe-for-python-sdk.ts`](../../../../scripts/build-exe-for-python-sdk.ts): runtime closure verification → `pnpm run build` → (after clearing) `pnpm --filter dsh-python-runtime-closure deploy --legacy --prod --config.allow-unused-patches=true --config.node-linker=hoisted --config.auto-install-peers=false --config.link-workspace-packages=true --config.hoist-workspace-packages=false` **directly into** `python/sdk-runtime/src/deepseek_harness_runtime/runtime/node/`, including [`python/sdk-runtime/runtime-bootstrap.mjs`](../../../../python/sdk-runtime/runtime-bootstrap.mjs) as the carrier-root `runtime-bootstrap.mjs` → restore direct workspace packages omitted by legacy deploy and reject any remaining manifest gap → replace staged dependency symlinks with their target bytes, remove package-manager `.bin` links, and fail if any symlink remains → verify the deployed bootstrap and inject pkg configuration with that bin plus assets covering dynamic profile, bundle, frontend, preset, native-library, and configuration reads → stage the target `node-pty` addon → invoke `pkg --sea` once per target → write `deepseek-harness-sdk-runtime-<platform>-<arch>` under `dist-exe/` and copy it with its target-specific Office directory into the runtime directory. The Python runtime owns that bootstrap. It calls the public CLI export for ordinary launches and dispatches a provider-private selection to the same `@deepseek-ai/dsh-subprocess-local/runner` core without changing CLI grammar or adding another executable; the [native-containment reference](../../../../packages/subprocess/subprocess-local/README.md) owns that private path. Linux CI rebuilds `pty.node` inside the matching manylinux 2.28 container because legacy deploy omits that install side effect. Every target copies its native `@vscode/ripgrep` binary beside the executable as the required `-rg` sidecar; pkg runtimes select that sidecar through `process.pkg`, while ordinary Node execution uses `@vscode/ripgrep` directly. macOS uses its target prebuild and also emits the required `-spawn-helper`. The deploy flags preserve the measured runtime layout: `--legacy` is the mandatory path with inject-workspace-packages off; hoisted supplies package directories that staging makes symlink-free; staging removes nested workspace copies only when pnpm records the same dependency identity at the payload root, preserving singleton services; disabling workspace hoisting excludes unrelated workspace source packages; disabling automatic peer installation prevents undeclared peers from expanding the closure; link-workspace-packages selects direct workspace dependencies. [`pnpm-workspace.yaml`](../../../../pnpm-workspace.yaml) overrides the transitive `@deepseek-ai/cosmokit` and `@deepseek-ai/schemastery` semver requests to the pinned vendor sources so legacy deploy never resolves those unpublished names from a registry. Production deployment permits unused workspace patches because development-only tools can be absent from its closure; included-package patch failures still fail deployment. Workspace installs retain strict unused-patch validation.
CI: [`.github/workflows/build-exe-for-python-sdk.yml`](../../../../.github/workflows/build-exe-for-python-sdk.yml) runs [installed-wheel validation](../testing/2026-08-23-installed-python-wheel-black-box-ci.md) on Linux/Windows x64 for pull requests and Linux ARM64 plus both macOS architectures for master pushes. The [public publication workflow](../../archived/process/2026-08-11-python-publication-workflow.md) calls it for all five targets; `workflow_dispatch` can still select a subset. Native builds run on linux-x64 / linux-arm64 (`ubuntu-24.04-arm`) / macos-arm64 / macos-x64 (`macos-15-intel`) / win-x64 (`windows-2025`), with `~/.pkg-cache` cached where applicable, and pkg handles macOS ad-hoc signing. Each leg installs the release-shaped SDK and runtime wheels into a clean venv outside the checkout, verifies their package and executable origin, then drives the complete keyless scenario set through the public SDK and direct NDJSON JSON-RPC. Trusted pull requests and master pushes additionally run a real DeepSeek two-turn tool smoke on their selected targets; fork and Dependabot heads receive no key. Linux inspects the executable and native addon's GLIBC requirements and runs an additional manylinux 2.28 smoke, while macOS checks the runtime, ripgrep, and PTY helper architectures and verifies that all three deployment targets fit the wheel tag. A full five-target run retains six artifacts, each containing one release file: the platform-independent SDK wheel and five native runtime wheels; a subset dispatch retains the SDK wheel and selected runtime wheels. Bare executables and source bundles remain intermediate test inputs. [`.gitlab-ci.yml`](../../../../.gitlab-ci.yml) accepts `python-v<repository-version>` tag pipelines whose version matches the root `package.json`, builds one SDK wheel and five native runtime wheels, then a single serialized job checks and publishes all six to the project PyPI registry. The [`python/sdk-runtime` README](../../../../python/sdk-runtime/README.md) owns the Windows target and the explicit exclusion of Windows arm64.
@@ -48,7 +48,7 @@ exe 的 VFS 内是**构建产物形态的真实包树**(各包的 `lib/` + 真
### 构建流水线与产物
[`scripts/build-exe-for-python-sdk.ts`](../../../../scripts/build-exe-for-python-sdk.ts):运行时闭包校验 → `pnpm run build` →(清空后)`pnpm --filter dsh-python-runtime-closure deploy --legacy --prod --config.allow-unused-patches=true --config.node-linker=hoisted --config.auto-install-peers=false --config.link-workspace-packages=true` **直接写入** `python/sdk-runtime/src/deepseek_harness_runtime/runtime/node/`,其中包含作为载体根目录 `runtime-bootstrap.mjs` 的 [`python/sdk-runtime/runtime-bootstrap.mjs`](../../../../python/sdk-runtime/runtime-bootstrap.mjs) → 恢复 legacy deploy 遗漏的直接工作区包,并拒绝剩余的 manifest 缺口 → 将暂存依赖中的符号链接替换为目标文件内容,删除包管理器的 `.bin` 链接,并在仍有任何符号链接时失败 → 验证已部署的 bootstrap,并注入以该文件为 bin 的 pkg 配置及覆盖动态读取 profile、bundle、前端、preset、原生库与配置文件的 assets → 暂存目标平台的 `node-pty` addon → 每个构建目标调用一次 `pkg --sea` → 将 `deepseek-harness-sdk-runtime-<platform>-<arch>` 写入 `dist-exe/`,并将它与对应目标的 Office 目录一同拷回运行时目录。该 bootstrap 由 Python runtime 拥有;普通启动时它调用公开 CLI export,而提供方私有选择会分派到同一个 `@deepseek-ai/dsh-subprocess-local/runner` 核心,不改变 CLI 语法,也不增加另一个可执行文件;[原生 containment 参考](../../../../packages/subprocess/subprocess-local/README.zh.md)负责这条私有路径。Linux CI 会在匹配的 manylinux 2.28 容器中重新构建 `pty.node`,因为 legacy deploy 会遗漏这一安装副作用。每个目标都会把对应的原生 `@vscode/ripgrep` 二进制复制到可执行文件旁,作为必需的 `-rg` 伴随文件;pkg 运行时通过 `process.pkg` 选择该伴随文件,普通 Node 执行则直接使用 `@vscode/ripgrep`。macOS 使用对应目标的预构建产物,并额外生成所需的 `-spawn-helper`。部署标志都有实测依据:未启用 `inject-workspace-packages` 时必须使用 `--legacy`;`hoisted` 为 pkg 提供稳定的单实例布局,再由显式物化步骤消除符号链接;关闭对等依赖自动安装可防止未声明的对等依赖扩大闭包;`link-workspace-packages` 选择直接工作区依赖。[`pnpm-workspace.yaml`](../../../../pnpm-workspace.yaml) 将传递的 `@deepseek-ai/cosmokit` 与 `@deepseek-ai/schemastery` semver 请求覆盖到固定的 vendor 源码,使 legacy deploy 不会从注册表解析这些未发布名称。 生产部署允许未使用的工作区补丁,因为开发工具可能不属于运行时闭包;闭包内包的补丁应用失败仍会中止部署。工作区安装继续严格校验未使用的补丁。
[`scripts/build-exe-for-python-sdk.ts`](../../../../scripts/build-exe-for-python-sdk.ts):运行时闭包校验 → `pnpm run build` →(清空后)`pnpm --filter dsh-python-runtime-closure deploy --legacy --prod --config.allow-unused-patches=true --config.node-linker=hoisted --config.auto-install-peers=false --config.link-workspace-packages=true --config.hoist-workspace-packages=false` **直接写入** `python/sdk-runtime/src/deepseek_harness_runtime/runtime/node/`,其中包含作为载体根目录 `runtime-bootstrap.mjs` 的 [`python/sdk-runtime/runtime-bootstrap.mjs`](../../../../python/sdk-runtime/runtime-bootstrap.mjs) → 恢复 legacy deploy 遗漏的直接工作区包,并拒绝剩余的 manifest 缺口 → 将暂存依赖中的符号链接替换为目标文件内容,删除包管理器的 `.bin` 链接,并在仍有任何符号链接时失败 → 验证已部署的 bootstrap,并注入以该文件为 bin 的 pkg 配置及覆盖动态读取 profile、bundle、前端、preset、原生库与配置文件的 assets → 暂存目标平台的 `node-pty` addon → 每个构建目标调用一次 `pkg --sea` → 将 `deepseek-harness-sdk-runtime-<platform>-<arch>` 写入 `dist-exe/`,并将它与对应目标的 Office 目录一同拷回运行时目录。该 bootstrap 由 Python runtime 拥有;普通启动时它调用公开 CLI export,而提供方私有选择会分派到同一个 `@deepseek-ai/dsh-subprocess-local/runner` 核心,不改变 CLI 语法,也不增加另一个可执行文件;[原生 containment 参考](../../../../packages/subprocess/subprocess-local/README.zh.md)负责这条私有路径。Linux CI 会在匹配的 manylinux 2.28 容器中重新构建 `pty.node`,因为 legacy deploy 会遗漏这一安装副作用。每个目标都会把对应的原生 `@vscode/ripgrep` 二进制复制到可执行文件旁,作为必需的 `-rg` 伴随文件;pkg 运行时通过 `process.pkg` 选择该伴随文件,普通 Node 执行则直接使用 `@vscode/ripgrep`。macOS 使用对应目标的预构建产物,并额外生成所需的 `-spawn-helper`。部署标志都有实测依据:未启用 `inject-workspace-packages` 时必须使用 `--legacy`;`hoisted` 提供包目录,再由暂存步骤消除符号链接;只有 pnpm 在载荷根目录记录了相同依赖标识时,暂存步骤才删除对应的嵌套工作区副本,以保留服务单例;关闭工作区提升可排除无关的工作区源码包;关闭对等依赖自动安装可防止未声明的对等依赖扩大闭包;`link-workspace-packages` 选择直接工作区依赖。[`pnpm-workspace.yaml`](../../../../pnpm-workspace.yaml) 将传递的 `@deepseek-ai/cosmokit` 与 `@deepseek-ai/schemastery` semver 请求覆盖到固定的 vendor 源码,使 legacy deploy 不会从注册表解析这些未发布名称。 生产部署允许未使用的工作区补丁,因为开发工具可能不属于运行时闭包;闭包内包的补丁应用失败仍会中止部署。工作区安装继续严格校验未使用的补丁。
CI 使用 [`.github/workflows/build-exe-for-python-sdk.yml`](../../../../.github/workflows/build-exe-for-python-sdk.yml):[安装后 wheel 包验证](../testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md)在拉取请求上运行 Linux/Windows x64,在 master 推送上运行 Linux ARM64 与两种 macOS 架构。[公开发布工作流](../../archived/process/2026-08-11-python-publication-workflow.md)调用它构建全部五个目标;`workflow_dispatch` 仍可选择部分目标。linux-x64、linux-arm64(`ubuntu-24.04-arm`)、macos-arm64、macos-x64(`macos-15-intel`)与 win-x64(`windows-2025`)分别进行原生构建,并在适用平台缓存 `~/.pkg-cache`;macOS 的 ad-hoc 签名由 pkg 处理。每个平台都把发布形态的 SDK wheel 包与运行时 wheel 包安装到 checkout 外的干净 venv,证明包与可执行文件来源,再通过公开 SDK 与直接 NDJSON JSON-RPC 运行完整 keyless 场景。可信拉取请求与 master 推送还会在各自选定的目标上运行真实 DeepSeek 双轮工具冒烟测试;fork 与 Dependabot head 不会获得密钥。Linux 会检查可执行文件和原生 addon 各自的 GLIBC 依赖,并额外运行 manylinux 2.28 冒烟测试;macOS 则检查 runtime、ripgrep 与 PTY helper 的架构,并验证三个载荷的部署目标都符合 wheel 包标签。完整构建五个目标时保留 6 个产物,每个产物只含一个发布文件:平台无关的 SDK wheel 包与 5 个原生运行时 wheel 包;手动选择部分目标时保留 SDK wheel 与所选运行时 wheel。裸 exe 与源码包只作为测试中间输入。[`.gitlab-ci.yml`](../../../../.gitlab-ci.yml) 只接受版本与根目录 `package.json` 匹配的 `python-v<repository-version>` 标签流水线,构建一个 SDK wheel 包和 5 个原生运行时 wheel 包,再由单个串行任务校验并将这 6 个文件发布到项目的 PyPI 注册表。[`python/sdk-runtime` README](../../../../python/sdk-runtime/README.zh.md)负责 Windows 目标及对 Windows arm64 的明确排除。
+2 -6
View File
@@ -95,12 +95,8 @@ try {
const publishOrder = [];
for (const dir of dirs) {
const manifest = readJson(path.join(root, dir, 'package.json'));
// Platform packages are packed with npm: pnpm pack (observed on 11.7.0)
// normalizes file modes and STRIPS the executable bit, which ships a
// launcher no consumer can spawn; npm pack preserves it. Platform packages
// have no dependencies by construction, so they need none of pnpm's
// workspace-protocol conversion — the entry packages do, and carry no
// executables, so they keep pnpm pack.
// Platform packages have no workspace dependencies; npm preserves their
// executable modes. Entry packages need pnpm's workspace version conversion.
if (platformSet.has(dir)) {
run('npm', ['pack', `./${dir}`, '--pack-destination', destination], packRoot);
} else {
+2 -2
View File
@@ -15,5 +15,5 @@
en: 641c647efc49c568
zh: a29371de9fbbebfd
/deepseek-harness-runtime-bin/build-and-distribution:
en: 9a4ba11944fbfc54
zh: 2ae046803108df2b
en: 437ab8ba3bbe7ed6
zh: 3f321a73e472220b
+1 -1
View File
@@ -50,6 +50,6 @@ External profile management uses `dsh plugin --profile <name> ...`. That command
Production deployment permits unused workspace patches for packages outside the runtime closure; patches for included packages must still apply successfully. This exception is confined to the deploy command; repository installation still rejects unused patches.
From the repository root, `pnpm exec tsx scripts/build-exe-for-python-sdk.ts` verifies the closure, builds packages, deploys a symlink-free tree, packages the selected target, and syncs the executable and sidecars into this module. `scripts/build-python-release.py` stages release-shaped wheels at the root repository version and pins `deepseek-harness-sdk` to the exact runtime version.
From the repository root, `pnpm exec tsx scripts/build-exe-for-python-sdk.ts` verifies the closure, builds packages, deploys a symlink-free tree, packages the selected target, and syncs the executable and sidecars into this module. Deployment disables workspace hoisting so unrelated source packages stay out of the payload. Staging shares root workspace packages with nested consumers of the same pnpm dependency identity, preserving singleton services while retaining different peer resolutions. `scripts/build-python-release.py` stages release-shaped wheels at the root repository version and pins `deepseek-harness-sdk` to the exact runtime version.
The installed-wheel smoke creates a clean virtual environment outside the checkout, proves the installed distribution and executable identities, then exercises default and customized SDK profiles, external plugins, MCP, native tools, direct JSON-RPC, committed snapshots, and the real provider on trusted runs. Its Office scenario relocates the complete target payload and converts DOCX with the required platform engine: the target’s declared native engine, or WASM when no native engine is declared. See the [Python contributor workflow](../development.md) and [installed-wheel testing decision](../../.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.md).
+1 -1
View File
@@ -50,6 +50,6 @@ Python bootstrap 从相邻目录解析 Office kit,让原生辅助程序与 URL
生产部署允许工作区中不属于运行时闭包的补丁保持未使用;闭包内包的补丁仍必须成功应用。此例外仅用于部署命令,仓库安装仍拒绝未使用的补丁。
在仓库根目录运行 `pnpm exec tsx scripts/build-exe-for-python-sdk.ts`,会校验闭包、构建包、部署无符号链接的文件树、打包所选目标,并把可执行程序及伴随文件同步到本模块。`scripts/build-python-release.py` 按仓库根版本暂存发布形态的 wheel 包,并将 `deepseek-harness-sdk` 固定到完全相同的运行时版本。
在仓库根目录运行 `pnpm exec tsx scripts/build-exe-for-python-sdk.ts`,会校验闭包、构建包、部署无符号链接的文件树、打包所选目标,并把可执行程序及伴随文件同步到本模块。部署会关闭工作区提升,避免无关源码包进入载荷。暂存步骤让使用相同 pnpm 依赖标识的嵌套消费者共享根目录的工作区包,保留服务单例,同时保留不同的对等依赖解析。`scripts/build-python-release.py` 按仓库根版本暂存发布形态的 wheel 包,并将 `deepseek-harness-sdk` 固定到完全相同的运行时版本。
已安装 wheel 包冒烟测试会在检出目录外创建干净的虚拟环境,验证分发物与可执行程序的来源,然后覆盖默认及自定义 SDK profile、外部插件、MCP、原生工具、直接 JSON-RPC、检入快照,以及可信运行中的真实提供方。Office 场景会迁移完整的目标载荷目录,并使用所需的平台引擎转换 DOCX:使用目标已声明的原生引擎,未声明原生引擎时使用 WASM。另见 [Python 贡献者工作流](../development.zh.md) 与 [installed-wheel 测试决策](../../.agents/notes/implemented/testing/2026-08-23-installed-python-wheel-black-box-ci.zh.md)。
@@ -1,9 +1,10 @@
import { existsSync } from 'node:fs'
import { mkdir, mkdtemp, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { join, relative } from 'node:path'
import { createRequire } from 'node:module'
import { afterEach, expect, it } from 'vitest'
import { materializeStagedLinks } from './build-exe-for-python-sdk-staging.ts'
import { deduplicateStagedWorkspacePackages, materializeStagedLinks } from './build-exe-for-python-sdk-staging.ts'
const temporaryDirectories: string[] = []
const directoryLinkType = process.platform === 'win32' ? 'junction' : 'dir'
@@ -64,3 +65,57 @@ it('copies workspace package links as files and removes command shims', async ()
.resolves.toBe('export {}\n')
expect(await listSymlinks(staging)).toEqual([])
})
it('shares an identical workspace package while retaining different peers and third-party copies', async () => {
const workspace = await fixture('dsh-python-staging-dedupe-')
const staging = join(workspace, 'payload')
const modules = join(staging, 'node_modules')
const root = join(modules, '@fixture', 'tools')
const duplicate = join(modules, 'app', 'node_modules', '@fixture', 'tools')
const differentPeers = join(modules, 'other', 'node_modules', '@fixture', 'tools')
const external = join(modules, 'external')
const externalCopy = join(modules, 'app', 'node_modules', 'external')
for (const directory of [root, duplicate, differentPeers, external, externalCopy]) {
await mkdir(directory, { recursive: true })
await writeFile(join(directory, 'index.js'), 'exports.scheduler = Symbol("scheduler")\n')
}
const locations = (paths: string[]) => paths.map(path => relative(workspace, path))
await writeFile(join(modules, '.modules.yaml'), JSON.stringify({ hoistedLocations: {
'@fixture/tools@file:packages/tools(peer@1)': locations([root, duplicate]),
'@fixture/tools@file:packages/tools(peer@2)': locations([differentPeers]),
'external@1.0.0(@fixture/tools@file:packages/tools)': locations([external, externalCopy]),
} }))
await deduplicateStagedWorkspacePackages(staging, workspace)
expect(existsSync(duplicate)).toBe(false)
for (const directory of [root, differentPeers, external, externalCopy]) {
expect(existsSync(join(directory, 'index.js'))).toBe(true)
}
const fromRoot = createRequire(join(staging, 'entry.cjs'))
const fromApp = createRequire(join(modules, 'app', 'entry.cjs'))
const fromOther = createRequire(join(modules, 'other', 'entry.cjs'))
expect(fromApp('@fixture/tools')).toBe(fromRoot('@fixture/tools'))
expect(fromOther('@fixture/tools')).not.toBe(fromRoot('@fixture/tools'))
})
it('rejects invalid deployment metadata before removing any package', async () => {
const workspace = await fixture('dsh-python-staging-metadata-')
const staging = join(workspace, 'payload')
const modules = join(staging, 'node_modules')
const root = join(modules, 'tools')
const duplicate = join(modules, 'app', 'node_modules', 'tools')
const outside = join(workspace, 'source')
for (const directory of [root, duplicate, outside]) await mkdir(directory, { recursive: true })
const valid = { 'tools@file:packages/tools': [root, duplicate].map(path => relative(workspace, path)) }
for (const invalid of [
{},
{ hoistedLocations: { invalid: [42] } },
{ hoistedLocations: { ...valid, 'other@file:packages/other': [relative(workspace, outside)] } },
]) {
await writeFile(join(modules, '.modules.yaml'), JSON.stringify(invalid))
await expect(deduplicateStagedWorkspacePackages(staging, workspace)).rejects.toThrow('hoistedLocations')
expect(existsSync(duplicate)).toBe(true)
expect(existsSync(outside)).toBe(true)
}
})
+48 -5
View File
@@ -1,6 +1,49 @@
/** Turn the deploy-time package links of the Python runtime payload into real files. */
import { cp, lstat, readdir, realpath, rm } from 'node:fs/promises'
import { join, sep } from 'node:path'
/** Prepare a symlink-free Python runtime payload with shared workspace module instances. */
import { cp, lstat, readFile, readdir, realpath, rm, unlink } from 'node:fs/promises'
import { isAbsolute, join, relative, resolve, sep } from 'node:path'
import { load } from 'js-yaml'
function isHoistedLocations(value: unknown): value is Record<string, string[]> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
&& Object.values(value).every(paths => Array.isArray(paths)
&& paths.every((path: unknown) => typeof path === 'string'))
}
/**
* Share root workspace packages with nested consumers of the same pnpm dependency identity.
* Different peer resolutions and third-party packages keep their installed locations.
* All metadata paths are checked before deletion; the caller first materializes links.
* @param staging - Symlink-free deployed payload root.
* @param workspace - Workspace root relative to which pnpm records hoisted locations.
*/
export async function deduplicateStagedWorkspacePackages(staging: string, workspace: string): Promise<void> {
const modules = resolve(staging, 'node_modules')
const file = join(modules, '.modules.yaml')
const manifest: unknown = load(await readFile(file, 'utf8'))
if (typeof manifest !== 'object' || manifest === null || !('hoistedLocations' in manifest)
|| !isHoistedLocations(manifest.hoistedLocations)) {
throw new Error(`${file}: hoistedLocations must map dependency identities to path lists`)
}
const duplicates: string[] = []
for (const [identity, locations] of Object.entries(manifest.hoistedLocations)) {
const paths = locations.map((location) => {
const path = resolve(workspace, location)
const local = relative(modules, path)
if (local === '' || local === '..' || local.startsWith(`..${sep}`) || isAbsolute(local)) {
throw new Error(`${file}: hoistedLocations path is outside deployed node_modules: ${location}`)
}
return { path, parts: local.split(sep) }
})
if (!/^(?:@[^/]+\/)?[^@()]+@file:/.test(identity)) continue
const root = paths.find(({ parts }) => parts.length === (parts[0]?.startsWith('@') === true ? 2 : 1))
if (root === undefined) continue
if (!(await lstat(root.path)).isDirectory()) {
throw new Error(`${file}: hoistedLocations root package is not a directory: ${root.path}`)
}
duplicates.push(...paths.filter(({ path }) => path !== root.path).map(({ path }) => path))
}
for (const path of duplicates) await rm(path, { recursive: true, force: true })
}
/** Return the first symbolic link below a directory, if one exists. */
async function findSymlink(directory: string): Promise<string | undefined> {
@@ -42,12 +85,12 @@ export async function materializeStagedLinks(staging: string): Promise<void> {
const destination = remaining
const source = await realpath(destination)
if (payload === source || payload.startsWith(source + sep)) {
await rm(destination, { recursive: true, force: true })
await unlink(destination)
remaining = await findSymlink(nodeModules)
continue
}
const nestedNodeModules = join(source, 'node_modules')
await rm(destination, { recursive: true, force: true })
await unlink(destination)
await cp(source, destination, {
recursive: true,
dereference: true,
+1
View File
@@ -135,6 +135,7 @@ describe('Python runtime executable builder CLI', () => {
expect(result.stdout).toContain(`${process.execPath} C:\\tools\\pnpm.cjs --filter dsh-python-runtime-closure deploy`)
const deploy = result.stdout.split('\n').find(line => line.includes(' --filter dsh-python-runtime-closure deploy'))
expect(deploy).toContain('--prod --config.allow-unused-patches=true')
expect(deploy).toContain('--config.hoist-workspace-packages=false')
expect(result.stdout.split('--config.allow-unused-patches=true')).toHaveLength(2)
expect(result.stdout).not.toContain(resolve(root, 'python/sdk-runtime/runtime-bootstrap.mjs'))
expect(result.stdout).toContain('"bin":"runtime-bootstrap.mjs"')
+3 -1
View File
@@ -14,7 +14,7 @@ import { basename, dirname, extname, join, resolve, sep } from 'node:path'
import { parseArgs } from 'node:util'
import { resolveLinuxNodePtyAddon, resolveWindowsNodePtyAddons } from './build-exe-for-python-sdk-native-pty.ts'
import { copyOfficeSidecar, OFFICE_ASSET_IGNORES } from './build-exe-for-python-sdk-office.ts'
import { materializeStagedLinks } from './build-exe-for-python-sdk-staging.ts'
import { deduplicateStagedWorkspacePackages, materializeStagedLinks } from './build-exe-for-python-sdk-staging.ts'
import { preparePrimaryRuntime, smokePrimaryRuntime, type PrimaryRuntimeTarget } from './primary-runtime/prepare.ts'
const root = resolve(import.meta.dirname, '..')
@@ -311,6 +311,7 @@ class SingleExeBuild {
'--config.node-linker=hoisted',
'--config.auto-install-peers=false',
'--config.link-workspace-packages=true',
'--config.hoist-workspace-packages=false',
this.staging,
])
await this.restoreLegacyHoists()
@@ -374,6 +375,7 @@ class SingleExeBuild {
return
}
await materializeStagedLinks(this.staging)
await deduplicateStagedWorkspacePackages(this.staging, root)
}
/** Add the executable entry and pkg assets to the staged manifest. */