Merge pull request #7004 from deepseek-harness/fix/shiki-go-backtracking

fix(client): 修复 Go 高亮正则回溯卡死

packages/client/ui-primitives/README.i18n.yaml

@@ -54,8 +54,8 @@
   en: ca75c51c89c2c9b0
   zh: de54f3467ef3b9be
 /deepseek-ai-dsh-client-ui-primitives/known-limitations-and-deferred-work:
-  en: 5a1e9d041b556b0e
-  zh: c66b4e7d5f77162d
+  en: 9c5bc17e35af9d0b
+  zh: 4f1a142c7d56cf72
 /deepseek-ai-dsh-client-ui-primitives/known-limitations-and-deferred-work/dev-note:
   en: e67f633dc6fe7773
   zh: 3bb0c20c93f16f98

packages/client/ui-primitives/README.md

@@ -220,6 +220,7 @@ These limits define how the atoms behave at the edges; they are current package
 - **Streaming defers cross-boundary reference resolution** — a reference-style link or footnote whose definition sits on the other side of the incremental freeze boundary renders as literal text while the reply streams; the settled full parse at finalize resolves it.
 - **A long highlighted fence retains its complete token DOM** — streaming avoids re-parsing, re-tokenizing, and reconciling the completed prefix, but it does not discard old colors or virtualize token spans. Final DOM cardinality therefore still follows the fence's token count; nested/container fences and a pathological single long line remain on the general tail path.
 - **Overlong lines stay uncolored** — a source line of 1,000 or more UTF-16 units renders as one plain run in every highlighted surface, because TextMate scanning cost grows superlinearly with line length and a single generated line can otherwise block the page for seconds. The grammar state skips that line, so a comment or template literal that opens or closes on it mis-colors the following lines. Highlighting still runs synchronously during render, so a file with thousands of ordinary lines costs proportionally more.
+- **The Go struct-field rule is patched locally** — when the Go grammar loads, its repeated one-line struct-field rule is replaced with the rule [worlpaker/go-syntax#25](https://github.com/worlpaker/go-syntax/pull/25) merged, because the vendored rule backtracks exponentially on lines such as a brace-bearing comment after `struct {`. If a dependency upgrade removes that rule, loading the grammar throws and Go code renders as plain text; delete the patch once `@shikijs/langs` ships the upstream rule.
 - **The fish logo is a redrawn approximation** — its font glyph has no exportable vector geometry in the local design data, so a hand-authored recreation stands in until an exact export path exists.
 - **`Pill` and `Input` have no design source** — both atoms are self-defined; the sidebar search field and view-tab strip that resemble them are consumer-owned compositions, not these atoms.
 - **Product icons need the ui-theme sprite** — each icon only references its symbol, so without ui-theme, or inside another document such as an iframe, the icon renders nothing.

packages/client/ui-primitives/README.zh.md

@@ -220,6 +220,7 @@ kind: "package-library"
 - **流式期间跨边界引用解析被推迟**：定义落在增量冻结边界另一侧的引用式链接或脚注，在回复流式输出期间渲染为字面文本；定稿时的全量解析会将其解析。
 - **长高亮 fence 会保留完整 token DOM**：流式路径避免重新解析、重新 tokenize 和 reconcile 已完成前缀，但不会丢弃旧颜色或虚拟化 token span。因此最终 DOM 数量仍随 fence 的 token 数增长；嵌套／容器内 fence 与病态的单个超长行仍走通用尾部路径。
 - **超长行不着色**：所有高亮界面中，长度达到 1,000 个 UTF-16 单元的源码行渲染为一段纯文本，因为 TextMate 扫描成本随行长超线性增长，单个生成行否则会阻塞页面数秒。语法状态会跳过该行，因此在该行开始或结束的注释、模板字符串会使后续行着色错误。高亮仍在渲染期间同步执行，因此数千个普通行的文件仍按比例增加开销。
+- **Go 结构体字段规则在本地打了补丁**：Go 语法加载时，重复匹配单行结构体字段的规则会替换为 [worlpaker/go-syntax#25](https://github.com/worlpaker/go-syntax/pull/25) 合入的规则，因为内置规则在 `struct {` 之后含花括号注释等行上会指数级回溯。若依赖升级后该规则消失，加载语法会抛错，Go 代码以纯文本显示；`@shikijs/langs` 发布上游规则后删除此补丁。
 - **鱼形标志是重新绘制的近似版本**：它来自字体字形，而本地设计数据无法导出其矢量几何；在获得精确导出路径前，使用手工重建版本代替。
 - **`Pill` 与 `Input` 没有设计来源**：两个原子组件均自行定义；与其相似的侧边栏搜索字段和视图标签条由消费方组合，不是这些原子组件。
 - **产品图标依赖 ui-theme 的 sprite**：每个图标只引用自己的 symbol，因此没有 ui-theme 时，或在 iframe 等另一个文档中，图标不渲染任何内容。

packages/client/ui-primitives/src/markdown/highlight.ts

@@ -41,6 +41,39 @@ type LangModule = { default: typeof langTs }
  */
 const LANGS = [langTs, langBash, langJson]
 
+/**
+ * Replace the Go grammar's repeated one-line struct-field rule with the rule
+ * worlpaker/go-syntax merged in 4a9b120a52c (https://github.com/worlpaker/go-syntax/pull/25):
+ * the repeated rule backtracks exponentially on lines such as a brace-bearing
+ * comment after `struct {`. Delete this patch once `@shikijs/langs` ships that
+ * commit; 4.5.0 does not. The rule is located as the only rule in
+ * `struct_variables_types_fields` containing the field-token literal below,
+ * taken from `@shikijs/langs` 4.4.3.
+ * @param grammars - the `@shikijs/langs/go` default export, left unmodified.
+ * @returns copies of `grammars` with that rule's match replaced in the `go` grammar.
+ * @throws When the module has no `go` grammar, the grammar lacks
+ * `struct_variables_types_fields`, or that entry has no or several such rules.
+ * The lazy load then rejects unhandled and Go code renders as plain text.
+ */
+export function patchGoGrammar(grammars: LangModule['default']): LangModule['default'] {
+  const go = grammars.find(grammar => grammar.name === 'go')
+  const fields = go?.repository.struct_variables_types_fields
+  const patterns = fields?.patterns ?? []
+  const rules = patterns.filter(rule => typeof rule.match === 'string' && rule.match.includes('[^"/`\\s]+;?'))
+  if (go === undefined || fields === undefined || rules.length !== 1) {
+    throw new Error(`ui-primitives: expected exactly one Go struct-field rule to patch for worlpaker/go-syntax#25, found ${
+      rules.length}; update or delete the patch`)
+  }
+  const repository = { ...go.repository }
+  repository.struct_variables_types_fields = {
+    ...fields,
+    patterns: patterns.map(rule => rule === rules[0]
+      ? { ...rule, match: '(?:(?<=\\{)([^/\\`"]*[^\\s/\\`"])\\s*(?=\\}))' }
+      : rule),
+  }
+  return grammars.map(grammar => grammar === go ? { ...go, repository } : grammar)
+}
+
 /**
  * The non-boot extension grammars, each behind a dynamic import so its module
  * stays out of the boot chunk until a code surface renders that language.
@@ -53,7 +86,7 @@ const LANGS = [langTs, langBash, langJson]
 const LAZY_GRAMMARS = new Map<string, () => Promise<LangModule>>([
   ['python', () => import('@shikijs/langs/python')],
   ['ruby', () => import('@shikijs/langs/ruby')],
-  ['go', () => import('@shikijs/langs/go')],
+  ['go', async () => ({ default: patchGoGrammar((await import('@shikijs/langs/go')).default) })],
   ['rust', () => import('@shikijs/langs/rust')],
   ['java', () => import('@shikijs/langs/java')],
   ['c', () => import('@shikijs/langs/c')],

packages/client/ui-primitives/tests/go-highlighting.client.spec.ts

@@ -0,0 +1,72 @@
+/** Go regex matching runs in a terminable source-mode worker, not on the test runner's event loop. */
+import { Worker } from 'node:worker_threads'
+import { expect, it } from 'vitest'
+import { patchGoGrammar } from '../src/markdown/highlight.ts'
+import { GO_SOURCE } from './go-source.ts'
+
+it('highlights struct comments with braces without blocking settled or streaming code', async () => {
+  // The worker and highlight.ts use erasable syntax and only package-name imports, so Node loads them directly.
+  const worker = new Worker(new URL('./go-highlighting.worker.ts', import.meta.url))
+  let timer: ReturnType<typeof setTimeout> | undefined
+  try {
+    const result = await new Promise<unknown>((resolve, reject) => {
+      const fail = (message: string): void => { reject(new Error(message)) }
+      timer = setTimeout(() => { fail('Go grammar did not load within 120 seconds') }, 120_000)
+      worker.on('message', (reply: unknown) => {
+        if (reply !== 'loaded') {
+          resolve(reply)
+          return
+        }
+        clearTimeout(timer)
+        // Bound runaway synchronous matching once imports and grammar loading are done.
+        timer = setTimeout(() => { fail('Go highlighting did not complete within 10 seconds') }, 10_000)
+      })
+      worker.once('error', reject)
+      worker.once('exit', (code) => { fail(`Go highlighting worker exited before its reply: ${code}`) })
+    })
+    expect(result).toMatchObject({ text: GO_SOURCE, streamingMatchesSettled: true })
+    expect(result).toHaveProperty('html', expect.stringContaining('var(--shiki-'))
+    const keyword = (...texts: string[]): string[] => texts.map(text => `${text}:keyword`)
+    expect(result).toHaveProperty('colored', [
+      ['package:keyword', 'example:function'],
+      [],
+      ['type:keyword', 'Record:function', 'struct:keyword'],
+      ['// docs: {id} explains the ordinary field names for an example record ending here:comment'],
+      ['int:keyword'],
+      [],
+      [],
+      ['type:keyword', 'Inline:function', ...keyword('struct', 'string', 'int')],
+      ['type:keyword', 'Nested:function', ...keyword('struct', 'map', 'string', 'struct', 'struct', 'int')],
+      ['type:keyword', 'OneLineFields:function', ...keyword('struct', 'int', 'chan<-', 'struct', 'string'),
+        'sync:function', 'Mutex:function'],
+      ['type:keyword', 'SplitOneLine:function', ...keyword('struct', 'int'), 'B:function', 'string:keyword'],
+      [],
+      [...keyword('var', '=', 'struct', 'string'), 'Port:function', 'int:keyword', 'Tags:function', 'string:keyword'],
+      [],
+      ['type:keyword', 'Many:function', ...keyword('struct', ...Array<string>(24).fill('int'), 'struct', 'int')],
+      ['type:keyword', 'Open:function', 'struct:keyword', ...Array<string>(47).fill('field:function')],
+    ])
+  } finally {
+    clearTimeout(timer)
+    await worker.terminate()
+  }
+}, 150_000)
+
+it('patches exactly the vendored struct-field rule and rejects a grammar without it', async () => {
+  const { default: grammars } = await import('@shikijs/langs/go')
+  const other = { name: 'go-embedded', scopeName: 'source.go.embedded', patterns: [], repository: {} }
+  const before = grammars[0]?.repository.struct_variables_types_fields?.patterns
+  const patched = patchGoGrammar([...grammars, other])
+  const after = patched[0]?.repository.struct_variables_types_fields?.patterns
+  const changed = after?.flatMap((rule, index) => rule === before?.[index] ? [] : [[before?.[index]?.match, rule.match]])
+  expect(changed).toEqual([[
+    expect.stringContaining('[^"/`\\s]+;?'),
+    '(?:(?<=\\{)([^/\\`"]*[^\\s/\\`"])\\s*(?=\\}))',
+  ]])
+  expect(patched[1]).toBe(other)
+  for (const repository of [{}, { struct_variables_types_fields: {} }]) {
+    expect(() => patchGoGrammar([{ name: 'go', scopeName: 'source.go', patterns: [], repository }]))
+      .toThrow('expected exactly one Go struct-field rule to patch for worlpaker/go-syntax#25, found 0')
+  }
+  expect(() => patchGoGrammar([other])).toThrow('found 0')
+})

packages/client/ui-primitives/tests/go-highlighting.worker.ts

@@ -0,0 +1,31 @@
+/** Source-mode highlighter fixture; the parent owns termination during synchronous regex matching. */
+import { parentPort } from 'node:worker_threads'
+import {
+  highlightLines, highlightToHtml, StreamingHighlightSession, subscribeGrammarLoaded,
+} from '../src/markdown/highlight.ts'
+import { GO_SOURCE } from './go-source.ts'
+
+const ready = Promise.withResolvers<undefined>()
+const dispose = subscribeGrammarLoaded(() => {
+  if (highlightLines('package example', 'go') !== undefined) ready.resolve(undefined)
+})
+try {
+  if (highlightLines('package example', 'go') === undefined) await ready.promise
+} finally {
+  dispose()
+}
+// The parent times matching separately from imports and grammar loading.
+parentPort?.postMessage('loaded')
+const lines = highlightLines(GO_SOURCE, 'go')
+const stream = new StreamingHighlightSession()
+const split = GO_SOURCE.indexOf('\t//')
+if (split <= 0) throw new Error('GO_SOURCE no longer contains the brace-bearing struct comment')
+stream.update(GO_SOURCE.slice(0, split), 'go')
+const streamed = stream.update(GO_SOURCE, 'go')
+parentPort?.postMessage({
+  text: lines?.map(line => line.map(span => span.text).join('')).join('\n'),
+  colored: lines?.map(line => line.flatMap(span =>
+    span.style.color === 'var(--shiki-foreground)' ? [] : [`${span.text}:${String(span.style.color).replace('var(--shiki-token-', '').replace(')', '')}`])),
+  streamingMatchesSettled: JSON.stringify(streamed) === JSON.stringify(new StreamingHighlightSession().update(GO_SOURCE, 'go')),
+  html: highlightToHtml(GO_SOURCE, 'go'),
+})

packages/client/ui-primitives/tests/go-source.ts

@@ -0,0 +1,21 @@
+/** Synthetic Go struct comments and declarations for syntax-highlighting regressions. */
+export const GO_SOURCE = [
+  'package example',
+  '',
+  'type Record struct {',
+  '\t// docs: {id} explains the ordinary field names for an example record ending here',
+  '\tValue int',
+  '}',
+  '',
+  'type Inline struct { Name string; Count int }',
+  'type Nested struct { Flags map[string]struct{}; Child struct { Value int } }',
+  // Shapes from worlpaker/go-syntax#25's test/semantic_tokens.go and test/stress.go.
+  'type OneLineFields struct{ A, B int; C chan<- struct{}; D []string; sync.Mutex }',
+  'type SplitOneLine struct { A int; B string',
+  '}',
+  'var stressFoo14 = struct{ Host string; Port int; Tags []string',
+  '}{}',
+  `type Many struct { ${Array.from({ length: 24 }, (_, index) => `F${index} int;`).join(' ')} Child struct { X int } }`,
+  // Last: its unclosed brace keeps the struct body open to the end of the source.
+  `type Open struct { ${'field '.repeat(48)}`,
+].join('\n')