From 6053e1a11b19325502ef8645f81d7097d521b87b Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Wed, 4 Nov 2026 11:28:28 -0500 Subject: [PATCH] fix(clv2): keep instinct-cli running on Python 3.6 and newer The hooks fall back to any Python 3 that runs, so users on an older interpreter (RHEL 8's 3.6, for one) still reach instinct-cli. It failed there at import: builtin generics such as list[dict] in signatures are evaluated when the function is defined, which raises TypeError before 3.9, and str.removeprefix is 3.9+. - Quote the builtin-generic annotations. Type checkers read them the same way, and they stay valid under 3.14's deferred annotations. - Strip the byte-order mark with startswith and a slice. - Open the registry lock file with an explicit UTF-8 encoding. - Say in the module docstring that 3.6 is the floor and what keeps it. The new test parses the script with the oldest grammar ast offers and rejects what a grammar check cannot see: unquoted builtin generics in evaluated annotations, `from __future__ import annotations`, and 3.7+ methods and keywords (capture_output, text, removeprefix and similar). A sample file proves it catches each of those. It also runs import, status, export, evolve, projects and prune under -X dev -X warn_default_encoding -W error, so any deprecation or default-encoding warning on the current Python fails the build. CI runs only current Pythons; this is a static guard, not a run on a real 3.6. --- .../scripts/instinct-cli.py | 34 +-- .../instinct-cli-python-compat.test.js | 212 ++++++++++++++++++ 2 files changed, 232 insertions(+), 14 deletions(-) create mode 100644 tests/scripts/instinct-cli-python-compat.test.js diff --git a/skills/continuous-learning-v2/scripts/instinct-cli.py b/skills/continuous-learning-v2/scripts/instinct-cli.py index 726fac8d6..49636b639 100755 --- a/skills/continuous-learning-v2/scripts/instinct-cli.py +++ b/skills/continuous-learning-v2/scripts/instinct-cli.py @@ -13,6 +13,10 @@ Commands: promote - Promote project instincts to global scope projects - List all known projects and their instinct counts prune - Delete pending instincts older than 30 days (TTL) + +Runs on Python 3.6 and newer. Keep it that way: annotations that subscript +builtins (list[dict]) stay quoted, and newer-only APIs such as +str.removeprefix or subprocess's capture_output/text are avoided. """ import argparse @@ -430,7 +434,7 @@ def _registry_lock(): lock_fd = None try: if _HAS_FCNTL: - lock_fd = open(lock_path, "w") + lock_fd = open(lock_path, "w", encoding="utf-8") fcntl.flock(lock_fd, fcntl.LOCK_EX) yield finally: @@ -524,7 +528,7 @@ def _validate_project_id(project_id: str) -> bool: # Instinct Parser # ───────────────────────────────────────────── -def parse_instinct_file(content: str) -> list[dict]: +def parse_instinct_file(content: str) -> "list[dict]": """Parse YAML-like instinct file format. Each instinct is delimited by a pair of ``---`` markers (YAML frontmatter). @@ -536,7 +540,9 @@ def parse_instinct_file(content: str) -> list[dict]: in_frontmatter = False content_lines = [] - for line in content.removeprefix('\ufeff').split('\n'): + if content.startswith('\ufeff'): + content = content[1:] + for line in content.split('\n'): if line.strip() == '---': if in_frontmatter: # End of frontmatter - content comes next @@ -582,7 +588,7 @@ def parse_instinct_file(content: str) -> list[dict]: return [i for i in instincts if i.get('id')] -def _load_instincts_from_dir(directory: Path, source_type: str, scope_label: str) -> list[dict]: +def _load_instincts_from_dir(directory: Path, source_type: str, scope_label: str) -> "list[dict]": """Load instincts from a single directory.""" instincts = [] if not directory.exists(): @@ -645,7 +651,7 @@ def _remove_project_storage(project_id: str) -> None: shutil.rmtree(project_dir) -def _project_instinct_ids(project_dir: Path, source_type: str) -> set[str]: +def _project_instinct_ids(project_dir: Path, source_type: str) -> "set[str]": instinct_dir = project_dir / "instincts" / source_type return { inst.get("id") @@ -654,7 +660,7 @@ def _project_instinct_ids(project_dir: Path, source_type: str) -> set[str]: } -def _merge_instinct_dir(from_dir: Path, into_dir: Path, existing_ids: set[str]) -> tuple[int, int]: +def _merge_instinct_dir(from_dir: Path, into_dir: Path, existing_ids: "set[str]") -> "tuple[int, int]": moved = 0 skipped = 0 if not from_dir.exists(): @@ -705,7 +711,7 @@ def _append_observations(from_project_dir: Path, into_project_dir: Path) -> int: return len([line for line in lines if line.strip()]) -def load_all_instincts(project: dict, include_global: bool = True) -> list[dict]: +def load_all_instincts(project: dict, include_global: bool = True) -> "list[dict]": """Load all instincts: project-scoped + global. Project-scoped instincts take precedence over global ones when IDs conflict. @@ -740,7 +746,7 @@ def load_all_instincts(project: dict, include_global: bool = True) -> list[dict] return instincts -def load_project_only_instincts(project: dict) -> list[dict]: +def load_project_only_instincts(project: dict) -> "list[dict]": """Load only project-scoped instincts (no global). In global fallback mode (no git project), returns global instincts. @@ -863,7 +869,7 @@ def _warn_legacy_data() -> None: print(f"{'!'*60}\n") -def _print_instincts_by_domain(instincts: list[dict]) -> None: +def _print_instincts_by_domain(instincts: "list[dict]") -> None: """Helper to print instincts grouped by domain.""" by_domain = defaultdict(list) for inst in instincts: @@ -1503,7 +1509,7 @@ def _show_promotion_candidates(project: dict) -> None: print(f" Run `instinct-cli.py promote` to promote these to global scope.\n") -def _frontmatter_scalar(lines: list[str], key: str) -> Optional[str]: +def _frontmatter_scalar(lines: "list[str]", key: str) -> Optional[str]: """Extract a simple scalar value from frontmatter lines.""" for line in lines: if ':' not in line: @@ -1520,7 +1526,7 @@ def _frontmatter_scalar(lines: list[str], key: str) -> Optional[str]: return None -def _remove_instinct_blocks(content: str, instinct_id: str) -> tuple[str, int]: +def _remove_instinct_blocks(content: str, instinct_id: str) -> "tuple[str, int]": """Remove raw frontmatter blocks with a matching instinct ID.""" lines = content.splitlines(keepends=True) retained = [] @@ -1967,7 +1973,7 @@ def _evolved_description(trigger: str, instincts: list, kind: str) -> str: return description.replace(': ', ' - ').replace('<', '(').replace('>', ')') -def _generate_evolved(skill_candidates: list, workflow_instincts: list, agent_candidates: list, evolved_dir: Path, limit: int = 0) -> list[str]: +def _generate_evolved(skill_candidates: list, workflow_instincts: list, agent_candidates: list, evolved_dir: Path, limit: int = 0) -> "list[str]": """Generate skill/command/agent files from analyzed instinct clusters. ``limit`` caps how many candidates of each kind are written; 0 writes them @@ -2078,7 +2084,7 @@ def _generate_evolved(skill_candidates: list, workflow_instincts: list, agent_ca # Pending Instinct Helpers # ───────────────────────────────────────────── -def _collect_pending_dirs() -> list[Path]: +def _collect_pending_dirs() -> "list[Path]": """Return all pending instinct directories (global + per-project).""" dirs = [] global_pending = GLOBAL_INSTINCTS_DIR / "pending" @@ -2139,7 +2145,7 @@ def _parse_created_date(file_path: Path) -> Optional[datetime]: return None -def _collect_pending_instincts() -> list[dict]: +def _collect_pending_instincts() -> "list[dict]": """Scan all pending directories and return info about each pending instinct. Each dict contains: path, created, age_days, name, parent_dir. diff --git a/tests/scripts/instinct-cli-python-compat.test.js b/tests/scripts/instinct-cli-python-compat.test.js new file mode 100644 index 000000000..999ab4f1f --- /dev/null +++ b/tests/scripts/instinct-cli-python-compat.test.js @@ -0,0 +1,212 @@ +/** + * Tests that instinct-cli.py keeps running on old and new Pythons. + * + * The CLI supports Python 3.6 and newer, but CI runs a current Python, so a + * newer-only construct would not fail anywhere until a user on an old + * interpreter hit it. The static check below parses the script with the 3.6 + * grammar and rejects the runtime constructs a grammar check cannot see: + * unquoted builtin generics in evaluated annotations and newer-only APIs. + * The last case runs the CLI under the strictest warning settings of the + * current Python. + * + * Run with: node tests/scripts/instinct-cli-python-compat.test.js + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +let passed = 0; +let failed = 0; + +const repoRoot = path.resolve(__dirname, '..', '..'); +const cliPath = path.join( + repoRoot, + 'skills', + 'continuous-learning-v2', + 'scripts', + 'instinct-cli.py' +); + +function detectPython3() { + for (const bin of ['python3', 'python']) { + const r = spawnSync(bin, ['--version'], { encoding: 'utf8' }); + if (r.status === 0 && /Python 3\.(\d+)/.test(r.stdout + r.stderr)) return bin; + } + return null; +} + +const PYTHON3 = detectPython3(); +if (!PYTHON3) { + console.log('\n=== Testing instinct-cli.py Python compatibility ===\n'); + console.log(' - skipped: Python 3 not found in PATH'); + console.log('\nPassed: 0'); + console.log('Failed: 0'); + process.exit(0); +} + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + passed += 1; + } catch (error) { + console.log(` ✗ ${name}`); + console.log(` Error: ${error.message}`); + failed += 1; + } +} + +function createTempDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-instinct-cli-compat-')); +} + +function cleanupDir(dir) { + fs.rmSync(dir, { recursive: true, force: true }); +} + +function runPython(args, options = {}) { + return spawnSync(PYTHON3, args, { + encoding: 'utf8', + timeout: 60000, + ...options, + env: { ...process.env, ...(options.env || {}) }, + }); +} + +// Prints one line per construct that would break on Python 3.6. +const COMPAT_CHECK = String.raw` +import ast, sys + +path = sys.argv[1] +src = open(path, encoding="utf-8").read() +problems = [] + +try: + tree = ast.parse(src, filename=path, feature_version=(3, 6)) +except SyntaxError as exc: + print("syntax newer than 3.6 at line %s: %s" % (exc.lineno, exc.msg)) + sys.exit(0) + +BUILTIN_GENERICS = {"list", "dict", "set", "tuple", "type", "frozenset"} +NEWER_METHODS = {"removeprefix", "removesuffix", "is_relative_to", "with_stem"} +# Keywords newer than 3.6, keyed by the callable they belong to. +NEWER_KEYWORDS = { + "run": {"capture_output", "text"}, + "Popen": {"text"}, + "check_output": {"text"}, + "copytree": {"dirs_exist_ok"}, + "unlink": {"missing_ok"}, + "zip": {"strict"}, +} + +def call_name(func): + if isinstance(func, ast.Attribute): + return func.attr + if isinstance(func, ast.Name): + return func.id + return "" + +def check_annotation(node, where): + if node is None: + return + for sub in ast.walk(node): + if (isinstance(sub, ast.Subscript) and isinstance(sub.value, ast.Name) + and sub.value.id in BUILTIN_GENERICS): + problems.append("line %d: unquoted %s[...] annotation in %s" % (sub.lineno, sub.value.id, where)) + +for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "__future__": + for alias in node.names: + if alias.name == "annotations": + problems.append("line %d: from __future__ import annotations needs 3.7" % node.lineno) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + all_args = getattr(node.args, "posonlyargs", []) + node.args.args + node.args.kwonlyargs + for extra in (node.args.vararg, node.args.kwarg): + if extra is not None: + all_args.append(extra) + for arg in all_args: + check_annotation(arg.annotation, node.name) + check_annotation(node.returns, node.name) + if isinstance(node, ast.AnnAssign): + check_annotation(node.annotation, "a variable annotation") + if isinstance(node, ast.Attribute) and node.attr in NEWER_METHODS: + problems.append("line %d: .%s() is newer than 3.6" % (node.lineno, node.attr)) + if isinstance(node, ast.Call): + newer = NEWER_KEYWORDS.get(call_name(node.func), set()) + for kw in node.keywords: + if kw.arg in newer: + problems.append("line %d: keyword %s= is newer than 3.6" % (node.lineno, kw.arg)) + +for problem in problems: + print(problem) +`; + +console.log('\n=== Testing instinct-cli.py Python compatibility ===\n'); + +test('the script uses nothing newer than Python 3.6', () => { + const r = runPython(['-c', COMPAT_CHECK, cliPath]); + assert.strictEqual(r.status, 0, r.stderr); + assert.strictEqual(r.stdout.trim(), '', `3.6-incompatible constructs:\n${r.stdout}`); +}); + +test('the compatibility check catches what it is meant to catch', () => { + const tmp = createTempDir(); + try { + const sample = path.join(tmp, 'sample.py'); + fs.writeFileSync( + sample, + [ + 'import subprocess', + 'def f(x: list[str]) -> dict[str, int]:', + ' subprocess.run(["git"], capture_output=True, text=True)', + ' return x[0].removeprefix("a")', + '', + ].join('\n') + ); + const r = runPython(['-c', COMPAT_CHECK, sample]); + assert.strictEqual(r.status, 0, r.stderr); + for (const expected of ['list[...]', 'dict[...]', 'capture_output=', 'text=', '.removeprefix()']) { + assert.ok(r.stdout.includes(expected), `missing ${expected} in:\n${r.stdout}`); + } + } finally { + cleanupDir(tmp); + } +}); + +test('commands run with every warning turned into an error', () => { + const tmp = createTempDir(); + try { + const projectDir = path.join(tmp, 'repo'); + fs.mkdirSync(projectDir, { recursive: true }); + const init = spawnSync('git', ['init', '-q'], { cwd: projectDir, encoding: 'utf8' }); + assert.strictEqual(init.status, 0, init.stderr); + + const env = { + CLV2_HOMUNCULUS_DIR: path.join(tmp, 'homunculus'), + CLAUDE_PROJECT_DIR: projectDir, + }; + const sample = path.join(repoRoot, '.claude', 'homunculus', 'instincts', 'inherited', 'ecc-instincts.yaml'); + const strict = ['-X', 'dev', '-X', 'warn_default_encoding', '-W', 'error', cliPath]; + const commands = [ + ['import', sample, '--force', '--scope', 'global'], + ['status'], + ['export', '--output', path.join(tmp, 'export.yaml')], + ['evolve'], + ['projects'], + ['prune', '--dry-run'], + ]; + for (const command of commands) { + const r = runPython([...strict, ...command], { cwd: projectDir, env }); + assert.strictEqual(r.status, 0, `${command.join(' ')} exited ${r.status}:\n${r.stderr}`); + } + } finally { + cleanupDir(tmp); + } +}); + +console.log(`\nPassed: ${passed}`); +console.log(`Failed: ${failed}`); +process.exit(failed > 0 ? 1 : 0);