From fa47edadbc259057a841db32181d1d73095f4a10 Mon Sep 17 00:00:00 2001 From: JavaZero <71128095+JavaZeroo@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:44:46 +0800 Subject: [PATCH 1/2] feat: execute frozen grapheme cost batches with evidence guards (#54) Preserve strict UTF-8 raw outputs, failed-run identity, interruption records and paired statistics. Add read-only host preflight; retain its rejected shared-host run without performance claims. --- CMakeLists.txt | 2 +- docs/EVIDENCE.md | 1 + docs/FEATURES.md | 2 +- experiments/measure/R54-COST.md | 33 ++- experiments/measure/r54_batch.py | 261 ++++++++++++++++++ experiments/measure/r54_environment.py | 102 +++++++ experiments/measure/r54_inputs.py | 131 +++++++++ .../2026-10-27-r54-runner/validation.json | 163 +++++++++++ experiments/measure/test_r54_batch.py | 202 ++++++++++++++ 9 files changed, 894 insertions(+), 3 deletions(-) create mode 100644 experiments/measure/r54_batch.py create mode 100644 experiments/measure/r54_environment.py create mode 100644 experiments/measure/r54_inputs.py create mode 100644 experiments/measure/results/2026-10-27-r54-runner/validation.json create mode 100644 experiments/measure/test_r54_batch.py diff --git a/CMakeLists.txt b/CMakeLists.txt index e5b3e38a..c6a2ffbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,7 +292,7 @@ if(BUILD_TESTING) endif() termforge_add_python_test(core -m unittest experiments.core.test_core experiments.core.test_unicode -v) termforge_add_python_test(recovery -m unittest experiments.core.test_recovery -v) - termforge_add_python_test(measure -m unittest experiments.measure.test_measure experiments.measure.test_r54_contract.CostPlanTests -v) + termforge_add_python_test(measure -m unittest experiments.measure.test_measure experiments.measure.test_r54_contract.CostPlanTests experiments.measure.test_r54_batch -v) termforge_add_python_test(binary_paths -m unittest experiments.test_build_paths -v) termforge_add_python_test(conpty_margin_contract -m unittest experiments.conpty.test_margin_contract -v) termforge_add_python_test(benchmark_tools -m unittest discover -s benchmarks/tests -v) diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index 015f836d..c04c3ced 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -162,5 +162,6 @@ python -m unittest experiments.shell.test_workflow_sessions -v | 当前成本调查 | 结果与合同 | 结论边界 | |---|---|---| | 候选 `.3`→`.4` 字素改动 | [固定字素验证](../experiments/shell/results/2026-10-26-streaming-graphemes/validation.json)、[性能限制与原始样本身份](research/streaming-graphemes.md)、[#54](https://github.com/JavaZeroo/termforge/issues/54) | 同宿主交错诊断中 project/interactive 吞吐中位数约下降 16%/9%；共享宿主、allocator 包装、无真实呈现/输入时延，使其不能作为正式预算或性能通过。#54 受控定位仍阻塞 #34 的性能裁决。 | +| 固定字素成本的批次协议与执行 | [R54 v1 合同和构建/环境记录](../experiments/measure/R54-COST.md)、[执行器](../experiments/measure/r54_batch.py)、[合成进程回归](../experiments/measure/test_r54_batch.py)、[实际预检及限定验证](../experiments/measure/results/2026-10-27-r54-runner/validation.json) | 固定两个种子、各 112 个位置，字节原样保留、首错停止、完整批次后逐组配对统计。共享工作站十区间活动预检未通过，未启动成本样本；测量相关 25 项测试通过。5% 是环境诊断护栏，并非产品性能容差；实际批次、扰动归因及用户预算确认由 #54 接续。 | 目前没有足够证据判断 TermForge 距离 WindTerm 快或慢多少。正式结论必须同时提供负载/配置/环境身份、正确性、完成端点、观测开销、重复样本与限制。未知字段保持未知；历史校准未完成不转成要求用户反复手动运行的前置。执行规则见 [ADR-0002](adr/0002-automation-first-investigation.md)，核心选型重审见 [ADR-0003](adr/0003-core-reference-and-reuse.md)。 diff --git a/docs/FEATURES.md b/docs/FEATURES.md index aa28f292..3eead283 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -71,7 +71,7 @@ | 自动正确性与构建 | **限定验收**：Windows shell/ConPTY/呈现及 Windows/Linux 核心 CI、差分/恢复/队列/生命周期场景 | 产品打包、干净机器使用、依赖升级、长期回归和更广平台矩阵；#40 的 CI 纯夹具超时、窄窗口选择 #73、四 shell 原事件 #80、hosted 准备 #81 和[复制夹具 OpenClipboard 5 #85](https://github.com/JavaZeroo/termforge/issues/85) 等独立失败仍未闭合；#72 保存判据与 #80 同步条件的限定修正不代替其他配置或未知根因验收 | R2 / R4 | | Windows 产品交付 | **实现中，未验收**：首个 [v0.1.0-alpha.1 预发布](releases/v0.1.0-alpha.1.md)提供 Windows x64、静态 MSVC 运行库、附资源许可和构建身份的免安装 ZIP；未签名，已验环境限 Windows 11 本机与 Server 2025 CI，布局仍写入 LOCALAPPDATA | 这是 R1 体验包，尚非 R2 日常开发版；干净机器与更广平台矩阵、签名/安装、配置升级及持续运行仍待完成。Windows 10 未验收，Server 2022 已知资源门槛未过 | R2 / R3 | | Linux/macOS 与其他架构 | **未实现**客户端；Linux 核心 CI 已有。[官方平台范围台账](../research/windterm-2.7.0-platform-scope.md)已固定 2.7.0 的 Windows x86/x64、macOS x64、Linux x64 资产及声明下限；这不是 TermForge 支持或实测结论 | 最终需真实 Linux/macOS 客户端；平台调查可提前进行，整合及完整验收排在 Windows 11 x64 完整版本之后。其他系统版本、最低版本和架构由用户后续明确；旧系统及 ARM 等保持调查记录，不自动承诺或排除，也不能由核心 CI 推断产品支持 | 持续调查；R5 全平台出口 | -| 吞吐、响应、内存与稳定性 | **实现中，未验收**：已有机制实验、连续 CPU 区间校准与合成回归；候选字素改动的受限诊断中 project/interactive 吞吐中位数约下降 16%/9%，由 [#54](https://github.com/JavaZeroo/termforge/issues/54) 跟踪 | 共享宿主、allocator 包装与缺少真实呈现/输入时延端点使该诊断不构成正式预算。仍需同负载/同正确性、无插桩对照、观测开销和长期/多会话公平性；可靠原版基线后提出逐项预算及允许波动范围，**由用户明确确认**。与 WindTerm 的实际差距**待取证** | 当前优先；贯穿 R1–R5 | +| 吞吐、响应、内存与稳定性 | **实现中，未验收**：已有机制实验、连续 CPU 区间校准与合成回归；候选字素改动的受限诊断中 project/interactive 吞吐中位数约下降 16%/9%，由 [#54](https://github.com/JavaZeroo/termforge/issues/54) 跟踪。[冻结批次执行器](../experiments/measure/R54-COST.md#冻结批次执行器windows)核对构建来源、原始输出、停止规则与逐轮配对统计；其环境护栏和合成进程测试不构成正式成本样本 | 共享宿主、allocator 包装与缺少真实呈现/输入时延端点使该诊断不构成正式预算。仍需同负载/同正确性、无插桩对照、观测开销和长期/多会话公平性；可靠原版基线后提出逐项预算及允许波动范围，**由用户明确确认**。与 WindTerm 的实际差距**待取证** | 当前优先；贯穿 R1–R5 | | 大历史与动态内存压缩 | **未实现**产品能力；WindTerm 压缩属于官方性能声明，内部机制尚未证实 | 1/10/100 万行等明确规模下的内存/滚动/搜索/淘汰实验；根据瓶颈决定独立存储/压缩实现 | R4 | | 完整目标验收 | **待取证**：已有固定版本和局部观察，没有完整行为/性能对照矩阵 | 先在 Windows 11 x64 完整验收功能/行为/UI/性能及独立构建，再完成 Linux/macOS 全平台出口；重要差异和性能容差须有用户确认，Windows 通过不代表全平台完成；不依赖 WindTerm 二进制或专有资源 | R5 分平台出口 | diff --git a/experiments/measure/R54-COST.md b/experiments/measure/R54-COST.md index f4a16c0b..15c7dd46 100644 --- a/experiments/measure/R54-COST.md +++ b/experiments/measure/R54-COST.md @@ -59,4 +59,35 @@ $env:TERMFORGE_R54_UNWRAPPED = '' python -m unittest experiments.measure.test_r54_contract.OptionalAllocatorBuildSmoke -v ``` -该批只交付准备合同与可完全移除包装的构建机制。两个实际 build manifest/EXE、真实空闲环境证据、冻结计划执行器和重复样本、成本归因、数值预算与优化验收仍由 #54 接续；生成计划或短 sanity 都不关闭该 issue。 +## 冻结批次执行器（Windows） + +[r54_batch.py](r54_batch.py) 执行原 v1 的完整种子顺序，没有重试、补样或缩短时长开关。它只读取已有构建；不会在计时阶段编译。`samples_complete` 只表示样本及顺序通过验证，外层 `complete_diagnostic` 还要求构建绑定与本节环境检查通过；二者均保留 `performance_gate=not_evaluated`，不等于性能验收。 + +执行前需要保留以下私有构建目录。其 v1 格式由 [r54_inputs.py](r54_inputs.py) 检查，不接受只填源码 SHA 和 EXE SHA 的简化声明： + +- `build-plan.json`：`termforge-r54-local-build-plan-v1`，包含原计划文件哈希、共同 harness 提交、编译器/工具版本、环境变量、完整构建命令、两份原提交/树及每个候选目录跟踪文件覆盖前后的 SHA。 +- `source-baseline/`、`source-candidate/`：固定 `.3/.4` checkout，只覆盖根 Cargo.toml 和 src/measure.rs；原锁文件和 vendor 不变。 +- `baseline-build.json`、`candidate-build.json`、各自 `*-build.stdout/stderr`、`*-harness.patch` 和 `*-identity.stdout/stderr`：实际构建、原始日志、覆盖 diff 与编译模式。 +- `build-audit.json`：原构建审计，包括 receipt 哈希、实际 linker 路径和 SHA、核心来源、版本与二进制身份。执行器重新核对当前源文件、全部日志和 EXE，再实际运行 `--identity`。这是一组可审计记录，不是二进制来源的密码学证明。 + +当前绑定明确支持已有 Rust 1.92.0 / x86_64-pc-windows-msvc / Release / locked/offline / no-default-features / jobs=1 / 独立 target 配置；其他构建需要另行形成记录和审查，不能静默混入。该准备格式及构建产物保持在 `local/`，不包含在公开 CI 的默认产物中。 + +环境声明 JSON 必须显式填写 `host_kind`（`shared_workstation`、`hosted_vm` 或 `dedicated_host`）、`known_concurrent_work` 字符串列表和 `no_known_concurrent_build_or_test=true`。声明不能取代实际观测。源工作树必须已提交且干净，输出目录必须不存在。 + +```powershell +python -m experiments.measure.r54_batch --plan local/r54-batch/plan-v1.json --build-bundle local/r54-batch --environment local/r54-environment.json --seed 20261026 --out local/r54-run-20261026 +# 第二批另行启动，使用原第二种子和新目录；第一批失败时先保留并调查原因。 +python -m experiments.measure.r54_batch --plan local/r54-batch/plan-v1.json --build-bundle local/r54-batch --environment local/r54-environment.json --seed 20261027 --out local/r54-run-20261027 +``` + +环境护栏 `termforge-r54-environment-guard-v1` 在任何正式成本样本之前冻结：一次观测十个连续约一秒区间，**每个**区间全机 busy CPU 必须不超过 5%；任一区间失败即保留完整预检，不启动成本负载，也不循环等到一次通过。该 5% 是保守的环境诊断筛选值，**不是用户批准的性能回归容差或 WindTerm 等价标准**。两批之间仍需重新观察；宿主背景工作发生变化是新环境证据，不能替换原失败。 + +Windows 1–64 逻辑 CPU 上使用 [GetSystemTimes](https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getsystemtimes) 原始 user/kernel/idle 计数，busy 为 user+kernel−idle，保留原始计数；kernel 已含 idle，不重复累加。每个子进程前后继续记录全机计数，将全机 busy 减去**内层**测量 CPU 后的余额作为背景活动的保守上界估计（包含测量区间外的准备、oracle、序列化及 runner 工作）。余额为负或其占全机总 CPU 时间超过 5% 时，保留该样本并停止批次。不从主样本成本扣除该余额，不把它当成精确的进程归因。该护栏不能证明虚拟机物理宿主独占、频率稳定或完全无干扰。 + +原始目录保留 OS build/UBR、CPU、逻辑/物理核数、电源策略前后、进程 CPU 身份、并行工作声明、所有环境区间及执行器源码身份。进程记录不读取命令行或秘密环境变量，但仍属私有证据；公开前须脱敏审核。 + +每个计划位置先落盘命令和编号，再把 stdout/stderr **直接写入二进制文件**；超时、无效 UTF-8、坏 JSON、非零退出和 oracle 错误都保留原字节与 manifest。子进程超时按原 90 秒处理；监督对象是不会再派生子进程的测量 EXE，不据此声称 Windows Job 回收。中断保留当前位置；不存在续跑或覆盖旧目录的入口。每个有效样本重查 EXE 哈希，结束后再核对源/构建证据及电源策略。 + +全部 112 个位置完整有效后才生成统计；缺失、重复、乱序或失败不能得到成功摘要。摘要逐批、逐 route/chunk 列出七个配对 CPU ns/B 比值，以及各源 CPU/墙钟 ns/B 全部值、中位数、范围和 MAD。合成进程测试覆盖整个顺序、原始字节和停止规则，其虚构时长只验证统计与进程协议，绝不作为性能样本。 + +实际受控重复样本、allocator/projection 扰动归因、预算提案和优化验收仍由 #54 接续；构建、短 sanity、执行器测试或环境预检均不关闭该 issue。 diff --git a/experiments/measure/r54_batch.py b/experiments/measure/r54_batch.py new file mode 100644 index 00000000..65f8523a --- /dev/null +++ b/experiments/measure/r54_batch.py @@ -0,0 +1,261 @@ +"""Execute the frozen R54 sequence without retries or a product speed verdict.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import statistics +import subprocess +import tempfile + +from experiments.measure.r54_contract import CHUNKS, CPU_SECONDS, SEEDS, SOURCES, sample_order, validate_sample + + +def utc(): + return datetime.now(timezone.utc).isoformat() + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def write_json(path, value): + """Replace only a live manifest; frozen inputs and stream files use create-only.""" + # A previous interrupted write may leave a partial temporary file. Give + # every attempt its own name so final failure/interruption recording still + # works; leave incomplete files for diagnosis rather than overwriting them. + with tempfile.NamedTemporaryFile(mode='w', encoding='utf-8', dir=path.parent, + prefix=path.name + '.', suffix='.tmp', delete=False) as stream: + temporary = Path(stream.name) + json.dump(value, stream, ensure_ascii=False, indent=2, allow_nan=False) + stream.write('\n') + stream.flush() + os.fsync(stream.fileno()) + temporary.replace(path) + + +def capture(command, output, timeout=90): + """Direct-child streams stay byte-exact even on invalid UTF-8 or timeout.""" + result = dict(command=command, started_utc=utc(), exit_code=None, timed_out=False) + with (output / 'stdout.bin').open('xb') as stdout, (output / 'stderr.bin').open('xb') as stderr: + try: + result['exit_code'] = subprocess.run(command, stdout=stdout, stderr=stderr, + stdin=subprocess.DEVNULL, timeout=timeout).returncode + except subprocess.TimeoutExpired: + result.update(timed_out=True, error=f'direct child timed out after {timeout}s') + except OSError as error: + result['error'] = str(error) + result['finished_utc'] = utc() + result['streams'] = {name: dict(bytes=(output / name).stat().st_size, sha256=sha(output / name)) + for name in ('stdout.bin', 'stderr.bin')} + return result + + +def run_schedule(commands, seed, output, context): + """Process/artifact seam. Provenance and environment checks belong to the CLI. + + samples_complete means only the ordered process results validated. It never + establishes host isolation, physical CPU exclusivity or performance parity. + """ + if seed not in SEEDS or set(commands) != set(SOURCES): + raise ValueError('unknown seed or source pair') + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + write_json(output / 'context.json', context) + manifest = dict(schema='termforge-r54-batch-v1', seed=seed, status='running', + started_utc=utc(), attempted=0, valid_samples=0, + active_sequence=None, performance_gate='not_evaluated', + context_sha256=sha(output / 'context.json')) + records = [] + try: + with (output / 'samples.jsonl').open('x', encoding='utf-8') as samples: + for item in sample_order(seed): + directory = output / f"{item['sequence']:03d}" + directory.mkdir() + command = list(commands[item['variant']]) + [item['route'], str(item['chunk']), '0', str(CPU_SECONDS)] + record = dict(item=item, status='started', command=command) + manifest.update(active_sequence=item['sequence'], attempted=manifest['attempted'] + 1) + write_json(output / 'manifest.json', manifest) + write_json(directory / 'record.json', record) + try: + monitored = context.get('environment_policy') is not None + if monitored: + from experiments.measure.r54_environment import POLICY, cpu_delta, cpu_snapshot + expected_sha = context['builds'][item['variant']]['binary']['sha256'] + if sha(command[0]) != expected_sha: + raise ValueError('binary changed before sample') + before = cpu_snapshot() + record.update(capture(command, directory)) + if monitored: + after = cpu_snapshot() + record['environment'] = dict(cpu_delta(before, after), before=before, after=after) + if record['timed_out'] or record['exit_code'] != 0: + raise ValueError(record.get('error', 'nonzero child exit')) + result = json.loads((directory / 'stdout.bin').read_bytes().decode('utf-8')) + validate_sample(result, item) + record['result'] = result + if monitored: + if sha(command[0]) != expected_sha or result['cpu_clock'] != 'GetProcessTimes kernel plus user': + raise ValueError('runtime binary or clock identity changed') + env = record['environment'] + # The outer host interval includes preparation/oracle and + # runner work excluded by the inner measured CPU interval. + # Retain this conservative estimate; never subtract it from + # costs or claim an exact per-process background partition. + remaining = env['busy_seconds'] - result['interval_cpu_seconds'] + env['unattributed_busy_percent_upper_estimate'] = remaining * 100 / env['total_seconds'] + if remaining < 0 or env['unattributed_busy_percent_upper_estimate'] > POLICY['max_unattributed_busy_percent']: + raise ValueError('host activity guard failed; sample retained, batch stopped') + record['status'] = 'valid' + except Exception as error: + record.update(status='failed', error=f'{type(error).__name__}: {error}') + write_json(directory / 'record.json', record) + samples.write(json.dumps(record, ensure_ascii=False, allow_nan=False) + '\n') + samples.flush() + os.fsync(samples.fileno()) + records.append(record) + if record['status'] != 'valid': + raise ValueError(f"sample {item['sequence']} failed: {record['error']}") + manifest['valid_samples'] += 1 + write_json(output / 'summary.json', summarize(records, seed)) + manifest.update(status='samples_complete', active_sequence=None) + except Exception as error: + manifest.update(status='failed', error=f'{type(error).__name__}: {error}') + except BaseException: + manifest['status'] = 'interrupted' + raise + finally: + manifest['finished_utc'] = utc() + write_json(output / 'manifest.json', manifest) + return manifest + + +def distribution(values): + median = statistics.median(values) + return dict(count=len(values), median=median, min=min(values), max=max(values), + mad=statistics.median(abs(value - median) for value in values)) + + +def summarize(records, seed): + """Require the whole ordered batch before computing within-repeat ratios.""" + if seed not in SEEDS or len(records) != len(sample_order(seed)): + raise ValueError('incomplete or unknown batch') + for record, expected in zip(records, sample_order(seed)): + if record['item'] != expected or record['status'] != 'valid': + raise ValueError('failed or reordered sample') + validate_sample(record['result'], expected) + groups = [] + for route, chunks in CHUNKS.items(): + for chunk in chunks: + pairs = {} + samples = {variant: dict(cpu_ns_per_byte=[], wall_ns_per_byte=[]) for variant in ('baseline', 'candidate')} + for record in records: + item, result = record['item'], record['result'] + if (item['route'], item['chunk']) != (route, chunk): + continue + volume = result['rounds'] * result['bytes_per_round'] + cpu = result['interval_cpu_seconds'] * 1e9 / volume + wall = result['interval_wall_seconds'] * 1e9 / volume + samples[item['variant']]['cpu_ns_per_byte'].append(cpu) + samples[item['variant']]['wall_ns_per_byte'].append(wall) + pairs.setdefault(item['repeat'], {})[item['variant']] = cpu + ratios = [pair['candidate'] / pair['baseline'] for _, pair in sorted(pairs.items())] + groups.append(dict(route=route, chunk=chunk, paired_cpu_ratios=ratios, + paired_cpu_ratio_median=statistics.median(ratios), + samples={variant: {metric: dict(distribution(values), values=values) + for metric, values in metrics.items()} + for variant, metrics in samples.items()})) + return dict(seed=seed, groups=groups, performance_gate='not_evaluated', + budget_status='not_set', receive_seconds=None, + input_latency_seconds=None, presented_done=None) + + +def run(plan_path, bundle, declaration_path, output, seed): + """Validate provenance and observe the host before any cost workload runs.""" + from experiments.measure.r54_environment import POLICY, preflight + from experiments.measure.r54_inputs import IDENTITY, ROOT, git, load_builds, read, require + if os.name != 'nt': + raise ValueError('v1 build binding and environment observation require Windows') + output = Path(output) + output.mkdir(parents=True, exist_ok=False) + manifest = dict(schema='termforge-r54-execution-v1', status='preparing', seed=seed, + started_utc=utc(), attempted=0, valid_samples=0, performance_gate='not_evaluated') + write_json(output / 'manifest.json', manifest) + try: + runner = dict(commit=git(ROOT, 'rev-parse', 'HEAD'), worktree_status=git(ROOT, 'status', '--porcelain'), + sources={name: sha(ROOT / name) for name in ( + 'experiments/measure/r54_batch.py', 'experiments/measure/r54_inputs.py', + 'experiments/measure/r54_environment.py', 'experiments/measure/r54_contract.py', + 'experiments/measure/run.py', 'experiments/measure/compare.py')}) + write_json(output / 'runner.json', runner) + manifest['runner_identity_sha256'] = sha(output / 'runner.json') + write_json(output / 'manifest.json', manifest) + require(seed in SEEDS, 'unknown batch seed') + require(not runner['worktree_status'], 'runner source must be committed and clean') + binding = load_builds(plan_path, bundle) + inputs = output / 'inputs' + inputs.mkdir() + (inputs / 'plan.json').write_bytes(Path(plan_path).read_bytes()) + for name, digest in binding['evidence'].items(): + payload = (Path(bundle) / name).read_bytes() + require(hashlib.sha256(payload).hexdigest() == digest, 'build evidence changed while copying') + (inputs / name).write_bytes(payload) + write_json(inputs / 'binding.json', binding) + for variant, command in binding['commands'].items(): + directory = output / (variant + '-identity') + directory.mkdir() + result = capture(command + ['--identity'], directory, timeout=10) + write_json(directory / 'record.json', result) + require(result['exit_code'] == 0 and read(directory / 'stdout.bin') == IDENTITY, + 'actual compiled identity mismatch') + environment = preflight(output, read(declaration_path)) + require(environment['status'] == 'low_activity_observed', 'host preflight rejected; no cost samples launched') + context = dict(binding, evidence_kind='fixed_core_cost', environment_policy=POLICY, + environment_sha256=sha(output / 'environment.json'), runner_commit=runner['commit'], + runner_sources=runner['sources'], runner_identity_sha256=manifest['runner_identity_sha256']) + result = run_schedule(binding['commands'], seed, output / 'samples', context) + manifest.update(status=result['status'], attempted=result['attempted'], valid_samples=result['valid_samples']) + require(result['status'] == 'samples_complete', result.get('error', 'sample execution failed')) + require(load_builds(plan_path, bundle) == binding, 'build inputs changed during batch') + for name, digest in context['runner_sources'].items(): + require(sha(ROOT / name) == digest, 'runner changed during batch') + power = output / 'power-policy-after' + power.mkdir() + observed = capture(['powercfg', '/getactivescheme'], power, timeout=10) + write_json(power / 'record.json', observed) + require(observed['exit_code'] == 0 and (power / 'stdout.bin').read_bytes() == + (output / 'power-policy/stdout.bin').read_bytes(), 'power policy changed during batch') + manifest['status'] = 'complete_diagnostic' + except Exception as error: + manifest.update(status='failed', error=f'{type(error).__name__}: {error}') + except BaseException: + manifest['status'] = 'interrupted' + raise + finally: + samples_manifest = output / 'samples/manifest.json' + if samples_manifest.exists(): + recorded = read(samples_manifest) + manifest.update(attempted=recorded['attempted'], valid_samples=recorded['valid_samples']) + manifest['finished_utc'] = utc() + write_json(output / 'manifest.json', manifest) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plan', type=Path, required=True) + parser.add_argument('--build-bundle', type=Path, required=True) + parser.add_argument('--environment', type=Path, required=True) + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--seed', type=int, choices=SEEDS, required=True) + args = parser.parse_args() + result = run(args.plan, args.build_bundle, args.environment, args.out, args.seed) + print(json.dumps(result, ensure_ascii=False)) + raise SystemExit(0 if result['status'] == 'complete_diagnostic' else 1) + + +if __name__ == '__main__': + main() diff --git a/experiments/measure/r54_environment.py b/experiments/measure/r54_environment.py new file mode 100644 index 00000000..559f7d0e --- /dev/null +++ b/experiments/measure/r54_environment.py @@ -0,0 +1,102 @@ +"""Read-only Windows activity evidence, with a predeclared diagnostic host guard.""" +import ctypes +from ctypes import wintypes +import math +import os +import platform +import time + +import psutil + +from experiments.measure.r54_batch import capture, utc, write_json + +POLICY = dict(schema='termforge-r54-environment-guard-v1', idle_intervals=10, + idle_interval_seconds=1, max_busy_percent=5.0, + max_unattributed_busy_percent=5.0, product_performance_budget=False, + measurement='GetSystemTimes; user + kernel - idle; all logical CPUs, at most 64', + scope='guest-visible CPU activity only; physical host contention and frequency stability remain unknown') + + +def cpu_snapshot(): + if os.name != 'nt' or not 0 < (psutil.cpu_count() or 0) <= 64: + raise ValueError('this host guard supports Windows with 1..64 logical CPUs only') + idle, kernel, user = wintypes.FILETIME(), wintypes.FILETIME(), wintypes.FILETIME() + function = ctypes.WinDLL('kernel32', use_last_error=True).GetSystemTimes + function.argtypes = [ctypes.POINTER(wintypes.FILETIME)] * 3 + function.restype = wintypes.BOOL + if not function(ctypes.byref(idle), ctypes.byref(kernel), ctypes.byref(user)): + raise ctypes.WinError(ctypes.get_last_error()) + seconds = lambda value: ((value.dwHighDateTime << 32) | value.dwLowDateTime) / 10_000_000 + return dict(user=seconds(user), kernel_including_idle=seconds(kernel), idle=seconds(idle), + system=seconds(kernel) - seconds(idle), perf_counter_seconds=time.perf_counter(), utc=utc()) + + +def cpu_delta(before, after): + deltas = {name: after[name] - before[name] for name in ('user', 'system', 'idle')} + if any(not math.isfinite(value) or value < 0 for value in deltas.values()) or sum(deltas.values()) <= 0: + raise ValueError('invalid host CPU interval') + total, busy = sum(deltas.values()), deltas['user'] + deltas['system'] + return dict(total_seconds=total, busy_seconds=busy, busy_percent=busy * 100 / total) + + +def qualify_idle(intervals): + return (len(intervals) == POLICY['idle_intervals'] and + all(math.isfinite(item['busy_percent']) and 0 <= item['busy_percent'] <= POLICY['max_busy_percent'] + for item in intervals)) + + +def process_snapshot(): + """Private evidence contains names/CPU identities, never command lines or env.""" + records = [] + for process in psutil.process_iter(['pid', 'name', 'create_time', 'cpu_times'], ad_value=None): + row = dict(process.info) + if row['cpu_times'] is not None: + row['cpu_times'] = row['cpu_times']._asdict() + records.append(row) + return records + + +def host_identity(): + import winreg + def registry(key, names): + with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) as handle: + return {name: winreg.QueryValueEx(handle, name)[0] for name in names} + return dict(platform=platform.platform(), python=platform.python_version(), psutil=psutil.__version__, + cpu_logical=psutil.cpu_count(), cpu_physical=psutil.cpu_count(logical=False), + cpu=registry(r'HARDWARE\DESCRIPTION\System\CentralProcessor\0', ['ProcessorNameString', 'Identifier']), + windows=registry(r'SOFTWARE\Microsoft\Windows NT\CurrentVersion', ['CurrentBuildNumber', 'UBR', 'DisplayVersion'])) + + +def preflight(output, declaration): + """One bounded idle observation; no waiting until green or modifying the host.""" + write_json(output / 'environment-policy.json', POLICY) + write_json(output / 'environment-declaration.json', declaration) + if (declaration.get('no_known_concurrent_build_or_test') is not True or + not isinstance(declaration.get('known_concurrent_work'), list) or + declaration.get('host_kind') not in ('shared_workstation', 'hosted_vm', 'dedicated_host')): + raise ValueError('explicit host/concurrent-work declaration required') + value = dict(status='observing', host=host_identity(), started_utc=utc(), intervals=[], + processes_before=process_snapshot(), declaration=declaration, policy=POLICY) + write_json(output / 'environment.json', value) + try: + power = output / 'power-policy' + power.mkdir() + value['power_policy'] = capture(['powercfg', '/getactivescheme'], power, timeout=10) + if value['power_policy']['exit_code'] != 0: + raise ValueError('power policy observation failed') + before = cpu_snapshot() + for _ in range(POLICY['idle_intervals']): + time.sleep(POLICY['idle_interval_seconds']) + after = cpu_snapshot() + value['intervals'].append(dict(cpu_delta(before, after), before=before, after=after)) + write_json(output / 'environment.json', value) + before = after + value['processes_after'] = process_snapshot() + value['status'] = 'low_activity_observed' if qualify_idle(value['intervals']) else 'rejected_busy_host' + except BaseException as error: + value.update(status='failed', error=f'{type(error).__name__}: {error}') + raise + finally: + value['finished_utc'] = utc() + write_json(output / 'environment.json', value) + return value diff --git a/experiments/measure/r54_inputs.py b/experiments/measure/r54_inputs.py new file mode 100644 index 00000000..2ce6d487 --- /dev/null +++ b/experiments/measure/r54_inputs.py @@ -0,0 +1,131 @@ +"""Verify the frozen plan against a retained Windows common-harness build bundle.""" +import json +from pathlib import Path +import subprocess + +from experiments.measure.r54_batch import sha +from experiments.measure.r54_contract import ROOT, SOURCES, canonical, protocol +import hashlib + +OVERLAYS = ('experiments/candidates/alacritty/Cargo.toml', + 'experiments/candidates/alacritty/src/measure.rs') +IDENTITY = dict(schema='termforge-core-cost-build-v1', allocator_wrapper_compiled=False, + timing_gates_compiled=True, projection_selectable=True) + + +def read(path): + return json.loads(Path(path).read_text(encoding='utf-8')) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def git(directory, *args): + return subprocess.check_output(['git', *args], cwd=directory, timeout=30).decode('utf-8').strip() + + +def load_plan(path): + value = read(path) + require(canonical(value['protocol']) == canonical(protocol()), 'frozen protocol changed') + require(value['protocol_sha256'] == hashlib.sha256(canonical(protocol())).hexdigest(), 'protocol hash mismatch') + require(value['phase'] == 'prepared_not_built_or_measured', 'plan phase changed') + require(value['performance_gate'] == 'not_evaluated', 'plan cannot pass performance') + require(value['builds'] == {name: None for name in SOURCES} and value['environment_evidence'] is None, + 'frozen preparation plan was rewritten with results') + for name, expected in value['harness_sha256'].items(): + require(sha(ROOT / name) == expected, f'harness changed: {name}') + require(set(value['harness_sha256']) == {*OVERLAYS, 'experiments/measure/run.py', + 'experiments/measure/r54_contract.py'}, 'incomplete harness identity') + return value + + +def load_builds(plan_path, bundle): + """Audit artifacts and current sources; logs remain evidence, not a signature. + + This consumes the retained v1 preparation layout. No source, build receipt or + binary is reconstructed, edited or retried by the measurement entry point. + """ + bundle = Path(bundle).resolve(strict=True) + plan = load_plan(plan_path) + build_path = bundle / 'build-plan.json' + build, audit = read(build_path), read(bundle / 'build-audit.json') + require(build['schema'] == 'termforge-r54-local-build-plan-v1', 'unknown build bundle') + require(build['frozen_plan']['sha256'] == sha(plan_path), 'build belongs to another plan') + require(build['protocol_sha256'] == plan['protocol_sha256'], 'build protocol mismatch') + require(set(build['builds']) == set(SOURCES) and audit['passed'] is True, 'incomplete build audit') + overrides = ('RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTFLAGS', + 'CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS', 'CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER', + 'RUSTC', 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER') + require(all(build['environment'][name] in (None, '') for name in overrides), 'unreviewed compiler override') + require(build['additional_build_environment'] == {'RUSTC_LOG': 'rustc_codegen_ssa::back::link=info'}, + 'unreviewed build environment') + for name in plan['harness_sha256']: + original = subprocess.check_output(['git', 'show', f"{build['harness_source']}:{name}"], cwd=ROOT, timeout=30) + require(original.replace(b'\r\n', b'\n') == (ROOT / name).read_bytes().replace(b'\r\n', b'\n'), + f'harness source mismatch: {name}') + commands, identities, evidence = {}, {}, {} + evidence['build-plan.json'] = sha(build_path) + evidence['build-audit.json'] = sha(bundle / 'build-audit.json') + targets = [] + for variant, source in SOURCES.items(): + spec = build['builds'][variant] + checkout = bundle / ('source-' + variant) + require(spec['commit'] == source['commit'] == git(checkout, 'rev-parse', 'HEAD'), 'source commit mismatch') + require(spec['tree'] == git(checkout, 'rev-parse', 'HEAD^{tree}'), 'source tree mismatch') + require(spec['candidate_tree'] == git(checkout, 'rev-parse', 'HEAD:experiments/candidates/alacritty'), 'candidate tree mismatch') + require(spec['version'] == source['version'], 'core version mismatch') + require(set(spec['overlay_files']) == set(OVERLAYS), 'unexpected harness overlay') + changed = git(checkout, 'diff', '--name-only', 'HEAD').splitlines() + require(set(changed) == set(spec['changed_paths']) == set(OVERLAYS), 'source changes outside harness') + files = set(git(checkout, 'ls-files', 'experiments/candidates/alacritty').splitlines()) + require(files == set(spec['before_files']) == set(spec['after_files']), 'incomplete source manifest') + for name in files: + require(sha(checkout / name) == spec['after_files'][name], f'changed build input: {name}') + if name in OVERLAYS: + require(spec['after_files'][name] == plan['harness_sha256'][name], 'different common harness') + else: + require(spec['before_files'][name] == spec['after_files'][name], 'changed original source') + target = Path(spec['target_dir']).resolve(strict=True) + targets.append(target) + expected_command = ['cargo', '+1.92.0', 'build', '--manifest-path', + str(checkout / 'experiments/candidates/alacritty/Cargo.toml'), + '--release', '--locked', '--offline', '--no-default-features', '--jobs', '1', + '--bin', 'alacritty-measure', '--target', 'x86_64-pc-windows-msvc', + '--target-dir', str(target), '-vv'] + require(spec['command'] == expected_command, 'build command is outside the fixed Windows configuration') + receipt_path = bundle / (variant + '-build.json') + receipt, observed = read(receipt_path), audit['builds'][variant] + require(observed['source'] == source['commit'] and observed['version'] == source['version'] and + observed['candidate_tree'] == spec['candidate_tree'], 'build audit source mismatch') + require(receipt['status'] == 'built_identity_only' and receipt['exit_code'] == 0, 'build failed') + require(receipt['variant'] == variant and receipt['command'] == spec['command'], 'build receipt command mismatch') + require(receipt['build_plan_sha256'] == sha(build_path), 'receipt belongs to another build plan') + require(observed['receipt_sha256'] == sha(receipt_path), 'build receipt changed') + require(receipt['identity'] == observed['identity'] == IDENTITY, 'unexpected compiled allocator mode') + binary = target / 'x86_64-pc-windows-msvc/release/alacritty-measure.exe' + binary_identity = dict(bytes=binary.stat().st_size, sha256=sha(binary)) + require(binary_identity == receipt['binary'] == observed['binary'], 'binary changed since build') + require(sha(Path(observed['linker']['path'])) == observed['linker']['sha256'], 'linker changed since audit') + require(set(receipt['logs']) == {variant + '-build.stdout', variant + '-build.stderr'}, 'missing build logs') + for name, identity in receipt['logs'].items(): + require(Path(name).name == name, 'log must stay in the bundle') + path = bundle / name + require(path.stat().st_size == identity['bytes'] and sha(path) == identity['sha256'], 'build log changed') + evidence[name] = sha(path) + patch = variant + '-harness.patch' + require(sha(bundle / patch) == spec['overlay_patch_sha256'], 'harness patch changed') + require(read(bundle / (variant + '-identity.stdout')) == IDENTITY, 'recorded binary identity mismatch') + evidence.update({receipt_path.name: sha(receipt_path), patch: sha(bundle / patch)}) + for suffix in ('-identity.stdout', '-identity.stderr'): + evidence[variant + suffix] = sha(bundle / (variant + suffix)) + commands[variant] = [str(binary)] + identities[variant] = dict(source=source, binary=binary_identity, + candidate_tree=spec['candidate_tree'], tracked_files=len(files)) + require(targets[0] != targets[1], 'shared target directory') + return dict(commands=commands, builds=identities, evidence=evidence, + bundle=str(bundle), plan_sha256=sha(plan_path), + protocol_sha256=plan['protocol_sha256'], harness_source=build['harness_source'], + rustc_vv=build['rustc_vv'], cargo_vv=build['cargo_vv'], + build_environment=build['environment'], additional_build_environment=build['additional_build_environment']) diff --git a/experiments/measure/results/2026-10-27-r54-runner/validation.json b/experiments/measure/results/2026-10-27-r54-runner/validation.json new file mode 100644 index 00000000..b43048d3 --- /dev/null +++ b/experiments/measure/results/2026-10-27-r54-runner/validation.json @@ -0,0 +1,163 @@ +{ + "schema": "termforge-r54-runner-validation-v1", + "scope": "frozen batch executor, build binding, raw artifacts, paired statistics and environment guard", + "performance_gate": "not_evaluated", + "budget_status": "not_set", + "formal_cost_samples": 0, + "protocol_sha256": "fb86f093a1c526979816574ef65896501eaeb9e9c34e7d9a6f2fdd5b61a8e7f6", + "build_harness_source": "8a13782aaac03780617e177b56ec55d3cb9c79be", + "builds": { + "baseline": { + "source": "65988f5f20f62294ff52fcac5b5b81e563f93f3b", + "version": "0.26.1-dev+termforge.3", + "tracked_candidate_files": 80, + "binary": { + "bytes": 523264, + "sha256": "33cb90beebffc57f07aa993b8ea635eaf461b49d86c143939a41f6242c087b6d" + } + }, + "candidate": { + "source": "3d6d44d3b3e9a9a3e41da98e8711e9f19a211fd8", + "version": "0.26.1-dev+termforge.4", + "tracked_candidate_files": 84, + "binary": { + "bytes": 579584, + "sha256": "d4d068be44a0a9ec733ad17f0323765ee38ffbf775e6efacb4632c1dce6b3f01" + } + } + }, + "actual_preflight": { + "runner_source": "fb5557c5519d9e0684ddca8332c6c57d6914a91f", + "source_attestation": "orchestrator recorded clean commit before invocation; this initial runner omitted runner.json on rejected preflight", + "status": "failed", + "environment_status": "rejected_busy_host", + "started_utc": "2026-10-27T09:34:09.959438+00:00", + "finished_utc": "2026-10-27T09:34:21.403306+00:00", + "attempted_cost_samples": 0, + "valid_cost_samples": 0, + "policy": { + "schema": "termforge-r54-environment-guard-v1", + "idle_intervals": 10, + "idle_interval_seconds": 1, + "max_busy_percent": 5.0, + "max_unattributed_busy_percent": 5.0, + "product_performance_budget": false, + "measurement": "GetSystemTimes; user + kernel - idle; all logical CPUs, at most 64", + "scope": "guest-visible CPU activity only; physical host contention and frequency stability remain unknown" + }, + "windows": { + "CurrentBuildNumber": "26200", + "UBR": 9711, + "DisplayVersion": "25H2" + }, + "cpu": { + "ProcessorNameString": "Intel(R) Core(TM) i7-9700 CPU @ 3.00GHz", + "Identifier": "Intel64 Family 6 Model 158 Stepping 13" + }, + "cpu_logical": 8, + "cpu_physical": 8, + "busy_percent_each_interval": [ + 11.9140625, + 18.26923076923077, + 11.153846153846153, + 21.27659574468085, + 25.436893203883496, + 32.88461538461539, + 31.73076923076923, + 17.348927875243664, + 16.184971098265898, + 17.692307692307693 + ], + "known_context": "shared workstation; personal desktop apps and read-only review remained; no known concurrent builds or tests", + "raw_evidence": { + "manifest.json": { + "bytes": 360, + "sha256": "fddc5f6daa6a387d329b8634e9aa7115c923cbbc83b7c212b2eaafa9a0704c6f" + }, + "environment.json": { + "bytes": 240917, + "sha256": "2c789e8ce31972d82ee6f2fe34466c8bc405625b00298fde51a479f4216b1c4b" + }, + "environment-policy.json": { + "bytes": 421, + "sha256": "2a002340e31ad68b80195d3c1d5b0d4902c89fb1a4795d58cf080d50f9091f4d" + }, + "inputs/binding.json": { + "bytes": 4266, + "sha256": "9bad6e1b46d49cb707a05366fb2c72cb96c06f5117e0c3f0dbadd9b72f348441" + }, + "baseline-identity/stdout.bin": { + "bytes": 135, + "sha256": "633be0c64cddc9990f89fc1c635d320a1d7561202cc16fd0b59187ed5713a611" + }, + "candidate-identity/stdout.bin": { + "bytes": 135, + "sha256": "633be0c64cddc9990f89fc1c635d320a1d7561202cc16fd0b59187ed5713a611" + } + }, + "process_identities_published": false, + "private_paths_published": false, + "repeated_after_rejection": false + }, + "related_regression": { + "source": "f72d13605b11493a8d9a8d8d438145c170b47d8e", + "cmake_configuration": "Windows x64, Release, default core, core_measure target only", + "native_build_source": "fb5557c5519d9e0684ddca8332c6c57d6914a91f", + "ctest_filter": "^measure$", + "ctest_exit_code": 0, + "python_tests": 25, + "python_elapsed_seconds": 12.868, + "native_measure_source_unchanged_since_build": true, + "logs": { + "0.stdout": { + "bytes": 786, + "sha256": "4142bee10ec21c168c53689bdc641f143d5977945aed8ada1f74e3cc8b224185" + }, + "0.stderr": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "1.stdout": { + "bytes": 294, + "sha256": "53171f7fc96805b2101a008dc99b9ce8d949b66fca649e6798f845ad0275ff36" + }, + "1.stderr": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ctest-final.stdout": { + "bytes": 6138, + "sha256": "1c3df9c5aa4f662ccf738f136ea9a2e52ad3f9a27459b0dfbc97dcc494310d19" + }, + "ctest-final.stderr": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "synthetic_process_fixture_calls_in_order_test": 112, + "synthetic_fixture_times_are_measurements": false, + "full_application_regression": "not run locally; delegated to normal CI after publication" + }, + "review_repairs": { + "source": "f72d13605b11493a8d9a8d8d438145c170b47d8e", + "changes": [ + "record runner identity before preparation can fail", + "strict UTF-8 decoding before JSON while retaining original bytes", + "unique atomic-write temporary names preserve interruption recording" + ], + "each_counterexample_observed_failing_then_passing": true, + "original_preflight_artifacts_rewritten": false + }, + "remaining": [ + "two original 112-sample cost batches on a controlled host", + "allocator and projection perturbation diagnosis", + "component cost attribution", + "user-confirmed numerical budget", + "any optimization and full correctness regression" + ], + "endpoints": { + "receive_seconds": null, + "input_latency_seconds": null, + "presented_done": null + } +} diff --git a/experiments/measure/test_r54_batch.py b/experiments/measure/test_r54_batch.py new file mode 100644 index 00000000..26e55411 --- /dev/null +++ b/experiments/measure/test_r54_batch.py @@ -0,0 +1,202 @@ +"""Synthetic process/artifact contracts; these tests do not measure core speed.""" +import copy +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest + +from experiments.measure.r54_contract import SOURCES, sample_order +from experiments.measure.test_r54_contract import sample + + +def records(seed): + baseline = (2, 4, 6, 8, 10, 12, 20) + candidate = (4, 20, 12, 40, 20, 48, 40) + result = [] + for item in sample_order(seed): + value = sample() + value.update(route=item['route'], chunk=item['chunk'], + bytes_per_round=78000 if item['route'] == 'project' else 22000, + state_fnv1a64=SOURCES[item['variant']]['states'][item['route']]) + cpu = (baseline if item['variant'] == 'baseline' else candidate)[item['repeat']] + value.update(interval_cpu_seconds=cpu, interval_wall_seconds=cpu + 1) + result.append(dict(item=item, result=value, status='valid')) + return result + + +class SummaryTests(unittest.TestCase): + def test_pairs_are_not_ratio_of_independent_medians(self): + from experiments.measure.r54_batch import summarize + value = summarize(records(20261026), 20261026) + self.assertEqual(len(value['groups']), 8) + self.assertEqual(value['performance_gate'], 'not_evaluated') + for group in value['groups']: + self.assertEqual(group['paired_cpu_ratios'], [2, 5, 2, 5, 2, 4, 2]) + self.assertEqual(group['paired_cpu_ratio_median'], 2) + cpu = group['samples'] + self.assertAlmostEqual(cpu['candidate']['cpu_ns_per_byte']['median'] / + cpu['baseline']['cpu_ns_per_byte']['median'], 2.5) + self.assertEqual(cpu['baseline']['cpu_ns_per_byte']['count'], 7) + + def test_missing_reordered_failed_or_wrong_identity_never_gets_summary(self): + from experiments.measure.r54_batch import summarize + valid = records(20261027) + broken = [] + broken.append(valid[:-1]) + broken.append([valid[1], valid[0], *valid[2:]]) + for field, value in (('status', 'failed'), ('item', valid[1]['item'])): + changed = copy.deepcopy(valid) + changed[0][field] = value + broken.append(changed) + wrong_state = copy.deepcopy(valid) + wrong_state[0]['result']['state_fnv1a64'] = 'bad' + broken.append(wrong_state) + for values in broken: + with self.assertRaises(ValueError): + summarize(values, 20261027) + + +class ProcessArtifactsTests(unittest.TestCase): + def test_interrupted_temporary_file_does_not_block_final_manifest(self): + from experiments.measure.r54_batch import write_json + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'manifest.json' + path.write_text('{"status":"running","active_sequence":3}', encoding='utf-8') + stale = Path(folder) / 'manifest.json.tmp' + stale.write_bytes(b'{"status":') + write_json(path, dict(status='interrupted', active_sequence=3)) + self.assertEqual(json.loads(path.read_text()), dict(status='interrupted', active_sequence=3)) + self.assertEqual(stale.read_bytes(), b'{"status":') + + def test_non_utf8_json_is_preserved_and_rejected_before_second_sample(self): + from experiments.measure.r54_batch import run_schedule + payload = json.dumps(records(20261026)[0]['result']).encode('utf-16') + with tempfile.TemporaryDirectory() as folder: + command = [sys.executable, '-c', 'import os,sys; os.write(1,bytes.fromhex(sys.argv[1]))', payload.hex()] + output = Path(folder) / 'batch' + manifest = run_schedule({variant: command for variant in SOURCES}, 20261026, output, + {'evidence_kind': 'synthetic_process_fixture'}) + self.assertEqual(manifest['status'], 'failed') + self.assertEqual(manifest['attempted'], 1) + self.assertEqual(manifest['valid_samples'], 0) + self.assertEqual((output / '000/stdout.bin').read_bytes(), payload) + + def test_timeout_keeps_exact_stream_bytes(self): + from experiments.measure.r54_batch import capture + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) + code = "import os,time; os.write(1,b'partial\\xff'); os.write(2,b'error\\xfe'); time.sleep(10)" + value = capture([sys.executable, '-c', code], output, timeout=1) + self.assertTrue(value['timed_out']) + self.assertEqual((output / 'stdout.bin').read_bytes(), b'partial\xff') + self.assertEqual((output / 'stderr.bin').read_bytes(), b'error\xfe') + + def test_failure_preserves_attempt_and_stops_without_overwriting(self): + from experiments.measure.r54_batch import run_schedule + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + fixture = root / 'fixture.py' + fixture.write_text("import os; os.write(1,b'invalid\\xff'); os.write(2,b'diagnostic\\xfe')", encoding='utf-8') + commands = {variant: [sys.executable, str(fixture)] for variant in SOURCES} + output = root / 'batch' + manifest = run_schedule(commands, 20261026, output, {'evidence_kind': 'synthetic_process_fixture'}) + self.assertEqual(manifest['status'], 'failed') + self.assertEqual(manifest['attempted'], 1) + self.assertEqual(manifest['valid_samples'], 0) + self.assertFalse((output / 'summary.json').exists()) + self.assertEqual((output / '000/stdout.bin').read_bytes(), b'invalid\xff') + self.assertEqual((output / '000/stderr.bin').read_bytes(), b'diagnostic\xfe') + original = (output / 'manifest.json').read_bytes() + with self.assertRaises(FileExistsError): + run_schedule(commands, 20261026, output, {}) + self.assertEqual((output / 'manifest.json').read_bytes(), original) + + def test_complete_real_subprocess_sequence_preserves_all_pairs(self): + from experiments.measure.r54_batch import run_schedule + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + # Independent literal fake results exercise process ordering and artifact + # handling. They are labelled synthetic and are never timing evidence. + (root / 'values.json').write_text(json.dumps(records(20261027)), encoding='utf-8') + fixture = root / 'fixture.py' + fixture.write_text('''import json,pathlib,sys +p=pathlib.Path(__file__).parent +count=p/'count' +i=int(count.read_text()) if count.exists() else 0 +record=json.loads((p/'values.json').read_text())[i] +item=record['item'] +assert sys.argv[1:]==[item['variant'],item['route'],str(item['chunk']),'0','2.0'] +count.write_text(str(i+1)) +print(json.dumps(record['result'])) +''', encoding='utf-8') + commands = {variant: [sys.executable, str(fixture), variant] for variant in SOURCES} + output = root / 'batch' + manifest = run_schedule(commands, 20261027, output, {'evidence_kind': 'synthetic_process_fixture'}) + self.assertEqual(manifest['status'], 'samples_complete') + self.assertEqual(manifest['valid_samples'], 112) + self.assertEqual(int((root / 'count').read_text()), 112) + rows = [json.loads(line) for line in (output / 'samples.jsonl').read_text().splitlines()] + self.assertEqual([row['item'] for row in rows], sample_order(20261027)) + self.assertEqual(len(list(output.glob('*/stdout.bin'))), 112) + self.assertEqual(json.loads((output / 'summary.json').read_text())['groups'][0]['paired_cpu_ratio_median'], 2) + + +class EnvironmentTests(unittest.TestCase): + def test_environment_qualification_does_not_hide_one_busy_interval(self): + from experiments.measure.r54_environment import qualify_idle, cpu_delta + # Windows user + system + idle count all logical CPUs. interrupt/dpc + # are already included in system and must not be added again. + before = dict(user=10, system=20, idle=70, interrupt=2, dpc=3) + after = dict(user=11, system=21, idle=118, interrupt=2.5, dpc=3.5) + observed = cpu_delta(before, after) + self.assertEqual(observed['busy_seconds'], 2) + self.assertEqual(observed['total_seconds'], 50) + self.assertEqual(observed['busy_percent'], 4) + self.assertTrue(qualify_idle([observed] * 10)) + self.assertFalse(qualify_idle([observed] * 9)) + self.assertFalse(qualify_idle([observed] * 9 + [dict(observed, busy_percent=6)])) + with self.assertRaises(ValueError): + cpu_delta(after, before) + + +class FrozenInputTests(unittest.TestCase): + @unittest.skipUnless(os.name == 'nt', 'the execution entry point binds Windows builds') + def test_failed_preparation_still_identifies_the_runner(self): + from experiments.measure.r54_batch import run + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) / 'failed' + manifest = run(Path(folder) / 'absent-plan', folder, Path(folder) / 'absent-environment', output, 20261026) + self.assertEqual(manifest['status'], 'failed') + self.assertEqual(manifest['attempted'], 0) + identity = json.loads((output / 'runner.json').read_text()) + self.assertEqual(len(identity['commit']), 40) + self.assertIn('experiments/measure/r54_batch.py', identity['sources']) + self.assertIn('worktree_status', identity) + + def test_changed_schedule_harness_or_result_injection_is_rejected(self): + from experiments.measure.r54_contract import prepare + from experiments.measure.r54_inputs import load_plan + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'plan.json' + original = prepare(path) + self.assertEqual(load_plan(path), original) + changes = [] + changed = copy.deepcopy(original) + changed['protocol']['schedules'][0]['samples'].reverse() + changes.append(changed) + changed = copy.deepcopy(original) + changed['harness_sha256'].pop('experiments/measure/run.py') + changes.append(changed) + changed = copy.deepcopy(original) + changed['environment_evidence'] = {'idle': True} + changes.append(changed) + for changed in changes: + path.write_text(json.dumps(changed), encoding='utf-8') + with self.assertRaises(ValueError): + load_plan(path) + + +if __name__ == '__main__': + unittest.main() From a7b9685a20396779dc9dff51018470b7638bfdce Mon Sep 17 00:00:00 2001 From: JavaZero <71128095+JavaZeroo@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:54:28 +0800 Subject: [PATCH 2/2] fix: isolate host sampler dependency from pure cost statistics (#54) Keep the initial candidate CI dependency failures and verify the import boundary without Python site packages. --- docs/EVIDENCE.md | 2 +- experiments/measure/R54-COST.md | 2 ++ experiments/measure/r54_environment.py | 5 +-- .../2026-10-27-r54-runner/validation.json | 34 +++++++++++++++++++ experiments/measure/test_r54_batch.py | 9 +++++ 5 files changed, 49 insertions(+), 3 deletions(-) diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index c04c3ced..3370ac10 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -162,6 +162,6 @@ python -m unittest experiments.shell.test_workflow_sessions -v | 当前成本调查 | 结果与合同 | 结论边界 | |---|---|---| | 候选 `.3`→`.4` 字素改动 | [固定字素验证](../experiments/shell/results/2026-10-26-streaming-graphemes/validation.json)、[性能限制与原始样本身份](research/streaming-graphemes.md)、[#54](https://github.com/JavaZeroo/termforge/issues/54) | 同宿主交错诊断中 project/interactive 吞吐中位数约下降 16%/9%；共享宿主、allocator 包装、无真实呈现/输入时延，使其不能作为正式预算或性能通过。#54 受控定位仍阻塞 #34 的性能裁决。 | -| 固定字素成本的批次协议与执行 | [R54 v1 合同和构建/环境记录](../experiments/measure/R54-COST.md)、[执行器](../experiments/measure/r54_batch.py)、[合成进程回归](../experiments/measure/test_r54_batch.py)、[实际预检及限定验证](../experiments/measure/results/2026-10-27-r54-runner/validation.json) | 固定两个种子、各 112 个位置，字节原样保留、首错停止、完整批次后逐组配对统计。共享工作站十区间活动预检未通过，未启动成本样本；测量相关 25 项测试通过。5% 是环境诊断护栏，并非产品性能容差；实际批次、扰动归因及用户预算确认由 #54 接续。 | +| 固定字素成本的批次协议与执行 | [R54 v1 合同和构建/环境记录](../experiments/measure/R54-COST.md)、[执行器](../experiments/measure/r54_batch.py)、[合成进程回归](../experiments/measure/test_r54_batch.py)、[实际预检及限定验证](../experiments/measure/results/2026-10-27-r54-runner/validation.json) | 固定两个种子、各 112 个位置，字节原样保留、首错停止、完整批次后逐组配对统计。共享工作站十区间活动预检未通过，未启动成本样本；测量相关 26 项测试通过，首次 Linux 候选 CI 的采样依赖导入错误及修正分别留证。5% 是环境诊断护栏，并非产品性能容差；实际批次、扰动归因及用户预算确认由 #54 接续。 | 目前没有足够证据判断 TermForge 距离 WindTerm 快或慢多少。正式结论必须同时提供负载/配置/环境身份、正确性、完成端点、观测开销、重复样本与限制。未知字段保持未知；历史校准未完成不转成要求用户反复手动运行的前置。执行规则见 [ADR-0002](adr/0002-automation-first-investigation.md)，核心选型重审见 [ADR-0003](adr/0003-core-reference-and-reuse.md)。 diff --git a/experiments/measure/R54-COST.md b/experiments/measure/R54-COST.md index 15c7dd46..fe49fa22 100644 --- a/experiments/measure/R54-COST.md +++ b/experiments/measure/R54-COST.md @@ -74,6 +74,8 @@ python -m unittest experiments.measure.test_r54_contract.OptionalAllocatorBuildS 环境声明 JSON 必须显式填写 `host_kind`（`shared_workstation`、`hosted_vm` 或 `dedicated_host`）、`known_concurrent_work` 字符串列表和 `no_known_concurrent_build_or_test=true`。声明不能取代实际观测。源工作树必须已提交且干净，输出目录必须不存在。 +实际 Windows 采样依赖 [benchmarks/requirements.txt](../../benchmarks/requirements.txt) 固定的 psutil；纯统计和计划检查仅依赖 Python 标准库，不因导入环境统计函数而要求安装宿主采样依赖。 + ```powershell python -m experiments.measure.r54_batch --plan local/r54-batch/plan-v1.json --build-bundle local/r54-batch --environment local/r54-environment.json --seed 20261026 --out local/r54-run-20261026 # 第二批另行启动，使用原第二种子和新目录；第一批失败时先保留并调查原因。 diff --git a/experiments/measure/r54_environment.py b/experiments/measure/r54_environment.py index 559f7d0e..49802986 100644 --- a/experiments/measure/r54_environment.py +++ b/experiments/measure/r54_environment.py @@ -6,8 +6,6 @@ import platform import time -import psutil - from experiments.measure.r54_batch import capture, utc, write_json POLICY = dict(schema='termforge-r54-environment-guard-v1', idle_intervals=10, @@ -18,6 +16,7 @@ def cpu_snapshot(): + import psutil if os.name != 'nt' or not 0 < (psutil.cpu_count() or 0) <= 64: raise ValueError('this host guard supports Windows with 1..64 logical CPUs only') idle, kernel, user = wintypes.FILETIME(), wintypes.FILETIME(), wintypes.FILETIME() @@ -47,6 +46,7 @@ def qualify_idle(intervals): def process_snapshot(): """Private evidence contains names/CPU identities, never command lines or env.""" + import psutil records = [] for process in psutil.process_iter(['pid', 'name', 'create_time', 'cpu_times'], ad_value=None): row = dict(process.info) @@ -57,6 +57,7 @@ def process_snapshot(): def host_identity(): + import psutil import winreg def registry(key, names): with winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, key) as handle: diff --git a/experiments/measure/results/2026-10-27-r54-runner/validation.json b/experiments/measure/results/2026-10-27-r54-runner/validation.json index b43048d3..869bd4a8 100644 --- a/experiments/measure/results/2026-10-27-r54-runner/validation.json +++ b/experiments/measure/results/2026-10-27-r54-runner/validation.json @@ -159,5 +159,39 @@ "receive_seconds": null, "input_latency_seconds": null, "presented_done": null + }, + "ci_dependency_repair": { + "source": "a86e7fc186b87720a46c689d7de29032d50c809c", + "initial_published_source": "fa47edadbc259057a841db32181d1d73095f4a10", + "initial_linux_candidate_jobs": [ + 122118298338, + 122118279790 + ], + "failure": "ModuleNotFoundError: No module named psutil while importing pure environment statistics", + "reproduced_with_python_no_site_packages": true, + "repair": "Import psutil only inside real host-sampling functions; pure statistics retain standard-library-only imports", + "python_tests": 26, + "python_elapsed_seconds": 12.949, + "ctest_exit_code": 0, + "formal_cost_samples": 0, + "logs": { + "ci-122118298338.log": { + "bytes": 102800, + "sha256": "9babc825cff5781142f42e741137654cbcb247518bad8bc0a6a14f03b4c93e89" + }, + "ci-122118279790.log": { + "bytes": 102205, + "sha256": "43e81e8186a9a0930a8bab823dadded525441be37fdd0b12cca3241240ed7c42" + }, + "ctest-dependency-fix.stdout": { + "bytes": 6327, + "sha256": "3b939917d3011e5618aff624fbe0bca8298099471a6b98d39bc761988944da11" + }, + "ctest-dependency-fix.stderr": { + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + "initial_preflight_repeated": false } } diff --git a/experiments/measure/test_r54_batch.py b/experiments/measure/test_r54_batch.py index 26e55411..d5918f7a 100644 --- a/experiments/measure/test_r54_batch.py +++ b/experiments/measure/test_r54_batch.py @@ -3,6 +3,7 @@ import json import os from pathlib import Path +import subprocess import sys import tempfile import unittest @@ -144,6 +145,14 @@ def test_complete_real_subprocess_sequence_preserves_all_pairs(self): class EnvironmentTests(unittest.TestCase): + def test_pure_environment_statistics_import_without_site_packages(self): + command = [sys.executable, '-S', '-c', + 'from experiments.measure.r54_environment import cpu_delta, qualify_idle; print("imported")'] + result = subprocess.run(command, cwd=Path(__file__).resolve().parents[2], + capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr.decode('utf-8', errors='replace')) + self.assertEqual(result.stdout.strip(), b'imported') + def test_environment_qualification_does_not_hide_one_busy_interval(self): from experiments.measure.r54_environment import qualify_idle, cpu_delta # Windows user + system + idle count all logical CPUs. interrupt/dpc