diff --git a/.github/workflows/r54-cost.yml b/.github/workflows/r54-cost.yml new file mode 100644 index 00000000..e84887ef --- /dev/null +++ b/.github/workflows/r54-cost.yml @@ -0,0 +1,55 @@ +name: Frozen R54 hosted cost batches + +on: + push: + branches: ['perf/grapheme-hosted-r54'] + paths: + - '.github/workflows/r54-cost.yml' + - 'experiments/measure/*.py' + - 'experiments/candidates/alacritty/Cargo.toml' + - 'experiments/candidates/alacritty/src/measure.rs' + +permissions: + contents: read + +concurrency: + group: r54-cost-${{ github.ref }} + cancel-in-progress: false + +jobs: + fixed-cost: + # A code/evidence correction must not silently launch replacement samples. + if: github.event.created == true + runs-on: windows-2025 + timeout-minutes: 30 + env: + PYTHONUTF8: '1' + CARGO_HOME: ${{ github.workspace }}/local/r54-cargo-home + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: '3.12' + - name: Install fixed host observer + run: python -m pip install -r benchmarks/requirements.txt + - name: Select fixed compiler + run: rustup default 1.92.0 + - name: Verify runner and publication contracts before measurement + run: python -m unittest experiments.measure.test_r54_contract.CostPlanTests experiments.measure.test_r54_batch experiments.measure.test_r54_hosted -v + - name: Build audited pair and execute the two frozen batches + id: measure + run: python -m experiments.measure.r54_hosted --out local/r54-raw + - name: Prepare the reviewed synthetic artifact whitelist + if: always() && steps.measure.outcome != 'skipped' + run: python -m experiments.measure.r54_publish --raw local/r54-raw --out local/r54-public + - name: Preserve complete or failed synthetic evidence + if: always() && steps.measure.outcome != 'skipped' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: r54-cost-${{ github.run_id }}-${{ github.run_attempt }} + retention-days: 30 + if-no-files-found: error + path: local/r54-public/ diff --git a/CMakeLists.txt b/CMakeLists.txt index c6a2ffbf..5825ca6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,7 +292,7 @@ if(BUILD_TESTING) endif() termforge_add_python_test(core -m unittest experiments.core.test_core experiments.core.test_unicode -v) termforge_add_python_test(recovery -m unittest experiments.core.test_recovery -v) - termforge_add_python_test(measure -m unittest experiments.measure.test_measure experiments.measure.test_r54_contract.CostPlanTests experiments.measure.test_r54_batch -v) + termforge_add_python_test(measure -m unittest experiments.measure.test_measure experiments.measure.test_r54_contract.CostPlanTests experiments.measure.test_r54_batch experiments.measure.test_r54_hosted -v) termforge_add_python_test(binary_paths -m unittest experiments.test_build_paths -v) termforge_add_python_test(conpty_margin_contract -m unittest experiments.conpty.test_margin_contract -v) termforge_add_python_test(benchmark_tools -m unittest discover -s benchmarks/tests -v) diff --git a/docs/EVIDENCE.md b/docs/EVIDENCE.md index 3370ac10..e7402838 100644 --- a/docs/EVIDENCE.md +++ b/docs/EVIDENCE.md @@ -163,5 +163,6 @@ python -m unittest experiments.shell.test_workflow_sessions -v |---|---|---| | 候选 `.3`→`.4` 字素改动 | [固定字素验证](../experiments/shell/results/2026-10-26-streaming-graphemes/validation.json)、[性能限制与原始样本身份](research/streaming-graphemes.md)、[#54](https://github.com/JavaZeroo/termforge/issues/54) | 同宿主交错诊断中 project/interactive 吞吐中位数约下降 16%/9%；共享宿主、allocator 包装、无真实呈现/输入时延，使其不能作为正式预算或性能通过。#54 受控定位仍阻塞 #34 的性能裁决。 | | 固定字素成本的批次协议与执行 | [R54 v1 合同和构建/环境记录](../experiments/measure/R54-COST.md)、[执行器](../experiments/measure/r54_batch.py)、[合成进程回归](../experiments/measure/test_r54_batch.py)、[实际预检及限定验证](../experiments/measure/results/2026-10-27-r54-runner/validation.json) | 固定两个种子、各 112 个位置，字节原样保留、首错停止、完整批次后逐组配对统计。共享工作站十区间活动预检未通过，未启动成本样本；测量相关 26 项测试通过，首次 Linux 候选 CI 的采样依赖导入错误及修正分别留证。5% 是环境诊断护栏，并非产品性能容差；实际批次、扰动归因及用户预算确认由 #54 接续。 | +| 首次 hosted 字素成本批次 | [来源、停止位置与归档](../experiments/measure/results/2026-10-27-r54-hosted/README.md)、[机器可读记录](../experiments/measure/results/2026-10-27-r54-hosted/validation.json) | 两份固定构建与4次sanity通过，空闲预检通过；第24位置的未归属CPU活动估计5.410%触发原5%护栏，保留23个有效记录和1个失败记录，第二批未启动。全部24次输出正确，但批次不完整，不生成配对成本摘要；临时VM逐进程归因、完整受控批次和预算仍未完成。 | 目前没有足够证据判断 TermForge 距离 WindTerm 快或慢多少。正式结论必须同时提供负载/配置/环境身份、正确性、完成端点、观测开销、重复样本与限制。未知字段保持未知；历史校准未完成不转成要求用户反复手动运行的前置。执行规则见 [ADR-0002](adr/0002-automation-first-investigation.md)，核心选型重审见 [ADR-0003](adr/0003-core-reference-and-reuse.md)。 diff --git a/docs/FEATURES.md b/docs/FEATURES.md index 3eead283..6619b2a3 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -71,7 +71,7 @@ | 自动正确性与构建 | **限定验收**：Windows shell/ConPTY/呈现及 Windows/Linux 核心 CI、差分/恢复/队列/生命周期场景 | 产品打包、干净机器使用、依赖升级、长期回归和更广平台矩阵；#40 的 CI 纯夹具超时、窄窗口选择 #73、四 shell 原事件 #80、hosted 准备 #81 和[复制夹具 OpenClipboard 5 #85](https://github.com/JavaZeroo/termforge/issues/85) 等独立失败仍未闭合；#72 保存判据与 #80 同步条件的限定修正不代替其他配置或未知根因验收 | R2 / R4 | | Windows 产品交付 | **实现中，未验收**：首个 [v0.1.0-alpha.1 预发布](releases/v0.1.0-alpha.1.md)提供 Windows x64、静态 MSVC 运行库、附资源许可和构建身份的免安装 ZIP；未签名，已验环境限 Windows 11 本机与 Server 2025 CI，布局仍写入 LOCALAPPDATA | 这是 R1 体验包，尚非 R2 日常开发版；干净机器与更广平台矩阵、签名/安装、配置升级及持续运行仍待完成。Windows 10 未验收，Server 2022 已知资源门槛未过 | R2 / R3 | | Linux/macOS 与其他架构 | **未实现**客户端；Linux 核心 CI 已有。[官方平台范围台账](../research/windterm-2.7.0-platform-scope.md)已固定 2.7.0 的 Windows x86/x64、macOS x64、Linux x64 资产及声明下限；这不是 TermForge 支持或实测结论 | 最终需真实 Linux/macOS 客户端；平台调查可提前进行，整合及完整验收排在 Windows 11 x64 完整版本之后。其他系统版本、最低版本和架构由用户后续明确；旧系统及 ARM 等保持调查记录，不自动承诺或排除，也不能由核心 CI 推断产品支持 | 持续调查；R5 全平台出口 | -| 吞吐、响应、内存与稳定性 | **实现中，未验收**：已有机制实验、连续 CPU 区间校准与合成回归；候选字素改动的受限诊断中 project/interactive 吞吐中位数约下降 16%/9%，由 [#54](https://github.com/JavaZeroo/termforge/issues/54) 跟踪。[冻结批次执行器](../experiments/measure/R54-COST.md#冻结批次执行器windows)核对构建来源、原始输出、停止规则与逐轮配对统计；其环境护栏和合成进程测试不构成正式成本样本 | 共享宿主、allocator 包装与缺少真实呈现/输入时延端点使该诊断不构成正式预算。仍需同负载/同正确性、无插桩对照、观测开销和长期/多会话公平性；可靠原版基线后提出逐项预算及允许波动范围，**由用户明确确认**。与 WindTerm 的实际差距**待取证** | 当前优先；贯穿 R1–R5 | +| 吞吐、响应、内存与稳定性 | **实现中，未验收**：已有机制实验、连续 CPU 区间校准与合成回归；候选字素改动的受限诊断中 project/interactive 吞吐中位数约下降 16%/9%，由 [#54](https://github.com/JavaZeroo/termforge/issues/54) 跟踪。[冻结批次执行器](../experiments/measure/R54-COST.md#冻结批次执行器windows)核对构建来源、原始输出、停止规则与逐轮配对统计；[独立 hosted 构建与实测](../experiments/measure/results/2026-10-27-r54-hosted/README.md)已验证来源和失败保留，但首批因环境护栏停止，尚无完整成本批次 | 旧诊断的共享宿主、allocator 包装与缺少真实呈现/输入时延端点使其不构成正式预算；新的不完整批次也不作吞吐结论。仍需同负载/同正确性、无插桩对照、观测开销和长期/多会话公平性；可靠原版基线后提出逐项预算及允许波动范围，**由用户明确确认**。与 WindTerm 的实际差距**待取证** | 当前优先；贯穿 R1–R5 | | 大历史与动态内存压缩 | **未实现**产品能力；WindTerm 压缩属于官方性能声明，内部机制尚未证实 | 1/10/100 万行等明确规模下的内存/滚动/搜索/淘汰实验；根据瓶颈决定独立存储/压缩实现 | R4 | | 完整目标验收 | **待取证**：已有固定版本和局部观察，没有完整行为/性能对照矩阵 | 先在 Windows 11 x64 完整验收功能/行为/UI/性能及独立构建，再完成 Linux/macOS 全平台出口；重要差异和性能容差须有用户确认，Windows 通过不代表全平台完成；不依赖 WindTerm 二进制或专有资源 | R5 分平台出口 | diff --git a/experiments/measure/R54-COST.md b/experiments/measure/R54-COST.md index fe49fa22..d9abf9f2 100644 --- a/experiments/measure/R54-COST.md +++ b/experiments/measure/R54-COST.md @@ -93,3 +93,5 @@ Windows 1–64 逻辑 CPU 上使用 [GetSystemTimes](https://learn.microsoft.com 全部 112 个位置完整有效后才生成统计；缺失、重复、乱序或失败不能得到成功摘要。摘要逐批、逐 route/chunk 列出七个配对 CPU ns/B 比值，以及各源 CPU/墙钟 ns/B 全部值、中位数、范围和 MAD。合成进程测试覆盖整个顺序、原始字节和停止规则，其虚构时长只验证统计与进程协议，绝不作为性能样本。 实际受控重复样本、allocator/projection 扰动归因、预算提案和优化验收仍由 #54 接续；构建、短 sanity、执行器测试或环境预检均不关闭该 issue。 + +独立 Windows 虚拟机上的来源构建、顺序执行、白名单产物和宿主限制见 [R54 hosted 流程](R54-HOSTED.md)。该入口复用本合同，不把CI可运行或环境预检通过解释为产品性能验收。 diff --git a/experiments/measure/R54-HOSTED.md b/experiments/measure/R54-HOSTED.md new file mode 100644 index 00000000..ee25c865 --- /dev/null +++ b/experiments/measure/R54-HOSTED.md @@ -0,0 +1,40 @@ +# R54：独立 Windows 虚拟机上的固定批次 + +本流程接续 [#54](https://github.com/JavaZeroo/termforge/issues/54) 和 [v1 合同](R54-COST.md)，不改变源码、负载、两批各112个位置、计时端点、统计或预算。主工作站已有一次活动预检拒绝；不在同一宿主循环等绿，改在一个新的 GitHub-hosted `windows-2025` 虚拟机上建立完整来源和环境记录。它是本次 headless 核心成本研究的环境，不代替 Windows 11 x64 产品或 WindTerm 性能验收。 + +## 运行与停止 + +[工作流](../../.github/workflows/r54-cost.yml)只在实验分支 `perf/grapheme-hosted-r54` 的创建 push 事件执行成本作业，不在 PR 事件或后续源码/证据修正时自动补跑，也不改变 main 的默认构建。每次工作流用独立 run ID、attempt、源码和输出目录；拒绝对旧 workflow run 的 attempt 2 及后续重跑。首次运行后，新的调查需先明确假设、范围和新身份，再审查执行入口，不能由导出器修正偷偷触发替代样本。 + +同一虚拟机依次完成固定 Python/Rust、纯合同测试、两份来源构建、四次短正确性检查，再运行原两个种子。构建和测试结束后才开始成本预检与计时。没有自动恢复、补样、重试或放宽5%环境诊断护栏的入口；第一批失败即保留当前位置，第二批不启动。两批成功也只写 `complete_diagnostic`，`performance_gate=not_evaluated`。 + +同一任务的新 `CARGO_HOME` 用于获取锁定依赖；两个编译目标目录独立。脚本拒绝未知编译环境覆盖与 Cargo 配置文件，只覆盖原 `.3/.4` 的 Cargo.toml/src/measure.rs。每份仍使用 Rust1.92.0、MSVC x64、Release、locked/offline、no-default-features和jobs=1。完整源码前后哈希、真实命令/日志、实际linker及EXE、编译模式和四次原状态摘要由 [builder](r54_build.py) 留证；由原 [输入校验器](r54_inputs.py)重新读取验证之后才进入测量。 + +构建/准备调用各有期限，构建为600秒、短正确性调用为10秒；这些不是产品性能门槛。构建超时终止直接子进程并阻止全部测量，不据此声称已回收其派生进程；虚拟机生命周期由CI提供者管理。实际成本仍使用原内部60秒/子进程90秒期限，整个job30分钟。正常构建返回后才允许成本入口，不能将超时残留视作空闲。 + +## 宿主限制 + +保留 GitHub事件、run ID/attempt、实际提交、runner类型、系统映像、OS build/UBR、CPU与电源策略。固定两个种子的同口径结果分别报告，不混入主工作站原构建或历史12样本。`hosted_identity` 只检查该流程需要的元数据；分析时还须从GitHub作业记录核对实际runner与源码身份，环境变量本身不是物理隔离证明。 + +GitHub runner服务、操作系统后台任务、虚拟机邻居、物理CPU频率和调度仍可能影响结果。原5%筛选只描述guest可观察CPU活动；它不是用户确认的数值性能容差，也不证明物理宿主独占。出现噪声、失败或两批不一致时保留全部观察，再提出下一项可检验的调查。 + +## 发布边界 + +[发布器](r54_publish.py)只接受本流程的真实GitHub-hosted首次作业，并使用固定文件白名单。它不递归上传工作目录、checkout、编译缓存或个人文件。仅保存本次公开源码/合成输入产生的结果；完整进程名称/PID记录与未知环境字段不导出，工作区、runner profile和临时路径前缀改为别名。没有读取完整进程命令行或任意环境变量的步骤。 + +Windows 长路径及实际8.3别名均纳入替换，兼容大小写、正斜杠和编译器转义形式；通过[GetShortPathNameW](https://learn.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getshortpathnamew)查询，失败则不发布，不能猜测短名。首次作业导出曾漏掉两份构建日志中的短profile前缀；原包保留，审阅后的公开副本另列哈希及转换，不回填为旧运行使用了修正版导出器。 + +两个核心的成本及短正确性 stdout/stderr 按原字节保留，包括失败或无效编码；其生产者只接收公开固定合成负载。完整构建日志保留内容并替换宿主路径前缀。JSON/JSONL记录只做上述元数据筛选和路径处理，不改变时间、工作量、状态摘要、顺序或失败状态。中断留下的JSONL半行不修补、不删除，保留全部行和解析失败位置，独立样本原始流仍导出。`publication.json`列出每份原文件和发布文件的SHA、大小及处理方式。导出前核对原始 `hosted.json` 的run ID/attempt/SHA；空目录或另一次作业不生成成功包。发布目录在所有文件完成后才生效，上传仅指向该目录；未通过检查的半成品目录不会上传。 + +原始完整进程身份只在临时虚拟机内保留，公开结果保留全机CPU原计数、全部成本原始流及其来源。不能用未导出的逐进程字段做外部归因声明；后续若需要这种归因，另行预声明可公开的受控观测。所有CI结果须下载复核其清单、原始样本和统计后才能写入 #54 的成本结论。 + +## 手动入口（只供同一受控作业） + +```powershell +python -m experiments.measure.r54_hosted --out local/r54-raw +python -m experiments.measure.r54_publish --raw local/r54-raw --out local/r54-public +``` + +本机不得伪造GitHub环境变量运行该流程。普通本地准备可使用 `r54_build --out <新目录>`，要求实际Windows、已提交的干净源码及显式任务缓存；本轮不重复已有主工作站成本预检。测试只验证既有构建输入/产物边界，正式成本样本来自上述实际作业。 + +首次真实作业的来源、全部原始产物、环境护栏停止位置和独立复核见[固定记录](results/2026-10-27-r54-hosted/README.md)。这是不完整批次，没有配对成本摘要或性能通过结论。 diff --git a/experiments/measure/r54_build.py b/experiments/measure/r54_build.py new file mode 100644 index 00000000..21fc4c8f --- /dev/null +++ b/experiments/measure/r54_build.py @@ -0,0 +1,196 @@ +"""Create the audited R54 Windows build bundle before any cost measurements.""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess + +from experiments.measure.r54_batch import capture, sha, utc, write_json +from experiments.measure.r54_contract import ROOT, SOURCES, expected_sample, prepare +from experiments.measure.r54_inputs import IDENTITY, OVERLAYS, git, load_builds, require +from experiments.measure.run import validate_result + +OVERRIDES = ('RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'CARGO_BUILD_TARGET', 'CARGO_BUILD_RUSTFLAGS', + 'CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_RUSTFLAGS', 'CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER', + 'RUSTC', 'RUSTC_WRAPPER', 'RUSTC_WORKSPACE_WRAPPER') +RECORDED_ENVIRONMENT = OVERRIDES + ('CARGO_HOME', 'RUSTUP_HOME', 'RUSTUP_TOOLCHAIN', 'CARGO_TERM_COLOR', 'LIB', 'INCLUDE') + + +def run_logged(command, directory, prefix, *, cwd=ROOT, environment=None, timeout=300): + """Build/setup logs are byte-exact; failure prevents entering the cost runner.""" + result = dict(command=command, started_utc=utc(), exit_code=None, timed_out=False) + stdout, stderr = directory / (prefix + '.stdout'), directory / (prefix + '.stderr') + try: + with stdout.open('xb') as output, stderr.open('xb') as error: + try: + process = subprocess.run(command, cwd=cwd, env=environment, stdin=subprocess.DEVNULL, + stdout=output, stderr=error, timeout=timeout) + result['exit_code'] = process.returncode + except subprocess.TimeoutExpired: + result.update(timed_out=True, error='build/setup direct-child timeout; no process-tree cleanup claim') + require(result['exit_code'] == 0, f'{prefix} failed; preserve logs, no automatic retry') + except BaseException as error: + result['error'] = f'{type(error).__name__}: {error}' + raise + finally: + result['finished_utc'] = utc() + result['logs'] = {path.name: dict(bytes=path.stat().st_size, sha256=sha(path)) + for path in (stdout, stderr) if path.exists()} + write_json(directory / (prefix + '.process.json'), result) + return result + + +def reject_hidden_configuration(checkout, environment): + # Cargo searches the working directory's ancestors and CARGO_HOME. A clean, + # explicit cache prevents inherited config from changing the fixed flags. + cargo_home = environment.get('CARGO_HOME') + require(bool(cargo_home), 'CARGO_HOME must name an explicit task-owned cache') + locations = [checkout, *checkout.parents, Path(cargo_home)] + for location in locations: + config_dir = location if location == Path(cargo_home) else location / '.cargo' + require(not any((config_dir / name).exists() for name in ('config', 'config.toml')), + 'unreviewed Cargo configuration file') + + +def prepare_builds(output): + output = Path(output).resolve() + output.mkdir(parents=True, exist_ok=False) + status = dict(schema='termforge-r54-build-preparation-v1', status='preparing', started_utc=utc(), + source=git(ROOT, 'rev-parse', 'HEAD'), runner_sha256=sha(Path(__file__)), + formal_cost_samples=0, performance_gate='not_evaluated') + write_json(output / 'preparation.json', status) + try: + environment = {name: os.environ.get(name) for name in RECORDED_ENVIRONMENT} + for name in OVERRIDES: + require(environment[name] in (None, ''), f'unreviewed compiler override: {name}') + for name, value in os.environ.items(): + if value and name.startswith(('CARGO_', 'RUST')) and name not in RECORDED_ENVIRONMENT: + raise ValueError(f'unreviewed compiler environment name: {name}') + require(os.name == 'nt', 'the frozen build pair targets Windows x64') + require(not git(ROOT, 'status', '--porcelain'), 'prepare from a committed, clean source tree') + frozen = prepare(output / 'plan-v1.json') + builds = {} + for variant, source in SOURCES.items(): + checkout = output / ('source-' + variant) + run_logged(['git', 'worktree', 'add', '--detach', str(checkout), source['commit']], + output, variant + '-checkout', timeout=60) + require(git(checkout, 'rev-parse', 'HEAD') == source['commit'] and + not git(checkout, 'status', '--porcelain'), 'checkout identity or cleanliness mismatch') + reject_hidden_configuration(checkout, environment) + files = git(checkout, 'ls-files', 'experiments/candidates/alacritty').splitlines() + before = {name: sha(checkout / name) for name in files} + for name in OVERLAYS: + (checkout / name).write_bytes((ROOT / name).read_bytes()) + after = {name: sha(checkout / name) for name in files} + changed = git(checkout, 'diff', '--name-only').splitlines() + require(set(changed) == set(OVERLAYS), 'unexpected source changes during overlay') + require(all(before[name] == after[name] for name in files if name not in OVERLAYS), + 'original lockfile, adapter or vendor changed') + patch = output / (variant + '-harness.patch') + patch.write_bytes(subprocess.check_output(['git', 'diff', '--', *OVERLAYS], cwd=checkout, timeout=30)) + target = output / ('target-' + variant + '-unwrapped') + require(not target.exists(), 'target directory already exists') + command = ['cargo', '+1.92.0', 'build', '--manifest-path', + str(checkout / 'experiments/candidates/alacritty/Cargo.toml'), + '--release', '--locked', '--offline', '--no-default-features', '--jobs', '1', + '--bin', 'alacritty-measure', '--target', 'x86_64-pc-windows-msvc', + '--target-dir', str(target), '-vv'] + builds[variant] = dict(commit=source['commit'], tree=git(checkout, 'rev-parse', 'HEAD^{tree}'), + candidate_tree=git(checkout, 'rev-parse', 'HEAD:experiments/candidates/alacritty'), + version=source['version'], overlay_files=list(OVERLAYS), changed_paths=changed, + before_files=before, after_files=after, overlay_patch_sha256=sha(patch), + command=command, target_dir=str(target), status='not_built') + plan = dict(schema='termforge-r54-local-build-plan-v1', harness_source=status['source'], + protocol_sha256=frozen['protocol_sha256'], frozen_plan=dict(sha256=sha(output / 'plan-v1.json')), + rustc_vv=subprocess.check_output(['rustc', '+1.92.0', '-Vv'], timeout=30).decode('utf-8'), + cargo_vv=subprocess.check_output(['cargo', '+1.92.0', '-Vv'], timeout=30).decode('utf-8'), + environment=environment, additional_build_environment={'RUSTC_LOG': 'rustc_codegen_ssa::back::link=info'}, + builds=builds, formal_samples=0, performance_gate='not_evaluated') + write_json(output / 'build-plan.json', plan) + for variant, spec in builds.items(): + checkout = output / ('source-' + variant) + run_logged(['cargo', '+1.92.0', 'fetch', '--locked', '--manifest-path', + str(checkout / 'experiments/candidates/alacritty/Cargo.toml'), + '--target', 'x86_64-pc-windows-msvc'], output, variant + '-fetch', cwd=checkout) + receipt = dict(variant=variant, build_plan_sha256=sha(output / 'build-plan.json'), + command=spec['command'], status='running', formal_samples=0, performance_gate='not_evaluated') + write_json(output / (variant + '-build.json'), receipt) + try: + require(all(sha(checkout / name) == digest for name, digest in spec['after_files'].items()), + 'fetch changed fixed build inputs') + receipt.update(run_logged(spec['command'], output, variant + '-build', cwd=checkout, + environment=dict(os.environ, **plan['additional_build_environment']), timeout=600)) + require(all(sha(checkout / name) == digest for name, digest in spec['after_files'].items()), + 'build changed fixed inputs') + binary = Path(spec['target_dir']) / 'x86_64-pc-windows-msvc/release/alacritty-measure.exe' + receipt['binary'] = dict(bytes=binary.stat().st_size, sha256=sha(binary)) + run_logged([str(binary), '--identity'], output, variant + '-identity', cwd=checkout, timeout=10) + receipt['identity'] = json.loads((output / (variant + '-identity.stdout')).read_bytes().decode('utf-8')) + require(receipt['identity'] == IDENTITY, 'compiled allocator mode mismatch') + receipt['status'] = 'built_identity_only' + except BaseException as error: + receipt.update(status='failed', error=f'{type(error).__name__}: {error}') + raise + finally: + write_json(output / (variant + '-build.json'), receipt) + audit(output, plan) + load_builds(output / 'plan-v1.json', output) + status['status'] = 'built_and_audited' + except BaseException as error: + status.update(status='failed', error=f'{type(error).__name__}: {error}') + raise + finally: + status['finished_utc'] = utc() + write_json(output / 'preparation.json', status) + return status + + +def audit(output, plan): + import tomllib + rustc = Path(subprocess.check_output(['rustup', 'which', '--toolchain', '1.92.0', 'rustc'], timeout=30).decode('utf-8').strip()) + report = dict(schema='termforge-r54-local-build-audit-v1', builds={}, short_sanity_calls=[], + compiler=dict(sha256=sha(rustc), rustc_vv=plan['rustc_vv']), passed=False, + formal_samples=0, performance_gate='not_evaluated', environment_idle='not_measured') + try: + for variant, spec in plan['builds'].items(): + receipt_path = output / (variant + '-build.json') + receipt = json.loads(receipt_path.read_text(encoding='utf-8')) + checkout = output / ('source-' + variant) + metadata = tomllib.loads((checkout / 'experiments/candidates/alacritty/vendor/alacritty_terminal/Cargo.toml').read_text(encoding='utf-8')) + require(metadata['package']['version'] == SOURCES[variant]['version'], 'vendor version mismatch') + log = (output / (variant + '-build.stderr')).read_text(encoding='utf-8') + names = {json.loads('"' + value + '"') for value in re.findall(r'invoking linker "([^\n]+?)"', log)} + require(len(names) == 1, 'build log must identify one actual linker') + linker = Path(next(iter(names))) + require(linker.name == 'link.exe' and linker.is_file(), 'actual MSVC linker missing') + report['builds'][variant] = dict(source=spec['commit'], version=metadata['package']['version'], + candidate_tree=spec['candidate_tree'], tracked_candidate_files=len(spec['after_files']), + changed_paths=spec['changed_paths'], binary=receipt['binary'], identity=receipt['identity'], + linker=dict(path=str(linker), sha256=sha(linker)), receipt_sha256=sha(receipt_path)) + binary = Path(spec['target_dir']) / 'x86_64-pc-windows-msvc/release/alacritty-measure.exe' + for route in ('project', 'interactive'): + directory = output / f'sanity-{variant}-{route}' + directory.mkdir() + response = capture([str(binary), route, '4096', '0', '0.1'], directory, timeout=10) + report['short_sanity_calls'].append(dict(variant=variant, route=route, passed=False, **response)) + require(response['exit_code'] == 0 and not response['timed_out'], 'short correctness call failed') + result = json.loads((directory / 'stdout.bin').read_bytes().decode('utf-8')) + validate_result(result, .1, expected_sample(dict(variant=variant, route=route, chunk=4096))) + report['short_sanity_calls'][-1]['passed'] = True + require(report['builds']['baseline']['linker'] == report['builds']['candidate']['linker'], 'source pair used different linkers') + report['passed'] = True + finally: + write_json(output / 'build-audit.json', report) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + value = prepare_builds(args.out) + print(json.dumps(value)) + + +if __name__ == '__main__': + main() diff --git a/experiments/measure/r54_hosted.py b/experiments/measure/r54_hosted.py new file mode 100644 index 00000000..a59cb725 --- /dev/null +++ b/experiments/measure/r54_hosted.py @@ -0,0 +1,70 @@ +"""One owned GitHub-hosted Windows build followed by the original R54 batches.""" +import argparse +import json +import os +from pathlib import Path + +from experiments.measure.r54_batch import run, sha, utc, write_json +from experiments.measure.r54_build import prepare_builds +from experiments.measure.r54_contract import ROOT, SEEDS +from experiments.measure.r54_inputs import git, require + + +def hosted_identity(): + names = ('GITHUB_ACTIONS', 'RUNNER_ENVIRONMENT', 'RUNNER_OS', 'GITHUB_REPOSITORY', + 'GITHUB_SHA', 'GITHUB_REF', 'GITHUB_RUN_ID', 'GITHUB_RUN_NUMBER', 'GITHUB_RUN_ATTEMPT', + 'GITHUB_EVENT_NAME', 'ImageOS', 'ImageVersion') + identity = {name: os.environ.get(name) for name in names} + require(identity['GITHUB_ACTIONS'] == 'true' and identity['RUNNER_ENVIRONMENT'] == 'github-hosted' and + identity['RUNNER_OS'] == 'Windows' and os.name == 'nt', 'requires an actual GitHub-hosted Windows job') + require(identity['GITHUB_REPOSITORY'] == 'JavaZeroo/termforge' and identity['GITHUB_EVENT_NAME'] == 'push', + 'unexpected repository or event') + require(identity['GITHUB_REF'] == 'refs/heads/perf/grapheme-hosted-r54', 'unexpected experiment branch') + require(identity['GITHUB_RUN_ATTEMPT'] == '1', 'a workflow rerun must not replace the first attempt') + require(identity['GITHUB_SHA'] == git(ROOT, 'rev-parse', 'HEAD'), 'workflow/source identity mismatch') + return identity + + +def execute(output): + output = Path(output).resolve() + require(output.is_relative_to(ROOT / 'local'), 'hosted artifacts must stay under the workspace local directory') + output.mkdir(parents=True, exist_ok=False) + manifest = dict(schema='termforge-r54-hosted-v1', status='preparing', started_utc=utc(), + source=git(ROOT, 'rev-parse', 'HEAD'), runner_sha256=sha(Path(__file__)), + performance_gate='not_evaluated', batches=[]) + try: + manifest['hosted_identity'] = hosted_identity() + write_json(output / 'hosted.json', manifest) + prepare_builds(output / 'bundle') + declaration = dict(host_kind='hosted_vm', no_known_concurrent_build_or_test=True, + known_concurrent_work=['This job completed setup, tests and both builds before measurement.', + 'GitHub runner service and operating-system background services remain.', + 'Physical host scheduling and neighboring virtual machines are unobserved.']) + write_json(output / 'environment-declaration.json', declaration) + for seed in SEEDS: + manifest['status'] = 'running' + manifest['active_seed'] = seed + write_json(output / 'hosted.json', manifest) + result = run(output / 'bundle/plan-v1.json', output / 'bundle', output / 'environment-declaration.json', + output / f'batch-{seed}', seed) + manifest['batches'].append(result) + require(result['status'] == 'complete_diagnostic', f'batch {seed} stopped; remaining seeds not launched') + manifest.update(status='complete_diagnostic', active_seed=None) + except BaseException as error: + manifest.update(status='failed', error=f'{type(error).__name__}: {error}') + raise + finally: + manifest['finished_utc'] = utc() + write_json(output / 'hosted.json', manifest) + return manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(execute(args.out))) + + +if __name__ == '__main__': + main() diff --git a/experiments/measure/r54_publish.py b/experiments/measure/r54_publish.py new file mode 100644 index 00000000..5a9787a0 --- /dev/null +++ b/experiments/measure/r54_publish.py @@ -0,0 +1,172 @@ +"""Publish a fixed synthetic-artifact whitelist from the owned hosted job.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re + +from experiments.measure.r54_batch import sha, write_json +from experiments.measure.r54_contract import ROOT, SEEDS +from experiments.measure.r54_hosted import hosted_identity +from experiments.measure.r54_inputs import require + + +def public_environment(value): + """CPU counters remain; process names/PIDs and unrecognized fields do not.""" + fields = ('status', 'host', 'started_utc', 'finished_utc', 'intervals', 'declaration', 'policy', 'power_policy', 'error') + result = {name: value[name] for name in fields if name in value} + if 'host' in result: + host = value['host'] + result['host'] = {name: host[name] for name in ('platform', 'python', 'psutil', 'cpu_logical', 'cpu_physical', 'cpu', 'windows') if name in host} + return result + + +def validate_raw_identity(root, identity): + value = json.loads((root / 'hosted.json').read_text(encoding='utf-8')) + require(value['schema'] == 'termforge-r54-hosted-v1' and value['hosted_identity'] == identity and + value['source'] == identity['GITHUB_SHA'], 'raw evidence belongs to another run, attempt or source') + + +def windows_short_path(path): + """Ask Windows for the actual alias; guessing RUNNER~1 is not reliable.""" + import ctypes + from ctypes import wintypes + function = ctypes.WinDLL('kernel32', use_last_error=True).GetShortPathNameW + function.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] + function.restype = wintypes.DWORD + length = function(path, None, 0) + if not length: + raise ctypes.WinError(ctypes.get_last_error()) + buffer = ctypes.create_unicode_buffer(length) + copied = function(path, buffer, length) + if not copied or copied >= length: + raise ValueError('short-path lookup failed or changed during publication') + return buffer.value + + +def replacement_forms(prefixes): + forms = [] + for prefix, alias in prefixes: + forms.extend((form, alias) for form in (prefix.replace('\\', '\\\\'), prefix, prefix.replace('\\', '/'))) + return sorted(set(forms), key=lambda item: len(item[0]), reverse=True) + + +def redact_paths(data, replacements): + # Preserve arbitrary log bytes. The hosted Windows path prefixes are ASCII + # and may use a different case or a compiler-escaped 8.3 alias. + for before, after in replacements: + data = re.sub(re.escape(before.encode('utf-8')), lambda _: after.encode('utf-8'), data, flags=re.IGNORECASE) + return data + + +def publish_jsonl(original, replacements): + # JSONL is appended, unlike the atomic JSON manifests. A stop can leave an + # incomplete final row. Preserve it and the complete rows, mark the parse + # problem, and continue exporting independently retained sample streams. + status = dict(complete=True, error=None) + for index, line in enumerate(original.splitlines(), start=1): + try: + json.loads(line.decode('utf-8')) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + status.update(complete=False, error=f'{type(error).__name__}: {error}', error_line=index) + break + return redact_paths(original, replacements), status + + +def selected_paths(root): + """No recursive directory export: only named builder and runner artifacts.""" + names = ['hosted.json', 'environment-declaration.json'] + names += ['bundle/' + name for name in ('preparation.json', 'plan-v1.json', 'build-plan.json', 'build-audit.json')] + for variant in ('baseline', 'candidate'): + names += [f'bundle/{variant}-harness.patch', f'bundle/{variant}-build.json'] + for step in ('checkout', 'fetch', 'build', 'identity'): + names += [f'bundle/{variant}-{step}.{suffix}' for suffix in ('stdout', 'stderr', 'process.json')] + for route in ('project', 'interactive'): + names += [f'bundle/sanity-{variant}-{route}/{stream}.bin' for stream in ('stdout', 'stderr')] + for seed in SEEDS: + prefix = f'batch-{seed}/' + names += [prefix + name for name in ('manifest.json', 'runner.json', 'environment.json', 'environment-policy.json', + 'environment-declaration.json', 'samples/context.json', 'samples/manifest.json', 'samples/samples.jsonl', 'samples/summary.json')] + for directory in ('baseline-identity', 'candidate-identity', 'power-policy', 'power-policy-after'): + names += [prefix + directory + '/' + name for name in ('record.json', 'stdout.bin', 'stderr.bin')] + for sequence in range(112): + names += [prefix + f'samples/{sequence:03d}/' + name for name in ('record.json', 'stdout.bin', 'stderr.bin')] + return [Path(name) for name in names if (root / name).is_file()] + + +def publish(root, output): + identity = hosted_identity() + root, output = Path(root).resolve(), Path(output).resolve() + require(root.is_relative_to(ROOT / 'local') and output.is_relative_to(ROOT / 'local'), 'publication paths must stay in workspace local') + require(root != output and not output.is_relative_to(root), 'public output must be separate from raw records') + require(not output.exists(), 'public output already exists') + validate_raw_identity(root, identity) + staging = output.with_name(output.name + '.partial') + require(staging.is_relative_to(ROOT / 'local'), 'staging path escaped workspace') + staging.mkdir(parents=True, exist_ok=False) + prefixes = [(str(ROOT), '${WORKSPACE}'), (os.environ.get('USERPROFILE', ''), '${PROFILE}'), + (os.environ.get('RUNNER_TEMP', ''), '${RUNNER_TEMP}')] + prefixes = [(prefix, alias) for prefix, alias in prefixes if prefix] + prefixes += [(windows_short_path(prefix), alias) for prefix, alias in prefixes] + replacements = replacement_forms(prefixes) + + def text(value): + return redact_paths(value.encode('utf-8'), replacements).decode('utf-8') + + def transform(value): + if isinstance(value, dict): + return {key: transform(item) for key, item in value.items()} + if isinstance(value, list): + return [transform(item) for item in value] + return text(value) if isinstance(value, str) else value + + inventory = [] + for relative in selected_paths(root): + source = root / relative + require(source.resolve().is_relative_to(root), 'artifact path escaped raw directory') + original = source.read_bytes() + jsonl_status = None + # The measurement EXE consumes only public, fixed synthetic workloads. + # Preserve its raw streams even on failure. Other logs can contain host + # path prefixes; redact those without discarding their full contents. + raw_sample = bool(re.fullmatch(r'(?:batch-\d+/samples/\d{3}|bundle/sanity-(?:baseline|candidate)-(?:project|interactive))/(?:stdout|stderr)\.bin', relative.as_posix())) + if raw_sample: + data, treatment = original, 'byte_exact_owned_synthetic_stream' + elif relative.suffix == '.json': + value = json.loads(original.decode('utf-8')) + if relative.name == 'environment.json': + value = public_environment(value) + data = (json.dumps(transform(value), ensure_ascii=False, indent=2, allow_nan=False) + '\n').encode('utf-8') + treatment = 'json_whitelist_and_path_redaction' + elif relative.suffix == '.jsonl': + data, jsonl_status = publish_jsonl(original, replacements) + treatment = 'jsonl_path_redaction' + else: + data = redact_paths(original, replacements) + treatment = 'complete_log_with_path_prefix_redaction' + target = staging / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + inventory.append(dict(path=relative.as_posix(), original_bytes=len(original), + original_sha256=hashlib.sha256(original).hexdigest(), + published_bytes=len(data), published_sha256=sha(target), treatment=treatment, + jsonl_status=jsonl_status)) + write_json(staging / 'publication.json', dict(schema='termforge-r54-hosted-publication-v1', + hosted_identity=identity, artifacts=inventory, raw_process_identities_published=False, + full_environment_variables_published=False, performance_gate='not_evaluated', + limitation='guest CPU evidence cannot establish exclusive physical host or stable frequency')) + staging.rename(output) + return dict(files=len(inventory), output=str(output), performance_gate='not_evaluated') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--raw', type=Path, required=True) + parser.add_argument('--out', type=Path, required=True) + args = parser.parse_args() + print(json.dumps(publish(args.raw, args.out))) + + +if __name__ == '__main__': + main() diff --git a/experiments/measure/results/2026-10-27-r54-hosted/README.md b/experiments/measure/results/2026-10-27-r54-hosted/README.md new file mode 100644 index 00000000..429335cf --- /dev/null +++ b/experiments/measure/results/2026-10-27-r54-hosted/README.md @@ -0,0 +1,24 @@ +# #54 首次 hosted 固定批次：环境护栏停止 + +固定源码 `f734d29bc4f8a120eac59da3acac3169d49231b2` 在 [GitHub run 40490861301](https://github.com/JavaZeroo/termforge/actions/runs/40490861301) 的首次 `windows-2025` 作业中完成两份构建、来源审计及4次短正确性检查。原协议、两个种子、各112位置、2秒CPU/墙钟下限、5%环境诊断护栏均未改变。 + +第一批的十个空闲区间为1.154%–3.125%，通过预检。随后执行24个位置：23个记录有效，第24个（sequence 23，candidate/interactive/chunk 1）输出正确且退出0，但未归属CPU活动上界估计为5.410447761%，超过5%护栏，故停止。原始数据全部保留，没有续跑、补样、删除慢样本或启动第二批。 + +该位置全机总CPU区间为8.375秒，busy为2.484375秒，内层测量进程CPU为2.03125秒，余额0.453125秒。余额包含内层区间以外的准备、oracle、启动和runner工作，不能直接称为其他进程的后台CPU，更不能据此定位具体进程。临时VM的逐进程身份未导出，不能追溯补造归因。 + +**完整批次数为0，不生成配对成本摘要，不从这个前缀计算或验收吞吐回归。** `performance_gate=not_evaluated`；5%不是用户批准的产品性能容差。物理宿主独占、邻居VM、频率稳定、输入/传输/呈现及WindTerm等价仍未测。 + +- [validation.json](validation.json)：源码/树、构建/二进制、宿主、停止位置、限制及同源码回归CI。 +- [evidence.zip](evidence.zip)：审阅后的164,608B公开副本，SHA-256 `9ade502f03419f55295b4e9f4df9dc136c5c4d50ace02d41417fea1de6952917`。内含131个文件：130份白名单证据和`publication.json`；56份合成stdout/stderr原字节保留。全部24次输出的正确性和完整字段已独立复查。 +- 提供者原归档165,170B、SHA-256 `6a2713dd96d86ce66e2462a142d46b72e915ccc4cfd45bd3b42c5e7486c3cef5`保存在local。产物复审发现两份构建日志各有10处未处理的8.3短profile前缀；未发现凭据。公开副本仅补做这些前缀的别名替换，并更新清单，保留转换前/后哈希和`post_publication_review`；成本数据、顺序和停止状态未改，不冒充原ZIP字节。 +- 复查副本的来源、固定白名单、长/短路径及秘密字段后，未发现逐进程名称/PID、凭据或未处理的workspace/profile前缀。旧作业确实使用旧导出器；后续实现修正不回填旧运行身份，也不触发补跑成本批次。 + +测量源码对应的默认PR/push回归均通过（Windows46/46、Linux29/29）；候选两条CI也通过（measurement Windows40/40，Linux39通过/1个Windows专属跳过）。当时相关本地measurement31/31；后续短路径修正另有失败反例、实际Windows别名查询及32/32回归。普通CI与成本作业分开，不用CI成功抵消成本批次失败。日志哈希指向本地保留的UTF-8/LF解码文本，并非提供者日志ZIP原件。 + +归档复查命令不会执行成本负载或修改数据： + +```powershell +python experiments/measure/results/2026-10-27-r54-hosted/verify.py +``` + +下一项调查由[#54](https://github.com/JavaZeroo/termforge/issues/54)接续：另行预声明有界诊断，观测整个测量子进程及runner的CPU，区分自身区间外工作与外部guest活动；本次失败和v1规则保持。扰动/组件归因、完整受控批次及用户预算确认仍未完成。 diff --git a/experiments/measure/results/2026-10-27-r54-hosted/evidence.zip b/experiments/measure/results/2026-10-27-r54-hosted/evidence.zip new file mode 100644 index 00000000..3f12caf7 Binary files /dev/null and b/experiments/measure/results/2026-10-27-r54-hosted/evidence.zip differ diff --git a/experiments/measure/results/2026-10-27-r54-hosted/validation.json b/experiments/measure/results/2026-10-27-r54-hosted/validation.json new file mode 100644 index 00000000..199ea78f --- /dev/null +++ b/experiments/measure/results/2026-10-27-r54-hosted/validation.json @@ -0,0 +1,316 @@ +{ + "schema": "termforge-r54-hosted-validation-v1", + "status": "incomplete_first_batch_stopped_by_environment_guard", + "source": "f734d29bc4f8a120eac59da3acac3169d49231b2", + "tree": "361683a329aa7c24d34fc34158b42ea55f4786f7", + "reviewed_local_source": "a8daaab2202fa5a36cbad216959355583eba78f0", + "base": "a7b9685a20396779dc9dff51018470b7638bfdce", + "pr_merge_source": "38c6703403948f79c7112d7340df4087d0689edc", + "pr_merge_tree_matches_reviewed": true, + "protocol_sha256": "fb86f093a1c526979816574ef65896501eaeb9e9c34e7d9a6f2fdd5b61a8e7f6", + "review": { + "standards_remaining": 0, + "spec_remaining": 0, + "read_only": true + }, + "local_validation": { + "measurement_tests": 31, + "passed": 31, + "unittest_seconds": 13.809, + "ctest_seconds": 13.99, + "stdout_sha256": "f31c7251649be1d7d9157eeb3a9d83c8d397e6cadd4a5f80347f689637b6caf5", + "stderr_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "hosted": { + "run": 40490861301, + "job": 122144148028, + "attempt": 1, + "conclusion": "failure", + "url": "https://github.com/JavaZeroo/termforge/actions/runs/40490861301", + "runner_labels": [ + "windows-2025" + ], + "contract_tests_passed": 22, + "decoded_job_log_sha256": "fbb8fad6fda8eaad64a940cbd0dae55fdec9690c9cde3b2da7df857c4b0ee1ad", + "build_status": "built_and_audited", + "sanity_passed": 4 + }, + "compiler": { + "rustc_vv": "rustc 1.92.0 (ded5c06cf 2025-12-08)\nbinary: rustc\ncommit-hash: ded5c06cf21d2b93bffd5d884aa6e96934ee4234\ncommit-date: 2025-12-08\nhost: x86_64-pc-windows-msvc\nrelease: 1.92.0\nLLVM version: 21.1.3\n", + "cargo_vv": "cargo 1.92.0 (344c4567c 2025-10-21)\nrelease: 1.92.0\ncommit-hash: 344c4567c634a25837e3c3476aac08af84cf9203\ncommit-date: 2025-10-21\nhost: x86_64-pc-windows-msvc\nlibgit2: 1.9.1 (sys:0.20.2 vendored)\nlibcurl: 8.15.0-DEV (sys:0.4.83+curl-8.15.0 vendored ssl:Schannel)\nos: Windows 10.0.26100 (Windows 11 ServerDatacenter) [64-bit]\n", + "sha256": "5f3b5087b95e250378dd7bd829ae627d51c149468aae1e38c0f2a85c17cd374e" + }, + "builds": { + "baseline": { + "source": "65988f5f20f62294ff52fcac5b5b81e563f93f3b", + "version": "0.26.1-dev+termforge.3", + "candidate_tree": "38f901ce2407d73ab7d4ae70987c294849bafa92", + "tracked_candidate_files": 80, + "changed_paths": [ + "experiments/candidates/alacritty/Cargo.toml", + "experiments/candidates/alacritty/src/measure.rs" + ], + "binary": { + "bytes": 523264, + "sha256": "ac7c1d4ba996f9fe2c0d911a077bcab557f5b0b7332cde757dfb3f7db2c499a1" + }, + "identity": { + "allocator_wrapper_compiled": false, + "projection_selectable": true, + "schema": "termforge-core-cost-build-v1", + "timing_gates_compiled": true + }, + "linker": { + "path": "C:\\Program Files\\Microsoft Visual Studio\\18\\Enterprise\\VC\\Tools\\MSVC\\14.51.36231\\bin\\HostX64\\x64\\link.exe", + "sha256": "a8c42e25c91ca2453e1110540ac08f0fc12d7539454a11b9af8baeecb469837d" + }, + "receipt_sha256": "7cabd0c3610017fdb2f296999e2d96083bf5f4ba245aa0b414aab1d1527fa9ea" + }, + "candidate": { + "source": "3d6d44d3b3e9a9a3e41da98e8711e9f19a211fd8", + "version": "0.26.1-dev+termforge.4", + "candidate_tree": "2cce55fcf2d379440110bbbf8612117cce3edf64", + "tracked_candidate_files": 84, + "changed_paths": [ + "experiments/candidates/alacritty/Cargo.toml", + "experiments/candidates/alacritty/src/measure.rs" + ], + "binary": { + "bytes": 580096, + "sha256": "e674f0df59af27151fe2c1ad916b6cb4f4dd6ccd783ed225fe0107e13b74ae0a" + }, + "identity": { + "allocator_wrapper_compiled": false, + "projection_selectable": true, + "schema": "termforge-core-cost-build-v1", + "timing_gates_compiled": true + }, + "linker": { + "path": "C:\\Program Files\\Microsoft Visual Studio\\18\\Enterprise\\VC\\Tools\\MSVC\\14.51.36231\\bin\\HostX64\\x64\\link.exe", + "sha256": "a8c42e25c91ca2453e1110540ac08f0fc12d7539454a11b9af8baeecb469837d" + }, + "receipt_sha256": "eeef0cfd10f469bb44ddade96cae6c81263c600cfa014a984bc4466a452c14f9" + } + }, + "batches": [ + { + "seed": 20261026, + "status": "failed", + "attempted": 24, + "valid_samples": 23, + "error": "ValueError: ValueError: sample 23 failed: ValueError: host activity guard failed; sample retained, batch stopped", + "host": { + "platform": "Windows-2025Server-10.0.26100-SP0", + "python": "3.12.10", + "psutil": "7.2.2", + "cpu_logical": 4, + "cpu_physical": 2, + "cpu": { + "ProcessorNameString": "AMD EPYC 9V45 96-Core Processor ", + "Identifier": "AMD64 Family 26 Model 2 Stepping 1" + }, + "windows": { + "CurrentBuildNumber": "26100", + "UBR": 33650, + "DisplayVersion": "24H2" + } + }, + "environment_status": "low_activity_observed", + "busy_percent": [ + 1.5625, + 1.5625, + 1.953125, + 1.171875, + 1.550387596899225, + 1.550387596899225, + 1.953125, + 3.125, + 1.171875, + 1.1538461538461537 + ] + } + ], + "all_attempted_output_oracles_passed": 24, + "all_attempted_processes_exit_zero": 24, + "stopped_sample": { + "item": { + "sequence": 23, + "repeat": 1, + "route": "interactive", + "chunk": 1, + "variant": "candidate", + "observe": 0 + }, + "status": "failed", + "error": "ValueError: host activity guard failed; sample retained, batch stopped", + "guest_total_cpu_seconds": 8.375, + "guest_busy_cpu_seconds": 2.484375, + "inner_measured_process_cpu_seconds": 2.03125, + "unattributed_cpu_seconds": 0.453125, + "unattributed_busy_percent_upper_estimate": 5.41044776119403 + }, + "second_seed_started": false, + "full_batches_completed": 0, + "paired_cost_summary_produced": false, + "performance_gate": "not_evaluated", + "numerical_product_budget": null, + "archive": { + "path": "evidence.zip", + "bytes": 164608, + "sha256": "9ade502f03419f55295b4e9f4df9dc136c5c4d50ace02d41417fea1de6952917", + "provider_artifact": 12728432619, + "files": 131, + "inventory_verified_files": 130, + "byte_exact_synthetic_streams": 56, + "individual_publication_sha256": "75f98a70ae4567eebb7686c5284a84df5f4b137220fe0032cbd6579aa82c36dd", + "privacy_review": "Reviewed public copy additionally replaces the observed Windows 8.3 profile aliases in two build logs. No process identities, credentials or remaining workspace/profile prefixes found. Measurement streams unchanged.", + "provider_archive_bytes": 165170, + "provider_archive_sha256": "6a2713dd96d86ce66e2462a142d46b72e915ccc4cfd45bd3b42c5e7486c3cef5", + "provider_original_retained_locally": true, + "post_publication_review": [ + { + "path": "bundle/baseline-build.stderr", + "previous_published_sha256": "812ba8957f4fcf9b5019361b2b022ad529b695dbc89f89b7bec7c95b89969626", + "reviewed_sha256": "585a85a2909b2e5325e6573ee5c9cf6e03a1a389adf09409b7f9da92dd87bee1", + "replacements": 10 + }, + { + "path": "bundle/candidate-build.stderr", + "previous_published_sha256": "44c59810fe10f93df597115060e6cf86749350a96f92fe9df174d0241da66579", + "reviewed_sha256": "b25d91f15ca587c2fa37420f3bc2a5ca357426a73eb5cc2885b522f87fb66860", + "replacements": 10 + } + ] + }, + "ci": [ + { + "run": 40490884320, + "event": "pull_request", + "name": "default", + "jobs": [ + { + "job": 122144220963, + "platform": "windows-2025", + "passed": 46, + "collected": 46, + "seconds": 276.62, + "decoded_log_sha256": "46d7dc7a0d2adcd51de07e6aa27c132a155f3e313a4bf947bd220df426d98002" + }, + { + "job": 122144221116, + "platform": "ubuntu-24.04", + "passed": 29, + "collected": 29, + "seconds": 16.67, + "decoded_log_sha256": "1df6af066a76fa8d090a46f38a1612d7d1d4c6e3b3db9a5b9a858d19fccb752d" + } + ], + "conclusion": "success", + "attempt": 1, + "url": "https://github.com/JavaZeroo/termforge/actions/runs/40490884320" + }, + { + "run": 40490861189, + "event": "push", + "name": "default", + "jobs": [ + { + "job": 122144147319, + "platform": "windows-2025", + "passed": 46, + "collected": 46, + "seconds": 282.38, + "decoded_log_sha256": "318d044f96dcccb27983100d3fdbd897073142e63e656e369f6d838d45991d16" + }, + { + "job": 122144147638, + "platform": "ubuntu-24.04", + "passed": 29, + "collected": 29, + "seconds": 16.02, + "decoded_log_sha256": "14366db79e80b31887b26d5e97d0bc9bdd7de001f83812400afd00c63c09f51b" + } + ], + "conclusion": "success", + "attempt": 1, + "url": "https://github.com/JavaZeroo/termforge/actions/runs/40490861189" + }, + { + "run": 40490883616, + "event": "pull_request", + "name": "candidate", + "measurement_jobs": [ + { + "job": 122144218593, + "platform": "windows-2025", + "passed": 40, + "collected": 40, + "skipped": 0, + "seconds": 30.304, + "decoded_log_sha256": "713d9b818aa14dd38480205e81c9d90eee74e94b717ca81ecdbb1b67dade0df9" + }, + { + "job": 122144218760, + "platform": "ubuntu-24.04", + "passed": 39, + "collected": 40, + "skipped": 1, + "seconds": 22.982, + "decoded_log_sha256": "b3e3033bd7c232de80b702c7945c1292dbe1ef1e9630416c8021318521fae7e0" + } + ], + "conclusion": "success", + "attempt": 1, + "url": "https://github.com/JavaZeroo/termforge/actions/runs/40490883616" + }, + { + "run": 40490861431, + "event": "push", + "name": "candidate", + "measurement_jobs": [ + { + "job": 122144148011, + "platform": "windows-2025", + "passed": 40, + "collected": 40, + "skipped": 0, + "seconds": 29.831, + "decoded_log_sha256": "f2ecb43c16ef0a8c3f6d75b6d1989f16b9027af852def85c03cb94dbd8f39f00" + }, + { + "job": 122144147926, + "platform": "ubuntu-24.04", + "passed": 39, + "collected": 40, + "skipped": 1, + "seconds": 25.717, + "decoded_log_sha256": "8015b82b14881aff82fa1d71db2a1e801d83aa604ac8a83a5a49d281c36f5f2f" + } + ], + "conclusion": "success", + "attempt": 1, + "url": "https://github.com/JavaZeroo/termforge/actions/runs/40490861431" + } + ], + "ci_logs_representation": "decoded UTF-8, LF; hashes identify retained local text, not provider ZIP bytes", + "limitations": [ + "The batch is incomplete: do not calculate or accept a throughput regression from this prefix.", + "5 percent is the frozen guest CPU diagnostic guard, not a user-approved product performance tolerance.", + "The remainder includes preparation/oracle/startup and runner CPU excluded by the inner timing interval; it does not identify the responsible process.", + "Per-process identities were not exported from the temporary VM; they cannot support retrospective attribution.", + "Physical host exclusivity, neighboring VM load, frequency stability, GUI/input/receive/presentation and WindTerm equivalence remain unmeasured.", + "No workflow rerun, skipped sample, replacement sample, relaxed guard or resumed second seed." + ], + "next_investigation": "Separately predeclare a bounded diagnostic of whole-child lifetime CPU and runner CPU to distinguish own work from external guest activity; preserve this first failure and the v1 acceptance rules.", + "post_run_publication_fix": { + "reason": "Windows compiler logs used short profile aliases absent from the original prefix list.", + "native_windows_short_path_lookup": "passed_same_directory", + "path_variant_regression": "red_then_green", + "measurement_tests": 32, + "passed": 32, + "unittest_seconds": 15.152, + "stdout_sha256": "c8c96d18f0a4f88593607294a831a3163735f74fa9e650a1c74424008b61e95c", + "cost_execution": "not_repeated; branch update no longer launches cost job", + "original_cost_source_unchanged": true + } +} diff --git a/experiments/measure/results/2026-10-27-r54-hosted/verify.py b/experiments/measure/results/2026-10-27-r54-hosted/verify.py new file mode 100644 index 00000000..ebe62162 --- /dev/null +++ b/experiments/measure/results/2026-10-27-r54-hosted/verify.py @@ -0,0 +1,78 @@ +"""Verify the immutable first hosted failure archive; do not run measurements.""" +import hashlib +import io +import json +from pathlib import Path +import sys +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[4])) +from experiments.measure.r54_contract import validate_sample + + +def main(): + data = Path(__file__).with_name('evidence.zip').read_bytes() + assert len(data) == 164608 + assert hashlib.sha256(data).hexdigest() == '9ade502f03419f55295b4e9f4df9dc136c5c4d50ace02d41417fea1de6952917' + with zipfile.ZipFile(io.BytesIO(data)) as archive: + read = lambda name: json.loads(archive.read(name).decode('utf-8')) + publication = read('publication.json') + review = publication['post_publication_review'] + assert review['source_provider_archive_sha256'] == '6a2713dd96d86ce66e2462a142d46b72e915ccc4cfd45bd3b42c5e7486c3cef5' + assert review['measurement_streams_unchanged'] is True + assert {item['path'] for item in review['changes']} == {'bundle/baseline-build.stderr', 'bundle/candidate-build.stderr'} + inventory = {item['path']: item for item in publication['artifacts']} + assert len(inventory) == len(publication['artifacts']) == 130 + assert len(archive.namelist()) == 131 + assert set(archive.namelist()) == set(inventory) | {'publication.json'} + exact = 0 + for name, item in inventory.items(): + payload = archive.read(name) + assert len(payload) == item['published_bytes'], name + assert hashlib.sha256(payload).hexdigest() == item['published_sha256'], name + if item['treatment'] == 'byte_exact_owned_synthetic_stream': + assert item['original_bytes'] == len(payload), name + assert item['original_sha256'] == item['published_sha256'], name + exact += 1 + assert exact == 56 + hosted = read('hosted.json') + assert hosted['status'] == 'failed' and hosted['performance_gate'] == 'not_evaluated' + assert hosted['source'] == 'f734d29bc4f8a120eac59da3acac3169d49231b2' + assert hosted['hosted_identity'] == publication['hosted_identity'] + assert hosted['hosted_identity']['GITHUB_RUN_ID'] == '40490861301' + assert hosted['hosted_identity']['GITHUB_RUN_ATTEMPT'] == '1' + prefix = 'batch-20261026/' + manifest = read(prefix + 'manifest.json') + assert manifest['status'] == 'failed' and manifest['attempted'] == 24 and manifest['valid_samples'] == 23 + plan = read('bundle/plan-v1.json') + schedule = plan['protocol']['schedules'][0] + assert schedule['seed'] == 20261026 + records = [] + for sequence in range(24): + directory = prefix + f'samples/{sequence:03d}/' + record = read(directory + 'record.json') + assert record['item'] == schedule['samples'][sequence] + assert record['status'] == ('valid' if sequence < 23 else 'failed') + assert record['exit_code'] == 0 and not record['timed_out'] + result = read(directory + 'stdout.bin') + assert result == record['result'] + validate_sample(result, record['item']) + for name, identity in record['streams'].items(): + payload = archive.read(directory + name) + assert len(payload) == identity['bytes'] and hashlib.sha256(payload).hexdigest() == identity['sha256'] + environment = record['environment'] + remainder = environment['busy_seconds'] - result['interval_cpu_seconds'] + upper = remainder * 100 / environment['total_seconds'] + assert abs(upper - environment['unattributed_busy_percent_upper_estimate']) < 1e-12 + assert (0 <= upper <= 5) if sequence < 23 else upper > 5 + records.append(record) + assert [json.loads(line) for line in archive.read(prefix + 'samples/samples.jsonl').splitlines()] == records + assert prefix + 'samples/summary.json' not in inventory + assert not any(name.startswith('batch-20261027/') for name in inventory) + print(json.dumps(dict(archive_files=131, hashed_evidence_files=130, byte_exact_streams=56, + attempted=24, valid_records=23, correct_outputs=24, + full_batches=0, performance_gate='not_evaluated'))) + + +if __name__ == '__main__': + main() diff --git a/experiments/measure/test_r54_hosted.py b/experiments/measure/test_r54_hosted.py new file mode 100644 index 00000000..a66be4d7 --- /dev/null +++ b/experiments/measure/test_r54_hosted.py @@ -0,0 +1,96 @@ +"""Build input and publication boundaries; no performance workloads in tests.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] + + +class BuildBoundaryTests(unittest.TestCase): + def test_unreviewed_compiler_override_stops_before_source_checkout(self): + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) / 'bundle' + result = subprocess.run( + [sys.executable, '-m', 'experiments.measure.r54_build', '--out', str(output)], + cwd=ROOT, env=dict(os.environ, RUSTFLAGS='-C opt-level=0'), + capture_output=True, timeout=20) + self.assertNotEqual(result.returncode, 0) + manifest = json.loads((output / 'preparation.json').read_text(encoding='utf-8')) + self.assertEqual(manifest['status'], 'failed') + self.assertIn('RUSTFLAGS', manifest['error']) + self.assertFalse((output / 'source-baseline').exists()) + self.assertFalse((output / 'plan-v1.json').exists()) + + +class PublicationBoundaryTests(unittest.TestCase): + def test_public_logs_redact_long_short_escaped_and_case_variants_without_changing_other_bytes(self): + from experiments.measure.r54_publish import replacement_forms, redact_paths + forms = replacement_forms([(r'C:\Users\runneradmin', '${PROFILE}'), + (r'C:\Users\RUNNER~1', '${PROFILE}')]) + original = (b'C:\\Users\\runneradmin\\Temp\\a C:\\\\Users\\\\RUNNER~1\\\\Temp\\\\b ' + b'c:/users/runner~1/Temp/c\xff\n') + self.assertEqual(redact_paths(original, forms), + b'${PROFILE}\\Temp\\a ${PROFILE}\\\\Temp\\\\b ${PROFILE}/Temp/c\xff\n') + + def test_interrupted_jsonl_tail_remains_visible_without_losing_complete_samples(self): + from experiments.measure.r54_publish import publish_jsonl + original = b'{"command":["/ci/work/tool.exe"],"cpu":2.0}\n{"command":[' + data, status = publish_jsonl(original, [('/ci/work', '${WORKSPACE}')]) + self.assertEqual(data, b'{"command":["${WORKSPACE}/tool.exe"],"cpu":2.0}\n{"command":[') + self.assertFalse(status['complete']) + self.assertIn('JSONDecodeError', status['error']) + self.assertEqual(status['error_line'], 2) + + def test_missing_or_different_raw_run_identity_is_rejected(self): + from experiments.measure.r54_publish import validate_raw_identity + identity = dict(GITHUB_RUN_ID='123', GITHUB_RUN_ATTEMPT='1', GITHUB_SHA='a' * 40) + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + with self.assertRaises(FileNotFoundError): + validate_raw_identity(root, identity) + path = root / 'hosted.json' + path.write_text(json.dumps(dict(schema='termforge-r54-hosted-v1', source='a' * 40, + hosted_identity=identity)), encoding='utf-8') + validate_raw_identity(root, identity) + with self.assertRaises(ValueError): + validate_raw_identity(root, dict(identity, GITHUB_RUN_ID='124')) + + def test_export_inventory_does_not_include_unrelated_or_arbitrary_files(self): + from experiments.measure.r54_publish import selected_paths + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + files = ['bundle/plan-v1.json', 'bundle/credentials.json', + 'batch-20261026/samples/000/stdout.bin', + 'batch-20261026/samples/000/unrelated.txt', + 'batch-20261026/samples/999/stdout.bin', 'personal-session.log'] + for name in files: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'fixture') + self.assertEqual({path.as_posix() for path in selected_paths(root)}, + {'bundle/plan-v1.json', 'batch-20261026/samples/000/stdout.bin'}) + + def test_environment_export_keeps_cpu_evidence_and_omits_process_identities(self): + from experiments.measure.r54_publish import public_environment + value = dict(status='rejected_busy_host', host=dict(platform='Windows', cpu_logical=4), + intervals=[dict(busy_percent=7.5, total_seconds=4, busy_seconds=.3)], + processes_before=[dict(pid=123, name='private-name', cpu_times=dict(user=2))], + processes_after=[dict(pid=123, name='private-name', cpu_times=dict(user=3))], + unexpected_private_field='do-not-publish') + result = public_environment(value) + self.assertEqual(result['intervals'], value['intervals']) + self.assertEqual(result['host']['cpu_logical'], 4) + self.assertEqual(result['status'], 'rejected_busy_host') + self.assertNotIn('private-name', json.dumps(result)) + self.assertNotIn('do-not-publish', json.dumps(result)) + self.assertNotIn('processes_before', result) + + + +if __name__ == '__main__': + unittest.main()