From 56459cabd697f999547b0dec77e59fd0422c67c5 Mon Sep 17 00:00:00 2001 From: JavaZeroo <71128095+JavaZeroo@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:05:45 +0800 Subject: [PATCH 1/3] perf: prepare R54 cost contract and allocator-free diagnostic builds --- .github/workflows/core-candidate.yml | 5 + CMakeLists.txt | 2 +- experiments/candidates/alacritty/Cargo.toml | 6 + .../candidates/alacritty/src/measure.rs | 88 +++++++---- experiments/measure/R54-COST.md | 62 ++++++++ experiments/measure/README.md | 2 + experiments/measure/r54_contract.py | 141 ++++++++++++++++++ .../validation.json | 119 +++++++++++++++ experiments/measure/run.py | 12 +- experiments/measure/test_r54_contract.py | 139 +++++++++++++++++ 10 files changed, 547 insertions(+), 29 deletions(-) create mode 100644 experiments/measure/R54-COST.md create mode 100644 experiments/measure/r54_contract.py create mode 100644 experiments/measure/results/2026-10-26-r54-preparation/validation.json create mode 100644 experiments/measure/test_r54_contract.py diff --git a/.github/workflows/core-candidate.yml b/.github/workflows/core-candidate.yml index 09070262..56091469 100644 --- a/.github/workflows/core-candidate.yml +++ b/.github/workflows/core-candidate.yml @@ -46,6 +46,8 @@ jobs: PYTHONUTF8: '1' TERMFORGE_ALACRITTY_TRACE: ${{ github.workspace }}/${{ matrix.binary }} TERMFORGE_TRACE: ${{ github.workspace }}/${{ matrix.core_binary }} + TERMFORGE_R54_WRAPPED: ${{ github.workspace }}/experiments/candidates/alacritty/target/release/alacritty-measure${{ runner.os == 'Windows' && '.exe' || '' }} + TERMFORGE_R54_UNWRAPPED: ${{ github.workspace }}/build/measure-no-allocator/release/alacritty-measure${{ runner.os == 'Windows' && '.exe' || '' }} TERMFORGE_TEST_ARTIFACTS: ${{ github.workspace }}/build/test-artifacts steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -68,6 +70,9 @@ jobs: - name: Build candidate measurement working-directory: experiments/candidates/alacritty run: cargo build --release --locked --bin alacritty-measure + - name: Build candidate measurement without allocator wrapper + working-directory: experiments/candidates/alacritty + run: cargo build --release --locked --no-default-features --bin alacritty-measure --target-dir "${{ github.workspace }}/build/measure-no-allocator" - name: Configure native trace on Windows if: runner.os == 'Windows' run: cmake -S . -B build -A x64 -DTERMFORGE_BUILD_CANDIDATE_ADAPTER=ON diff --git a/CMakeLists.txt b/CMakeLists.txt index e864d422..c1b44f08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -289,7 +289,7 @@ if(BUILD_TESTING) endif() termforge_add_python_test(core -m unittest experiments.core.test_core experiments.core.test_unicode -v) termforge_add_python_test(recovery -m unittest experiments.core.test_recovery -v) - termforge_add_python_test(measure -m unittest experiments.measure.test_measure -v) + termforge_add_python_test(measure -m unittest experiments.measure.test_measure experiments.measure.test_r54_contract.CostPlanTests -v) termforge_add_python_test(binary_paths -m unittest experiments.test_build_paths -v) termforge_add_python_test(benchmark_tools -m unittest discover -s benchmarks/tests -v) termforge_add_python_test(differential -m unittest discover -s experiments/differential -p test_*.py -v) diff --git a/experiments/candidates/alacritty/Cargo.toml b/experiments/candidates/alacritty/Cargo.toml index e596f796..8160830e 100644 --- a/experiments/candidates/alacritty/Cargo.toml +++ b/experiments/candidates/alacritty/Cargo.toml @@ -7,6 +7,12 @@ publish = false [workspace] members = ["vendor/alacritty_terminal"] +[features] +# Preserve historical diagnostics by default. A separate --no-default-features +# build of alacritty-measure removes the allocator wrapper at compile time. +default = ["measure-allocation-counting"] +measure-allocation-counting = [] + [dependencies] alacritty_terminal = { path = "vendor/alacritty_terminal", default-features = false } serde_json = "1" diff --git a/experiments/candidates/alacritty/src/measure.rs b/experiments/candidates/alacritty/src/measure.rs index 0c94bed8..89fe5759 100644 --- a/experiments/candidates/alacritty/src/measure.rs +++ b/experiments/candidates/alacritty/src/measure.rs @@ -2,9 +2,7 @@ #[path = "main.rs"] mod trace; -use std::alloc::{GlobalAlloc, Layout, System}; use std::env; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Instant; use alacritty_terminal::term::{Config, Term}; @@ -12,27 +10,58 @@ use alacritty_terminal::vte::ansi::Processor; use serde_json::{Value, json}; use trace::{Listener, Size}; -struct CountingAllocator; -static OBSERVE: AtomicBool = AtomicBool::new(false); -static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); -static ALLOCATED_BYTES: AtomicUsize = AtomicUsize::new(0); +#[cfg(feature = "measure-allocation-counting")] +mod allocation { + use std::alloc::{GlobalAlloc, Layout, System}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -unsafe impl GlobalAlloc for CountingAllocator { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - if OBSERVE.load(Ordering::Relaxed) { - ALLOCATIONS.fetch_add(1, Ordering::Relaxed); - ALLOCATED_BYTES.fetch_add(layout.size(), Ordering::Relaxed); + struct CountingAllocator; + static OBSERVE: AtomicBool = AtomicBool::new(false); + static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); + static ALLOCATED_BYTES: AtomicUsize = AtomicUsize::new(0); + + unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + if OBSERVE.load(Ordering::Relaxed) { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(layout.size(), Ordering::Relaxed); + } + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) } } - unsafe { System.alloc(layout) } } - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { System.dealloc(ptr, layout) } + #[global_allocator] + static ALLOCATOR: CountingAllocator = CountingAllocator; + + pub fn start(observe: bool) { + ALLOCATIONS.store(0, Ordering::Relaxed); + ALLOCATED_BYTES.store(0, Ordering::Relaxed); + OBSERVE.store(observe, Ordering::Relaxed); + } + + pub fn stop() { OBSERVE.store(false, Ordering::Relaxed); } + + pub fn counts(observe: bool) -> (Option, Option) { + if observe { + (Some(ALLOCATIONS.load(Ordering::Relaxed)), Some(ALLOCATED_BYTES.load(Ordering::Relaxed))) + } else { (None, None) } } } -#[global_allocator] -static ALLOCATOR: CountingAllocator = CountingAllocator; +#[cfg(not(feature = "measure-allocation-counting"))] +mod allocation { + // No #[global_allocator], atomic counters or allocator forwarding path is + // compiled in this mode. Projection remains independently selectable. + pub fn start(_: bool) {} + pub fn stop() {} + pub fn counts(_: bool) -> (Option, Option) { (None, None) } +} + +fn allocator_wrapper_compiled() -> bool { cfg!(feature = "measure-allocation-counting") } #[cfg(windows)] mod platform { @@ -201,6 +230,15 @@ fn duration_arg(value: &str) -> Result { fn run() -> Result<(), String> { let args: Vec = env::args().collect(); + if args.len() == 2 && args[1] == "--identity" { + println!("{}", json!({ + "schema": "termforge-core-cost-build-v1", + "allocator_wrapper_compiled": allocator_wrapper_compiled(), + "timing_gates_compiled": true, + "projection_selectable": true, + })); + return Ok(()); + } if args.len() != 5 || (args[1] != "project" && args[1] != "interactive") { return Err("alacritty-measure project|interactive chunk observe(0|1) min_cpu_seconds".into()); } @@ -235,9 +273,7 @@ fn run() -> Result<(), String> { } drop(warmup); let preparation = oracle_started.elapsed().as_secs_f64(); - ALLOCATIONS.store(0, Ordering::Relaxed); - ALLOCATED_BYTES.store(0, Ordering::Relaxed); - OBSERVE.store(observe, Ordering::Relaxed); + allocation::start(observe); let mut term: Option> = None; let interval = measure(duration, || { term = Some(new_term()); @@ -254,15 +290,14 @@ fn run() -> Result<(), String> { } Ok(()) })?; - OBSERVE.store(false, Ordering::Relaxed); + allocation::stop(); let checks = Instant::now(); let (final_hash, final_value) = projection(term.as_mut().unwrap(), &listener)?; if final_hash != expected_hash || !correct(&final_value, route) { return Err("final state oracle failed".into()); } let oracle_seconds = checks.elapsed().as_secs_f64(); - let allocations = ALLOCATIONS.load(Ordering::Relaxed); - let allocated_bytes = ALLOCATED_BYTES.load(Ordering::Relaxed); + let (allocations, allocated_bytes) = allocation::counts(observe); let peak = platform::peak_working_set()?; let result = json!({ "schema_version": 2, @@ -287,9 +322,10 @@ fn run() -> Result<(), String> { "final_oracle_seconds": oracle_seconds, "oracle_scope": if observe { "warmup and final; every measured projection also hashed" } else { "warmup and final iteration" }, "state_fnv1a64": expected_hash.to_string(), - "ordinary_allocations": if observe { Some(allocations) } else { None }, - "ordinary_allocated_bytes": if observe { Some(allocated_bytes) } else { None }, - "allocation_scope": "Rust global allocator requests; not equivalent to C++ new or live memory", + "ordinary_allocations": allocations, + "ordinary_allocated_bytes": allocated_bytes, + "allocator_wrapper_compiled": allocator_wrapper_compiled(), + "allocation_scope": if allocator_wrapper_compiled() { "Rust global allocator requests; not equivalent to C++ new or live memory" } else { "allocator wrapper absent at compile time; allocation metrics unknown" }, "peak_working_set_bytes": peak, "cpu_interval_differences": 1, "counted_parser_destructions": interval.iterations - 1, @@ -303,7 +339,7 @@ fn run() -> Result<(), String> { fn main() { if let Err(error) = run() { - OBSERVE.store(false, Ordering::Relaxed); + allocation::stop(); eprintln!("alacritty measure failed: {error}"); std::process::exit(2); } diff --git a/experiments/measure/R54-COST.md b/experiments/measure/R54-COST.md new file mode 100644 index 00000000..f4a16c0b --- /dev/null +++ b/experiments/measure/R54-COST.md @@ -0,0 +1,62 @@ +# R2 流式字素成本：测量前合同 v1 + +本合同是 [#54](https://github.com/JavaZeroo/termforge/issues/54) 的测量准备，不是性能结果。机器可读来源为 [r54_contract.py](r54_contract.py)；执行前生成独立计划，固定其提交、`protocol_sha256` 和 harness 文件哈希。计划故意保留 `builds=null`、`environment_evidence=null`：源码身份和二进制哈希不等于已证明二进制来自该源码。实际构建、环境与运行记录必须另附，不能以计划生成成功宣布 #54 完成。 + +## 固定比较对象 + +| 对象 | 核心源码 | trace 全对象摘要：project / interactive | +|---|---|---| +| 修前 `.3` | `65988f5f20f62294ff52fcac5b5b81e563f93f3b` | `11966868740884721388` / `8035857386658397897` | +| 修后 `.4` | `3d6d44d3b3e9a9a3e41da98e8711e9f19a211fd8` | `16178973312633948535` / `6154007251599136798` | + +两份固定核心使用**同一份已审核的测量 harness**；只把本合同提交的根 `Cargo.toml` 与 `src/measure.rs` 应用到相应候选目录，记录覆盖前后文件哈希与 diff。保留各自原有 `Cargo.lock`、`src/main.rs` 和整个 vendor 树；`.3` 没有新增计数器字段，因此全对象摘要不同，不能统一成一个摘要或略掉新字段。若覆盖产生这些文件之外的变化，停止并审查新配置，不把它算进 v1。 + +每份用 `cargo build --release --locked --bin alacritty-measure --no-default-features`，独立 target 目录；记录完整命令、退出、编译器/链接器和 Rust target/toolchain、构建环境变量、锁文件、完整源码树身份及 EXE SHA-256。记录 `--identity` 的实际 JSON，但该 JSON 只说明编译配置，不证明源码与二进制绑定。构建步骤的记录和独立哈希审计仍不可省。该对照是 headless `Term/Processor` 层，系统/官方 ConPTY 后端不会进入这些成本数值。 + +## allocator 和观察端点 + +`measure-allocation-counting` 是默认 Cargo feature，原命令继续编译历史 Rust global allocator 包装。`observe=0` 只关闭计数和逐轮投影，仍不等于无包装。新增 `--no-default-features` 模式在编译期完全排除 `#[global_allocator]`、atomic 计数器和转发路径；`--identity` 与每个结果都给出 `allocator_wrapper_compiled=false`。该模式的 `observe=1` 仍执行原本的每轮快照/JSON/完整哈希检查，但分配次数/字节均为 null。标准 allocator 的内部开销仍存在。 + +两个模式都保留区间计时门槛，所以 `counter_off_is_uninstrumented=false` 不改成 true。未执行真正产品级无测量探针对照前，不称为整体“无插桩”。C++ `core_measure` 的替换 new 仍在；本合同没有移除它，也不将 C++ 普通 new 和 Rust global allocator 请求视为等价指标。 + +为定位包装开销，另做同一固定核心上的 wrapper 开/关 × projection 开/关对照，记录独立二进制身份、次序、全部原始成本与正确性。该诊断不与主 A/B 样本合并，也不从主结果扣掉估算开销。 + +## 负载、区间与顺序 + +- project 为 1,000 次 `76*x + CR/LF`，78,000B；分块 512/4096/65536B。 +- interactive 为 1,000 次 `abcdefgh`、三次左移、`Z`、退格、`Q`、CR/LF，22,000B；分块 1/16/64/4096/22000B。4096 保留原 12 样本的诊断粒度，其他边界继续覆盖。 +- 两者屏幕 24×80、历史 100 行。计划写出完整字节 SHA-256；ASCII 成本负载不代表完整 Unicode 工作流。 +- 每个样本至少 2 秒 CPU 且至少 2 秒墙钟，一个连续区间；Windows CPU 为 `GetProcessTimes` kernel+user 的单次结束减开始，Unix 为 `getrusage RUSAGE_SELF` user+system。保留 8 轮墙钟检查与至少约 250ms CPU 查询间距，内部 60 秒、子进程 90 秒期限不放宽。 +- 区间含每轮构造、上一轮析构、feed/状态和门槛；输入准备、预热、最终 oracle、结果序列化与最后一次析构在区间外。预热和最终完整状态检查必须成功，中间轮次不逐轮检查。 +- 主 A/B 固定 `observe=0` 且无 allocator 包装；两批独立开始的种子为 20261026 / 20261027，每个 route/chunk 各 7 对，单批 112 样本。组顺序冻结为计划中的完整数组；同组两个源码相邻，每轮交替 AB/BA，不根据结果改变次序。 +- 112 来自 8 种块配置 × 7 对 × 2 源码，单批最低约 224 秒 CPU，而非长时间 soak。保留两次独立批次用于暴露宿主漂移，7 对便于同时报告配对中位数和离散度；这些选择不保证统计显著性。全块配置防止只在一个 4096B 粒度上推导普遍成本，本合同不自动展开更多负载或无限重跑。 +- 记录 OS/build、CPU、电源策略、toolchain、flags、实际二进制、背景 CPU 与已知并行任务。构建和其他测试结束后才能启动正式批次；交错顺序不能证明宿主空闲，共享宿主始终注明限制。不同宿主/电源/编译环境不能无说明拼成同一批。 + +## 正确性、统计和预算程序 + +主统计为**同 repeat/route/chunk 配对的 candidate/baseline CPU ns/B 比值的中位数**；同时列出全部配对比值、每组样本 CPU 与墙钟 ns/B 的中位数、最小/最大值和中位绝对偏差。两批、各块大小分别报告；不混合 workload/块大小形成一个总分，也不以比值的中位数冒充两个独立中位数之比。样本量小，不自动生成有保证的置信区间或“显著”结论。 + +任何非零退出、超时、错误状态/负载/编译身份或缺失端点字段都保留原 stdout/stderr 与当前 manifest，将批次标失败并停止。不原地重跑、丢弃异常值、补替代样本或只保留最快批次。噪声或两批方向不一致时保留不确定性；下一次调查有新身份和明确假设，不能偷偷替换原批次。 + +**v1 尚无数值回归预算，`performance_gate=not_evaluated`。** 先用受控基线和开销对照了解成本/噪声，在单独可审查记录中说明用户交互与正确性理由、环境、可接受 project/interactive 限值及裁决规则，然后冻结预算；之后才能评估拟提交优化。不能看到优化成绩后把阈值改成刚好通过。#34 的产品决定仍需要真实 GUI/输入/并发/长期资源证据，本 headless 合同不能单独代替。 + +若优化 segmentation、宽度、Cell 保留、投影或分配，先有同一输入上可证伪的成本归因对照，不能只凭总吞吐推断某一层；完整控制边界、Unicode 17 语料、全部输入分块、实际 glyph、原文选择/复制及取消边界继续回归。接收/输入延迟/实际呈现端点仍为 null，不声明 WindTerm 等价。 + +原 [12 样本](../shell/results/2026-10-26-streaming-graphemes/cost/) 及其二进制/源码身份保持原样；共享宿主、allocator 包装和旧顺序都不同，只作为历史诊断，不能升格、覆盖或与 v1 合并统计。 + +## 自动入口和当前缺口 + +```powershell +# 不执行构建或负载；输出文件必须不存在，父目录先准备好。 +python -m experiments.measure.r54_contract --out local/r54-plan-v1.json + +# 纯计划、oracle、身份和失败负例检查；不占用正式计时环境。 +python -m unittest experiments.measure.test_r54_contract.CostPlanTests -v + +# 两个实际构建完成后才设置本进程环境，执行短 sanity；不是正式样本。 +$env:TERMFORGE_R54_WRAPPED = '' +$env:TERMFORGE_R54_UNWRAPPED = '' +python -m unittest experiments.measure.test_r54_contract.OptionalAllocatorBuildSmoke -v +``` + +该批只交付准备合同与可完全移除包装的构建机制。两个实际 build manifest/EXE、真实空闲环境证据、冻结计划执行器和重复样本、成本归因、数值预算与优化验收仍由 #54 接续；生成计划或短 sanity 都不关闭该 issue。 diff --git a/experiments/measure/README.md b/experiments/measure/README.md index c2122c86..8d474d53 100644 --- a/experiments/measure/README.md +++ b/experiments/measure/README.md @@ -2,6 +2,8 @@ `core_measure` 使用原创固定输入与独立状态检查。v2 修正了 v1 将很多短轮次 CPU 差值相加的计量问题；`compare.py` 另提供自研/候选 project 与 interactive 成本诊断。解码、屏幕更新、历史仍未完整独立归因，不能据此判断 WindTerm 性能差距。R2 流式字素成本的受控定位与预算由 [#54](https://github.com/JavaZeroo/termforge/issues/54)承接，完整产品性能和跨目标基线仍按 [R4 工作包 #9](https://github.com/JavaZeroo/termforge/issues/9)验收。 +#54 的[测量前合同 v1](R54-COST.md)与 `r54_contract.py` 固定 `.3`/`.4` 源码、负载、相邻配对顺序、统计和预算决定程序；新增 Rust `--no-default-features` 测量构建可在编译期排除 allocator 包装。该准备仍缺实际两构建身份、受控重复测量和数值预算，不改变下述历史结果或性能边界。 + ```powershell # 先按 CONTRIBUTING.md 从仓库根目录构建。输出目录必须不存在。 python experiments/measure/run.py --bin-dir build/bin/Release --out local/experiments/core-cost-v2-a --cpu-seconds 2 --repeats 3 --seed 20261023 diff --git a/experiments/measure/r54_contract.py b/experiments/measure/r54_contract.py new file mode 100644 index 00000000..d436f113 --- /dev/null +++ b/experiments/measure/r54_contract.py @@ -0,0 +1,141 @@ +"""Prepare the pre-measurement R54 plan; preparation never passes a speed gate.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import random + +from experiments.measure.compare import WORKLOADS +from experiments.measure.run import validate_result + +ROOT = Path(__file__).resolve().parents[2] +SCHEMA = "termforge-r54-cost-plan-v1" +SOURCES = { + "baseline": dict(commit="65988f5f20f62294ff52fcac5b5b81e563f93f3b", version="0.26.1-dev+termforge.3", + states=dict(project="11966868740884721388", interactive="8035857386658397897")), + "candidate": dict(commit="3d6d44d3b3e9a9a3e41da98e8711e9f19a211fd8", version="0.26.1-dev+termforge.4", + states={route: value["states"]["alacritty"] for route, value in WORKLOADS.items()}), +} +CHUNKS = dict(project=(512, 4096, 65536), interactive=(1, 16, 64, 4096, 22000)) +SEEDS = (20261026, 20261027) +REPEATS = 7 +CPU_SECONDS = 2.0 + + +def canonical(value): + return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def sample_order(seed): + """Adjacent source pairs limit drift; alternate AB/BA with frozen randomness.""" + rng = random.Random(seed) + result = [] + configs = [(route, chunk) for route, chunks in CHUNKS.items() for chunk in chunks] + first = {(route, chunk): rng.randrange(2) for route, chunk in configs} + for repeat in range(REPEATS): + order = list(configs) + rng.shuffle(order) + for route, chunk in order: + variants = ("baseline", "candidate") + if (first[(route, chunk)] + repeat) % 2: + variants = tuple(reversed(variants)) + for variant in variants: + result.append(dict(sequence=len(result), repeat=repeat, route=route, + chunk=chunk, variant=variant, observe=0)) + return result + + +def protocol(): + return dict( + schema=SCHEMA, sources=SOURCES, + workloads={route: dict(bytes=len(value["fixture"]), + sha256=hashlib.sha256(value["fixture"]).hexdigest(), + rows=24, columns=80, history=100, chunks=list(CHUNKS[route])) + for route, value in WORKLOADS.items()}, + build=dict(profile="release", cargo_locked=True, default_features=False, + allocator_wrapper_compiled=False, same_measurement_harness_required=True, + separate_target_directories_required=True), + timing=dict(cpu_floor_seconds=CPU_SECONDS, wall_floor_seconds=CPU_SECONDS, + independent_batch_seeds=list(SEEDS), repeats_per_group=REPEATS, + wall_gate_every_iterations=8, cpu_poll_min_wall_seconds=0.25, + interval_wall_budget_seconds=60, child_timeout_seconds=90, + endpoint="construction + previous destruction + feed + timing gates; warmup, final oracle, JSON and final destruction excluded", + cpu_windows="GetProcessTimes kernel plus user; one end-minus-start difference", + cpu_unix="getrusage RUSAGE_SELF user plus system; one end-minus-start difference"), + schedules=[dict(seed=seed, samples=sample_order(seed)) for seed in SEEDS], + correctness=dict(warmup=True, final_iteration=True, intermediate_iterations=False, + full_projection_digest_per_source=SOURCES, + separate_unicode_all_chunks_glyph_selection_regression_required=True), + statistics=dict(primary="median of paired candidate/baseline CPU ns per byte, each route/chunk/batch separately", + report=["all samples", "paired ratios", "median", "min", "max", "median absolute deviation", "wall ns per byte"], + combine_chunks=False, combine_batches=False, + failure_policy="preserve and stop the batch; no retries, outlier removal or replacement samples", + unstable_policy="retain uncertainty and diagnose; a new batch needs a new identity"), + budget=dict(status="not_set", numerical_regression_limit=None, + decision="review controlled costs and noise, document user/correctness rationale and freeze numeric limits before evaluating any optimization; never backfill a threshold to pass existing results", + performance_gate="not_evaluated"), + endpoints=dict(receive_seconds=None, input_latency_seconds=None, presented_done=None), + environment=dict(idle_status="unverified", dedicated_host=False, + required_record=["OS/build", "CPU", "power policy", "toolchain", "compiler flags", "source trees", "lockfiles", "binary SHA256", "background CPU evidence", "known concurrent work"], + simultaneous_build_or_test_forbidden=True), + allocation=dict(counter_off_is_uninstrumented=False, + absent_wrapper_is_not_absent_timing_instrumentation=True, + metrics_unknown=True, + overhead_diagnostic="same fixed core, wrapped/unwrapped factorial with identical projection switch; report raw costs, do not subtract from primary samples"), + original_diagnostic=dict(preserved=True, samples=12, + path="experiments/shell/results/2026-10-26-streaming-graphemes/cost", + comparable_to_this_protocol=False), + ) + + +def expected_sample(item): + return dict(kind="core_cost", route=item["route"], chunk=item["chunk"], observe=0, + bytes_per_round=len(WORKLOADS[item["route"]]["fixture"]), + state_fnv1a64=SOURCES[item["variant"]]["states"][item["route"]], + allocator_wrapper_compiled=False) + + +def validate_sample(result, item): + validate_result(result, CPU_SECONDS, expected_sample(item)) + if result["wall_gate_every_iterations"] != 8 or result["cpu_gate_min_wall_spacing_seconds"] != 0.25: + raise ValueError("timing gate contract mismatch") + if result["interval_wall_seconds"] >= 60: + raise ValueError("internal wall budget exceeded") + if result["cpu_clock"] not in ("GetProcessTimes kernel plus user", "getrusage RUSAGE_SELF user plus system"): + raise ValueError("CPU clock contract mismatch") + if result["wall_clock"] != "std::time::Instant": + raise ValueError("wall clock contract mismatch") + + +def prepare(output): + value = protocol() + result = dict(protocol=value, protocol_sha256=hashlib.sha256(canonical(value)).hexdigest(), + prepared_utc=datetime.now(timezone.utc).isoformat(), + phase="prepared_not_built_or_measured", performance_gate="not_evaluated", + builds={name: None for name in SOURCES}, environment_evidence=None, + harness_sha256={name: hashlib.sha256((ROOT / name).read_bytes()).hexdigest() + for name in ("experiments/candidates/alacritty/Cargo.toml", + "experiments/candidates/alacritty/src/measure.rs", + "experiments/measure/run.py", + "experiments/measure/r54_contract.py")}) + # Exclusive creation preserves already frozen plans; fresh timing results and + # build manifests must be separate artifacts rather than rewriting this file. + with Path(output).open("x", encoding="utf-8") as stream: + stream.write(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + value = prepare(args.out) + print(json.dumps({"phase": value["phase"], "protocol_sha256": value["protocol_sha256"], + "performance_gate": value["performance_gate"]})) + + +if __name__ == "__main__": + main() diff --git a/experiments/measure/results/2026-10-26-r54-preparation/validation.json b/experiments/measure/results/2026-10-26-r54-preparation/validation.json new file mode 100644 index 00000000..047263dc --- /dev/null +++ b/experiments/measure/results/2026-10-26-r54-preparation/validation.json @@ -0,0 +1,119 @@ +{ + "schema": "termforge-r54-preparation-validation-v1", + "source_base_commit": "3d6d44d3b3e9a9a3e41da98e8711e9f19a211fd8", + "source_identity": "source_sha256 records actual tested checkout bytes; committed Git blobs can differ only by checkout newline normalization", + "source_sha256": { + ".github/workflows/core-candidate.yml": "c758f8acc205b9bb0f5bca78e0505242c4695195425063ea96c6394d6b5c244e", + "CMakeLists.txt": "185832b2b503091d9e1a9538e120e27fa8d8bcf8c8a6804f062b46a57868d325", + "experiments/candidates/alacritty/Cargo.toml": "2b4be12e31053e65dabca8ca9c234a59b73ab2ab3e1520a6092c7dc94e12ffd7", + "experiments/candidates/alacritty/Cargo.lock": "896c6e60909d55d61fb26d277b1a98daf9245e8aee811caff37fc3e9d06ca627", + "experiments/candidates/alacritty/src/measure.rs": "701aecfd5bbd951434f7b8278b9efb700e7644c9eccfd19cec1b36aa6aeb3f8a", + "experiments/measure/run.py": "5643194c95c710790bee3a10e825cd338626222e88a9e2af194ad905b0718852", + "experiments/measure/r54_contract.py": "08c744f3d78e23439fecd8cc01348fdc1673c5397e16738266774fb49fbc76fa", + "experiments/measure/test_r54_contract.py": "7b242d92f6cb7d7c97d1174a1c21e25c0682fb576baa4b115d7162a597b1a95d", + "experiments/measure/R54-COST.md": "af52b1d7e19cee2d7f5d3da3a9503e7bf372ff32d5584d2d0408ade9edc0d289", + "experiments/measure/README.md": "6b898e0d938c7aba4c638ecee3e875a0eafca7a0001219c5c812d7c2938a16ba" + }, + "core_version": "0.26.1-dev+termforge.4", + "builds": { + "wrapped": { + "binary_sha256": "8c7ef2b3ea1d56544536e9982c8fb3dfd8871584b45b750d46613ac298f1c95c", + "identity": { + "allocator_wrapper_compiled": true, + "projection_selectable": true, + "schema": "termforge-core-cost-build-v1", + "timing_gates_compiled": true + }, + "command": [ + "cargo", + "build", + "--manifest-path", + "experiments/candidates/alacritty/Cargo.toml", + "--release", + "--locked", + "--offline", + "--jobs", + "1", + "--bin", + "alacritty-measure", + "--target-dir", + "local/r54-build/wrapped" + ], + "build_log_sha256": "1b5e7d536cd105428e67ac6c7f6ada1896bfa819ddf896481c4be3046b23b4d3" + }, + "unwrapped": { + "binary_sha256": "8934c3150419c4a73b9adb6c434d3598a48bb8697747978611665af87b22f964", + "identity": { + "allocator_wrapper_compiled": false, + "projection_selectable": true, + "schema": "termforge-core-cost-build-v1", + "timing_gates_compiled": true + }, + "command": [ + "cargo", + "build", + "--manifest-path", + "experiments/candidates/alacritty/Cargo.toml", + "--release", + "--locked", + "--offline", + "--jobs", + "1", + "--bin", + "alacritty-measure", + "--no-default-features", + "--target-dir", + "local/r54-build/unwrapped" + ], + "build_log_sha256": "83d4fc1709141411d96f63a1d9f72c48c70d445cb89f46bfe0e49572f1d92587" + } + }, + "test_scope": { + "pure_tests": 12, + "actual_build_mode_test": 1, + "real_measurement_calls": 8, + "routes": [ + "project", + "interactive" + ], + "chunk": 4096, + "observes": [ + 0, + 1 + ], + "cpu_floor_seconds": 0.1, + "failures": 0, + "skipped": 0, + "checks": [ + "both compile identities", + "warmup/final fixed full state", + "projection independent of allocator", + "null allocations in absent-wrapper mode", + "positive allocation counts only in wrapped observed mode", + "legacy and strict validator negatives", + "exclusive plan creation", + "failure evidence preservation" + ] + }, + "test_log_sha256": "e1fb57272bbcbbdb820c3a89e0bb9dff52a59650b14c4213f239d152f8005a19", + "rustc": [ + "rustc 1.92.0 (ded5c06cf 2025-12-08)", + "binary: rustc", + "commit-hash: ded5c06cf21d2b93bffd5d884aa6e96934ee4234", + "commit-date: 2025-12-08", + "host: x86_64-pc-windows-msvc", + "release: 1.92.0", + "LLVM version: 21.1.3" + ], + "cargo": "cargo 1.92.0 (344c4567c 2025-10-21)", + "limitations": [ + "Host shared with root three-configuration CTest; no idle-host claim", + "Only .4 wrapper-on/off build sanity; pinned .3/.4 source-to-binary pair build manifests still pending", + "No formal performance batches or cost attribution", + "No numeric regression budget", + "Rust timing gates remain compiled; not a product-level uninstrumented baseline", + "No transport/input/presentation or product workflow acceptance" + ], + "original_12_cost_samples_changed": false, + "performance_gate": "not_evaluated" +} diff --git a/experiments/measure/run.py b/experiments/measure/run.py index 2549414b..a8350b1f 100644 --- a/experiments/measure/run.py +++ b/experiments/measure/run.py @@ -52,6 +52,14 @@ def validate_result(result, floor, expected): raise ValueError("mismatched requested CPU floor") if result.get("counter_off_is_uninstrumented") is not False: raise ValueError("counter-off provenance missing") + # Historical binaries lack this field and remain wrapped diagnostics. + # Requiring an absent wrapper is opt-in and must be reported explicitly by + # the binary; observe=0 alone never satisfies that requirement. + wrapper = expected.get("allocator_wrapper_compiled", True) + reported_wrapper = result.get("allocator_wrapper_compiled", True) + if (("allocator_wrapper_compiled" in expected and "allocator_wrapper_compiled" not in result) or + reported_wrapper is not wrapper): + raise ValueError("allocator wrapper build identity mismatch") if result["kind"] == "calibration": if expected["kind"] != "calibration": raise ValueError("unexpected calibration response") @@ -81,11 +89,11 @@ def validate_result(result, floor, expected): raise ValueError("unexpected continuous interval endpoint") if result["receive_seconds"] is not None or result["presented_done"] is not None: raise ValueError("unmeasured endpoints must remain unknown") - if result["observe"]: + if result["observe"] and wrapper: if result["ordinary_allocations"] <= 0 or result["ordinary_allocated_bytes"] <= 0: raise ValueError("allocation observer did not record any work") elif result["ordinary_allocations"] is not None or result["ordinary_allocated_bytes"] is not None: - raise ValueError("counter-off counts should be unknown") + raise ValueError("allocation metrics must be unknown for this observer/build") def execute(command, floor, expected, timeout=90): diff --git a/experiments/measure/test_r54_contract.py b/experiments/measure/test_r54_contract.py new file mode 100644 index 00000000..757ac0a3 --- /dev/null +++ b/experiments/measure/test_r54_contract.py @@ -0,0 +1,139 @@ +"""Pure R54 schedule/provenance/oracle guards; these do not run performance loads.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest + +from experiments.measure.r54_contract import ( + CHUNKS, CPU_SECONDS, REPEATS, SEEDS, SOURCES, canonical, expected_sample, + prepare, protocol, sample_order, validate_sample, +) +from experiments.measure.run import run_command, validate_result + + +def sample(observe=False, wrapped=False): + return dict(schema_version=2, kind="core_cost", valid=True, cpu_floor_seconds=CPU_SECONDS, + counter_off_is_uninstrumented=False, route="project", chunk=4096, observe=observe, + rounds=8, iterations=8, bytes_per_round=78000, + interval_cpu_seconds=2.1, interval_wall_seconds=2.2, cpu_gate_polls=8, + cpu_interval_differences=1, counted_parser_destructions=7, + state_fnv1a64=SOURCES["candidate"]["states"]["project"], + receive_seconds=None, presented_done=None, + ordinary_allocations=100 if observe and wrapped else None, + ordinary_allocated_bytes=200 if observe and wrapped else None, + allocator_wrapper_compiled=wrapped, + cpu_clock="GetProcessTimes kernel plus user", wall_clock="std::time::Instant", + wall_gate_every_iterations=8, cpu_gate_min_wall_spacing_seconds=0.25) + + +class CostPlanTests(unittest.TestCase): + def test_complete_adjacent_source_pairs_and_alternating_order(self): + for seed in SEEDS: + order = sample_order(seed) + self.assertEqual(order, sample_order(seed)) + self.assertEqual(len(order), sum(map(len, CHUNKS.values())) * REPEATS * 2) + groups = {} + for index in range(0, len(order), 2): + first, second = order[index:index + 2] + self.assertEqual(first["sequence"], index) + self.assertEqual(second["sequence"], index + 1) + self.assertEqual((first["route"], first["chunk"], first["repeat"]), + (second["route"], second["chunk"], second["repeat"])) + self.assertEqual({first["variant"], second["variant"]}, set(SOURCES)) + groups.setdefault((first["route"], first["chunk"]), []).append(first["variant"]) + self.assertEqual(len(groups), sum(map(len, CHUNKS.values()))) + for values in groups.values(): + self.assertEqual(len(values), REPEATS) + self.assertTrue(all(a != b for a, b in zip(values, values[1:]))) + self.assertNotEqual(sample_order(SEEDS[0]), sample_order(SEEDS[1])) + + def test_workload_bytes_and_full_state_identities_remain_fixed(self): + p = protocol() + fixtures = dict(project=(b"x" * 76 + b"\r\n") * 1000, + interactive=b"abcdefgh\x1b[D\x1b[D\x1b[DZ\x08Q\r\n" * 1000) + for route, payload in fixtures.items(): + self.assertEqual(p["workloads"][route]["bytes"], len(payload)) + self.assertEqual(p["workloads"][route]["sha256"], hashlib.sha256(payload).hexdigest()) + self.assertNotEqual(SOURCES["baseline"]["states"][route], SOURCES["candidate"]["states"][route]) + self.assertIsNone(p["budget"]["numerical_regression_limit"]) + self.assertEqual(p["budget"]["performance_gate"], "not_evaluated") + self.assertFalse(p["original_diagnostic"]["comparable_to_this_protocol"]) + + def test_prepare_is_exclusive_and_reports_unmeasured_identity(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "plan.json" + value = prepare(path) + self.assertEqual(json.loads(path.read_text(encoding="utf-8")), value) + self.assertEqual(value["protocol_sha256"], hashlib.sha256(canonical(value["protocol"])).hexdigest()) + self.assertEqual(value["phase"], "prepared_not_built_or_measured") + self.assertEqual(value["builds"], dict(baseline=None, candidate=None)) + self.assertIsNone(value["environment_evidence"]) + with self.assertRaises(FileExistsError): + prepare(path) + + def test_absent_allocator_must_be_explicit_and_does_not_claim_no_instrumentation(self): + item = dict(variant="candidate", route="project", chunk=4096) + validate_sample(sample(), item) + for field, value in (("allocator_wrapper_compiled", True), ("allocator_wrapper_compiled", None), + ("counter_off_is_uninstrumented", True), ("state_fnv1a64", "wrong"), + ("interval_cpu_seconds", .1), ("interval_wall_seconds", float("nan")), + ("interval_wall_seconds", 60), ("ordinary_allocations", 0), + ("bytes_per_round", 77999), ("wall_gate_every_iterations", 1), + ("cpu_gate_min_wall_spacing_seconds", .01), ("cpu_clock", "unknown"), + ("wall_clock", "unknown")): + with self.subTest(field=field, value=value): + broken = sample() + broken[field] = value + with self.assertRaises(ValueError): + validate_sample(broken, item) + + def test_projection_switch_is_independent_of_allocator_build(self): + for wrapped in (False, True): + result = sample(observe=True, wrapped=wrapped) + expected = expected_sample(dict(variant="candidate", route="project", chunk=4096)) + expected.update(observe=1, allocator_wrapper_compiled=wrapped) + validate_result(result, CPU_SECONDS, expected) + if wrapped: + result["ordinary_allocations"] = None + with self.assertRaises((ValueError, TypeError)): + validate_result(result, CPU_SECONDS, expected) + + def test_legacy_probes_remain_wrapped_and_absent_provenance_is_not_accepted(self): + result = sample() + del result["allocator_wrapper_compiled"] + expected = expected_sample(dict(variant="candidate", route="project", chunk=4096)) + with self.assertRaises(ValueError): + validate_result(result, CPU_SECONDS, expected) + del expected["allocator_wrapper_compiled"] + validate_result(result, CPU_SECONDS, expected) + result["allocator_wrapper_compiled"] = False + with self.assertRaises(ValueError): + validate_result(result, CPU_SECONDS, expected) + + +class OptionalAllocatorBuildSmoke(unittest.TestCase): + """Opt-in sanity on two actually built modes; not part of pure plan checks.""" + def test_default_and_absent_wrapper_identity(self): + paths = [os.environ.get(name) for name in ("TERMFORGE_R54_WRAPPED", "TERMFORGE_R54_UNWRAPPED")] + if not all(paths): + self.skipTest("requires explicit default-feature and no-default-feature measurement binaries") + for path, wrapped in zip(paths, (True, False)): + response = run_command([path, "--identity"], timeout=10) + self.assertEqual(response["exit_code"], 0, response) + identity = json.loads(response["stdout"]) + self.assertIs(identity["allocator_wrapper_compiled"], wrapped) + self.assertTrue(identity["timing_gates_compiled"]) + for route in ("project", "interactive"): + for observe in (0, 1): + response = run_command([path, route, "4096", str(observe), ".1"], timeout=90) + self.assertEqual(response["exit_code"], 0, response) + expected = expected_sample(dict(variant="candidate", route=route, chunk=4096)) + expected.update(observe=observe, allocator_wrapper_compiled=wrapped) + validate_result(json.loads(response["stdout"]), .1, expected) + + +if __name__ == "__main__": + unittest.main() From 3e2ba43ffb4f6601a18b1e2d742781c1abf43272 Mon Sep 17 00:00:00 2001 From: JavaZeroo <71128095+JavaZeroo@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:17:04 +0800 Subject: [PATCH 2/3] ci: resolve measurement executable paths from the job matrix --- .github/workflows/core-candidate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core-candidate.yml b/.github/workflows/core-candidate.yml index 56091469..0b8d962c 100644 --- a/.github/workflows/core-candidate.yml +++ b/.github/workflows/core-candidate.yml @@ -46,8 +46,8 @@ jobs: PYTHONUTF8: '1' TERMFORGE_ALACRITTY_TRACE: ${{ github.workspace }}/${{ matrix.binary }} TERMFORGE_TRACE: ${{ github.workspace }}/${{ matrix.core_binary }} - TERMFORGE_R54_WRAPPED: ${{ github.workspace }}/experiments/candidates/alacritty/target/release/alacritty-measure${{ runner.os == 'Windows' && '.exe' || '' }} - TERMFORGE_R54_UNWRAPPED: ${{ github.workspace }}/build/measure-no-allocator/release/alacritty-measure${{ runner.os == 'Windows' && '.exe' || '' }} + TERMFORGE_R54_WRAPPED: ${{ github.workspace }}/experiments/candidates/alacritty/target/release/alacritty-measure${{ matrix.os == 'windows-2025' && '.exe' || '' }} + TERMFORGE_R54_UNWRAPPED: ${{ github.workspace }}/build/measure-no-allocator/release/alacritty-measure${{ matrix.os == 'windows-2025' && '.exe' || '' }} TERMFORGE_TEST_ARTIFACTS: ${{ github.workspace }}/build/test-artifacts steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From 1937a07f49410e1194f6b59f7ec0204e0648de77 Mon Sep 17 00:00:00 2001 From: JavaZeroo <71128095+JavaZeroo@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:18:13 +0800 Subject: [PATCH 3/3] docs: retain the first workflow failure and separate its correction --- .../2026-10-26-r54-preparation/validation.json | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/experiments/measure/results/2026-10-26-r54-preparation/validation.json b/experiments/measure/results/2026-10-26-r54-preparation/validation.json index 047263dc..34c393eb 100644 --- a/experiments/measure/results/2026-10-26-r54-preparation/validation.json +++ b/experiments/measure/results/2026-10-26-r54-preparation/validation.json @@ -1,7 +1,7 @@ { "schema": "termforge-r54-preparation-validation-v1", "source_base_commit": "3d6d44d3b3e9a9a3e41da98e8711e9f19a211fd8", - "source_identity": "source_sha256 records actual tested checkout bytes; committed Git blobs can differ only by checkout newline normalization", + "source_identity": "source_sha256 preserves tested checkout bytes at 56459cabd697f999547b0dec77e59fd0422c67c5; CI-only follow-up below is a separate identity", "source_sha256": { ".github/workflows/core-candidate.yml": "c758f8acc205b9bb0f5bca78e0505242c4695195425063ea96c6394d6b5c244e", "CMakeLists.txt": "185832b2b503091d9e1a9538e120e27fa8d8bcf8c8a6804f062b46a57868d325", @@ -115,5 +115,14 @@ "No transport/input/presentation or product workflow acceptance" ], "original_12_cost_samples_changed": false, - "performance_gate": "not_evaluated" + "performance_gate": "not_evaluated", + "ci_followup": { + "first_failed_run": "https://github.com/JavaZeroo/termforge/actions/runs/40305230147", + "first_failure": "workflow did not instantiate jobs: runner context is unavailable in job-level env", + "correction_commit": "3e2ba43", + "correction": "derive executable suffix from matrix.os in job-level env", + "workflow_sha256": "ff68936d25ec3057fe780155bea663ec086aaff3819de7233894a8e63b75a05b", + "measurement_runtime_changed": false, + "validation": "new GitHub validation pending; no formal cost samples" + } }