Repository navigation
diagnose(conpty): 重复原探针并在控制台测试后采样 refs #145 #173
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| """Bounded repeat of the recorded lifecycle sample (issue #145). | ||
|
|
||
| Every iteration runs one gated probe host inside its own owned job and keeps the | ||
| independent result. A failing iteration retains the native failure record | ||
| instead of being retried to green. | ||
| """ | ||
| from __future__ import annotations | ||
|
|
||
| import argparse | ||
| import importlib.util | ||
| import json | ||
| import platform | ||
| import statistics | ||
| import time | ||
| from datetime import datetime, timezone | ||
| from pathlib import Path | ||
|
|
||
|
|
||
| def load_run(): | ||
| path = Path(__file__).resolve().with_name("run.py") | ||
| spec = importlib.util.spec_from_file_location("termforge_conpty_run", path) | ||
| module = importlib.util.module_from_spec(spec) | ||
| spec.loader.exec_module(module) | ||
| return module | ||
|
|
||
|
|
||
| def summarize(records, iterations, repeats, mode): | ||
| failures = [row for row in records if row["status"] != "passed"] | ||
| elapsed = [row["elapsed_s"] for row in records] | ||
| return { | ||
| "iterations": iterations, | ||
| "sessions_per_iteration": repeats + 1, | ||
| "sessions_total": iterations * (repeats + 1), | ||
| "mode": mode, | ||
| "failures": len(failures), | ||
| "elapsed_s": {"min": min(elapsed), "median": statistics.median(elapsed), "max": max(elapsed)} if elapsed else {}, | ||
| "failure_samples": failures[:5], | ||
|
There was a problem hiding this comment.
When this tool is run using the documented standalone reproduction command, Useful? React with 👍 / 👎. |
||
| } | ||
|
|
||
|
|
||
| def main(): | ||
| parser = argparse.ArgumentParser(description=__doc__) | ||
| parser.add_argument("--binary", type=Path, required=True) | ||
| parser.add_argument("--iterations", type=int, default=40) | ||
| parser.add_argument("--repeats", type=int, default=3) | ||
| parser.add_argument("--mode", choices=("normal", "large", "cancel", "close"), default="normal") | ||
| parser.add_argument("--out", type=Path, required=True) | ||
| args = parser.parse_args() | ||
| if not 1 <= args.iterations <= 200: | ||
| parser.error("iterations must be 1..200") | ||
| if not 1 <= args.repeats <= 20: | ||
| parser.error("repeats must be 1..20") | ||
| if not args.binary.is_file(): | ||
| parser.error("probe binary missing") | ||
| args.out.mkdir(parents=True, exist_ok=False) | ||
| run = load_run() | ||
| summary = { | ||
| "schema": "termforge-conpty-probe-repeat-v1", | ||
| "issue": 145, | ||
| "status": "running", | ||
| "started_utc": datetime.now(timezone.utc).isoformat(), | ||
| "platform": platform.platform(), | ||
| "binary": str(args.binary.resolve()), | ||
| "performance_claim": False, | ||
| } | ||
| records = [] | ||
| for index in range(1, args.iterations + 1): | ||
| begin = time.monotonic() | ||
| try: | ||
| result = run.execute(args.binary, args.mode, args.repeats, timeout=120) | ||
| record = {"iteration": index, "status": result["status"], | ||
| "elapsed_s": round(time.monotonic() - begin, 3)} | ||
| except BaseException as error: | ||
| detail = getattr(error, "conpty_result", {}) | ||
| record = {"iteration": index, "status": "failed", | ||
| "elapsed_s": round(time.monotonic() - begin, 3), | ||
| "error": str(error), | ||
| # Keep the native diagnostics that the lifecycle probe | ||
| # already collects for a red sample. | ||
| "failure": {key: detail.get(key) for key in | ||
| ("returncode", "timed_out", "job_after", "startup", | ||
| "stdout", "stderr", "error") if key in detail}} | ||
| records.append(record) | ||
| summary.update(summarize(records, args.iterations, args.repeats, args.mode)) | ||
| summary["completed_iterations"] = index | ||
| (args.out / "repeat.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", | ||
| encoding="utf-8") | ||
| print(f"iteration {index}: {record['status']} ({record['elapsed_s']}s)", flush=True) | ||
| summary["status"] = "passed" if not summary["failures"] else "failed" | ||
| summary["finished_utc"] = datetime.now(timezone.utc).isoformat() | ||
| (args.out / "repeat.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", | ||
| encoding="utf-8") | ||
| # A red sample is the evidence we are looking for, not a job failure. | ||
| return 0 | ||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| raise SystemExit(main()) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| # 原生命周期探针重复与“测试后采样”（#145） | ||
|
|
||
| 本片回答一个具体问题：**把 [#145](../..#145) 记录的同一条样本原样重复，或在其它 ConPTY 测试之后采样，能否复现 resize 后仍回复旧尺寸？** | ||
|
|
||
| 结论：**都没有复现 resize 旧尺寸**；但在“先跑控制台密集测试、再采样”的条件下，复现了另一类红色——探针的会话后**线程基线**在 1 秒结算窗口内未回到基线（4→6），`steady_resources=false`，该样本的 resize 回复本身是正确的（`resized=1`、`TFSIZE:120x40`、child exit 37）。 | ||
|
|
||
| ## 方法 | ||
|
|
||
| 新增 `experiments/conpty/repeat_run.py`：每次迭代复用既有 `run.execute()`（自有 Job、GO 门、原 5 秒等待、原断言），保留每次独立结果；红色迭代保留探针原始失败字段，不重跑取绿。两个 dispatch-only 作业： | ||
|
|
||
| | 作业 | 前置 | 采样 | | ||
| |---|---|---| | ||
| | `windows-2025-original-probe-repeat` | 无（隔离） | 40 次 ×（1 startup + 3 normal） | | ||
| | `windows-2025-probe-after-conpty-tests` | 先跑 `conpty_ctrl_c`（5.31s）与 `conpty_margin`（0.31s），两者通过；其退出只记录不设门 | 20 次 ×（1 startup + 3 normal） | | ||
|
|
||
| ## 结果 | ||
|
|
||
| | 条件 | run/job | 样本 | resize 旧尺寸 | 其它红色 | | ||
| |---|---|---|---|---| | ||
| | isolated（hosted windows-2025） | [40847785625](https://github.com/JavaZeroo/termforge/actions/runs/40847785625) | 160 | 0 | 0 | | ||
| | 控制台密集测试之后 | [40849464556](https://github.com/JavaZeroo/termforge/actions/runs/40849464556) | 80 | 0 | 1（iteration 10, repeat 1） | | ||
| | 对照：克隆探针（#171） | 40841303342 / 40841934880 | 2800 | 0 | 0 | | ||
|
|
||
| - 隔离重复：每迭代 1.312 / 1.344 / 1.390 s（min/median/max），40/40 完成。 | ||
| - 测试后采样：20/20 完成；唯一红色样本为 `valid=false`，原因 `steady_resources=false`：`threads_before=4`、`threads_after=6`、`resource_settle_ms=1011.02`；同一样本 `resized=1`、`output_bytes=608`、`exit_code=37`、`read_error=109`，最终 VT 以 `TFSIZE:120x40:END` 结束。同迭代的 startup 样本因 `startup=true` 豁免基线比较（`handles 45→51` 属已知首次初始化）。 | ||
| - 未捕获任何 `resized=0` 或 80×24 旧回复。 | ||
|
|
||
| ## 解释与边界 | ||
|
|
||
| - `#145` 的旧尺寸回复在这三种条件下都没有复现；因此“原样本 × 原宿主 × 隔离或轻度前置”仍不足以触发，根因继续未知。 | ||
| - 复现的红是**另一类机构**：会话结束后控制台宿主线程比基线多 2 个且超过 1 秒结算窗口。它属于既有 `steady_resources` 夹具合同的边界（首次初始化只对 startup 豁免），与 resize 回复无关；记录为 #38 族的可靠性观察，不并入 #145 根因。 | ||
| - 本片没有改变原探针、其 5 秒等待、断言或任何产品代码；两个作业都是 `workflow_dispatch`，不进入常规 CI 门槛。 | ||
| - 分支上曾有“先跑完整 ctest 再采样”的作业，因该 dispatch 作业缺少 build.yml 的夹具准备而在 `benchmark_tools`/`differential`/`workflow_*`/`shell` 失败，未进入采样阶段；该失败保留在 run 40848143873，作为“作业环境不等价”的反例，不用于 #145 推断。 | ||
|
|
||
| ## 复现 | ||
|
|
||
| ```powershell | ||
| cmake -S . -B build | ||
| cmake --build build --config Release --target termforge_conpty_probe --parallel 2 | ||
| python -m experiments.conpty.repeat_run --binary build/bin/Release/termforge_conpty_probe.exe ` | ||
| --iterations 40 --repeats 3 --mode normal --out build/r145-repeat | ||
| ``` | ||
|
|
||
| 作业入口见 [conpty-compat.yml](../../../../.github/workflows/conpty-compat.yml) 的两个 `windows-2025-*` 作业。 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| { | ||
| "schema": "termforge-conpty-probe-repeat-r145-v1", | ||
| "issue": 145, | ||
| "commit": "f1c48d9", | ||
| "runs": { | ||
| "isolated": { | ||
| "run": 40847785625, | ||
| "job": "windows-2025-original-probe-repeat", | ||
| "iterations": 40, | ||
| "sessions": 160, | ||
| "failures": 0, | ||
| "elapsed_s": { | ||
| "min": 1.312, | ||
| "median": 1.344, | ||
| "max": 1.39 | ||
| } | ||
| }, | ||
| "after_console_tests": { | ||
| "run": 40849464556, | ||
| "job": "windows-2025-probe-after-conpty-tests", | ||
| "predecessors": [ | ||
| "conpty_ctrl_c (5.31s, passed)", | ||
| "conpty_margin (0.31s, passed)" | ||
| ], | ||
| "iterations": 20, | ||
| "sessions": 80, | ||
| "failures": 1 | ||
| }, | ||
| "clone_control": { | ||
| "runs": [ | ||
| 40841303342, | ||
| 40841934880 | ||
| ], | ||
| "samples": 2800, | ||
| "stale": 0 | ||
| } | ||
| }, | ||
| "revealed_red": { | ||
| "iteration": 10, | ||
| "sample": "normal repeat 1", | ||
| "valid": false, | ||
| "reason": "steady_resources=false", | ||
| "threads_before": 4, | ||
| "threads_after": 6, | ||
| "resource_settle_ms": 1011.02, | ||
| "resized": 1, | ||
| "output_bytes": 608, | ||
| "exit_code": 37, | ||
| "read_error": 109, | ||
| "note": "resize reply was correct (TFSIZE:120x40); the red is the post-session thread baseline, not a stale dimension" | ||
| }, | ||
| "preserved_failures": [ | ||
| "run 40848143873: whole-suite predecessor variant failed on fixtures the dispatch job does not prepare (benchmark_tools, differential, workflow_sessions, workflow_resources, workflow_resource_phases, shell); sampling stage never ran. Environment not equivalent to build.yml." | ||
| ], | ||
| "conclusion": "#145 的旧尺寸回复在隔离重复、控制台测试后采样和克隆探针放大三种条件下均未复现；复现的是另一类线程稳态红色。", | ||
| "performance_claim": false | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a probe sample fails,
probe.cppimmediately returns from the host, so the remaining repeats never execute. Nevertheless,sessions_totalalways reports the full scheduled count. For example, six iterations failing during startup report 24 sessions although only six ran. This also affects the committed post-predecessor evidence: a failure in normal repeat 1 skips a subsequent session, so 80 scheduled sessions do not establish 80 observed samples. Count sessions from the captured probe records and report the planned count separately.Useful? React with 👍 / 👎.