{ "schema_version": 1, "group": "G001", "conditions": [ { "id": "E009-E010", "label": "Generation \u00b7 fresh codebase \u00b7 1 imperative stage", "experiments": [ "E009", "E010" ], "card": { "question": "Does an algebraic design-derivation framework produce cleaner generated code than a semantic design framework when the builder must derive the design before coding?", "hypothesis": "Deriving substrate identities, capabilities, and their composition with algebraic laws and proofs improve code generation quality by reducing inference via proofs which reuse repeated reasoning", "prediction": "The algebraic arm's quality score exceeds the semantic arm's by at least 0.1.", "falsifier": "The algebraic arm's quality score is no higher than the semantic arm's. A positive difference below 0.1 is inconclusive." }, "decision_rule": { "metric": "score", "not_supported_if_difference_at_most": 0.099, "supported_if_difference_at_least": 0.1 }, "original_decision_rule": null, "stages": [ "stage-1" ], "arms": [ { "name": "control", "form": "null_form", "control": true }, { "name": "semantic", "form": "semantic", "control": false }, { "name": "algebraic", "form": "algebraic", "control": false } ], "hypothesis_arms": [ "algebraic" ], "limits": { "confidence_threshold": 0.9, "spending_limit_usd": 1.0, "time_limit_s": 6000 }, "varies": [ "arms.algebraic.controls.model_id", "arms.algebraic.controls.reasoning_effort", "arms.algebraic.model.id", "arms.control.controls.model_id", "arms.control.controls.reasoning_effort", "arms.control.model.id", "arms.semantic.controls.model_id", "arms.semantic.controls.reasoning_effort", "arms.semantic.model.id" ], "runs": [ { "run_id": "run-20260923T172943Z-c25ba500", "experiment_id": "E009", "started_at": "2026-09-23T17:29:43.988200+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-fable-5-1-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "8f042454c23c3a574ea6bdedb4946dde7f4cac9f" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "13a44dc710f2b634651bdb3112ca3d999c3c35c2" }, "harness_commit": "d5aee3415eb68ac4bcab918f6e5aeba5e51a746d", "arms": [ "control", "semantic", "algebraic" ], "stages": [ { "stage": 1, "name": "stage-1", "arms": { "control": { "status": "completed", "score": 0.59, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1199395.0, "of": null }, "system.elapsed_s": { "value": 460.479, "of": null }, "environment.tests_passed": { "value": 3.0, "of": 3.0 }, "environment.lines": { "value": 414.0, "of": null }, "environment.complexity_mean": { "value": 6.25, "of": null } }, "commands": { "all": 34, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.58, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1264360.0, "of": null }, "system.elapsed_s": { "value": 456.727, "of": null }, "environment.tests_passed": { "value": 4.0, "of": 4.0 }, "environment.lines": { "value": 564.0, "of": null }, "environment.complexity_mean": { "value": 5.923076923076923, "of": null } }, "commands": { "all": 36, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.52, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1586640.0, "of": null }, "system.elapsed_s": { "value": 471.006, "of": null }, "environment.tests_passed": { "value": 1.0, "of": 1.0 }, "environment.lines": { "value": 466.0, "of": null }, "environment.complexity_mean": { "value": 7.2105263157894735, "of": null } }, "commands": { "all": 35, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.01, "verdict": "not_supported" }, "algebraic": { "difference": -0.07, "verdict": "not_supported" } } } ], "totals": [ { "arm": "control", "stages": 1, "scores": [ { "stage": 1, "score": 0.59 } ], "mean_score": 0.59, "scored_stages": 1, "build_tokens": { "value": 1199395.0, "stages": 1 }, "review_tokens": { "value": 193197.0, "stages": 1 }, "total_tokens": { "value": 1392592.0, "stages": 1 }, "build_s": { "value": 460.488, "stages": 1 }, "review_s": { "value": 178.354, "stages": 1 }, "total_s": { "value": 640.904, "stages": 1 }, "final_stage": 1, "final_lines": 414.0, "final_complexity": 6.25 }, { "arm": "semantic", "stages": 1, "scores": [ { "stage": 1, "score": 0.58 } ], "mean_score": 0.58, "scored_stages": 1, "build_tokens": { "value": 1264360.0, "stages": 1 }, "review_tokens": { "value": 152083.0, "stages": 1 }, "total_tokens": { "value": 1416443.0, "stages": 1 }, "build_s": { "value": 456.731, "stages": 1 }, "review_s": { "value": 254.986, "stages": 1 }, "total_s": { "value": 713.799, "stages": 1 }, "final_stage": 1, "final_lines": 564.0, "final_complexity": 5.923076923076923 }, { "arm": "algebraic", "stages": 1, "scores": [ { "stage": 1, "score": 0.52 } ], "mean_score": 0.52, "scored_stages": 1, "build_tokens": { "value": 1586640.0, "stages": 1 }, "review_tokens": { "value": 206568.0, "stages": 1 }, "total_tokens": { "value": 1793208.0, "stages": 1 }, "build_s": { "value": 471.014, "stages": 1 }, "review_s": { "value": 204.453, "stages": 1 }, "total_s": { "value": 677.558, "stages": 1 }, "final_stage": 1, "final_lines": 466.0, "final_complexity": 7.2105263157894735 } ] }, { "run_id": "run-20260923T174224Z-7bf8e1fb", "experiment_id": "E010", "started_at": "2026-09-23T17:42:24.459757+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-luna", "reasoning_effort": "xhigh" }, "judge": [ "claude-fable-5-1-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "8f042454c23c3a574ea6bdedb4946dde7f4cac9f" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "13a44dc710f2b634651bdb3112ca3d999c3c35c2" }, "harness_commit": "d5aee3415eb68ac4bcab918f6e5aeba5e51a746d", "arms": [ "control", "semantic", "algebraic" ], "stages": [ { "stage": 1, "name": "stage-1", "arms": { "control": { "status": "completed", "score": 0.52, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 9281671.0, "of": null }, "system.elapsed_s": { "value": 1841.852, "of": null }, "environment.tests_passed": { "value": 5.0, "of": 5.0 }, "environment.lines": { "value": 2104.0, "of": null }, "environment.complexity_mean": { "value": 5.65, "of": null } }, "commands": { "all": 58, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.58, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 12035140.0, "of": null }, "system.elapsed_s": { "value": 2409.349, "of": null }, "environment.tests_passed": { "value": 3.0, "of": 3.0 }, "environment.lines": { "value": 1455.0, "of": null }, "environment.complexity_mean": { "value": 5.478260869565218, "of": null } }, "commands": { "all": 69, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.59, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 8632861.0, "of": null }, "system.elapsed_s": { "value": 1817.498, "of": null }, "environment.tests_passed": { "value": 4.0, "of": 4.0 }, "environment.lines": { "value": 1978.0, "of": null }, "environment.complexity_mean": { "value": 5.0, "of": null } }, "commands": { "all": 55, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.06, "verdict": "not_supported" }, "algebraic": { "difference": 0.07, "verdict": "not_supported" } } } ], "totals": [ { "arm": "control", "stages": 1, "scores": [ { "stage": 1, "score": 0.52 } ], "mean_score": 0.52, "scored_stages": 1, "build_tokens": { "value": 9281671.0, "stages": 1 }, "review_tokens": { "value": 413075.0, "stages": 1 }, "total_tokens": { "value": 9694746.0, "stages": 1 }, "build_s": { "value": 1841.86, "stages": 1 }, "review_s": { "value": 163.401, "stages": 1 }, "total_s": { "value": 2007.191, "stages": 1 }, "final_stage": 1, "final_lines": 2104.0, "final_complexity": 5.65 }, { "arm": "semantic", "stages": 1, "scores": [ { "stage": 1, "score": 0.58 } ], "mean_score": 0.58, "scored_stages": 1, "build_tokens": { "value": 12035140.0, "stages": 1 }, "review_tokens": { "value": 377906.0, "stages": 1 }, "total_tokens": { "value": 12413046.0, "stages": 1 }, "build_s": { "value": 2409.358, "stages": 1 }, "review_s": { "value": 189.575, "stages": 1 }, "total_s": { "value": 2600.914, "stages": 1 }, "final_stage": 1, "final_lines": 1455.0, "final_complexity": 5.478260869565218 }, { "arm": "algebraic", "stages": 1, "scores": [ { "stage": 1, "score": 0.59 } ], "mean_score": 0.59, "scored_stages": 1, "build_tokens": { "value": 8632861.0, "stages": 1 }, "review_tokens": { "value": 648277.0, "stages": 1 }, "total_tokens": { "value": 9281138.0, "stages": 1 }, "build_s": { "value": 1817.507, "stages": 1 }, "review_s": { "value": 178.287, "stages": 1 }, "total_s": { "value": 1997.776, "stages": 1 }, "final_stage": 1, "final_lines": 1978.0, "final_complexity": 5.0 } ] } ], "checks": [ { "kind": "contrast", "arms": [ "algebraic" ], "than": "control", "measurement": null, "require": "every", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T172943Z-c25ba500", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": -0.07, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260923T174224Z-7bf8e1fb", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.07, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] }, { "kind": "difference", "arms": [ "algebraic" ], "than": "semantic", "measurement": null, "require": "every", "verdict": false, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T172943Z-c25ba500", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": -0.06, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260923T174224Z-7bf8e1fb", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.01, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] } ], "verdict": "not_supported", "readings": { "every": false, "most": false, "mean": false }, "reading_independent": true, "effect": { "algebraic": { "cells": 2, "mean": 0.0, "min": -0.07, "max": 0.07, "differences": [ -0.07, 0.07 ] } }, "spread": { "sets": [] } }, { "id": "E015-E017", "label": "Generation \u00b7 fresh codebase \u00b7 5 imperative stages", "experiments": [ "E015", "E016" ], "card": { "question": "Does an algebraic design-derivation framework produce cleaner generated code than a semantic design framework when the builder must derive the design before coding?", "hypothesis": "Deriving substrate identities, capabilities, and their composition with algebraic laws and proofs improve code generation quality by reducing inference via proofs which reuse repeated reasoning", "prediction": "The algebraic arm's quality score exceeds the semantic arm's by at least 0.1.", "falsifier": "The algebraic arm's quality score is no higher than the semantic arm's. A positive difference below 0.1 is inconclusive." }, "decision_rule": { "metric": "score", "not_supported_if_difference_at_most": 0.099, "supported_if_difference_at_least": 0.1 }, "original_decision_rule": null, "stages": [ "initial-build", "telemetry-add", "parity-substrate", "process-ontology", "rust-conversion" ], "arms": [ { "name": "control", "form": "null_form", "control": true }, { "name": "semantic", "form": "semantic", "control": false }, { "name": "algebraic", "form": "algebraic", "control": false } ], "hypothesis_arms": [ "algebraic" ], "limits": { "confidence_threshold": 0.9, "spending_limit_usd": 1.0, "time_limit_s": 600000 }, "varies": [ "arms.algebraic.controls.model_id", "arms.algebraic.controls.reasoning_effort", "arms.algebraic.model.id", "arms.control.controls.model_id", "arms.control.controls.reasoning_effort", "arms.control.model.id", "arms.semantic.controls.model_id", "arms.semantic.controls.reasoning_effort", "arms.semantic.model.id" ], "runs": [ { "run_id": "run-20260923T223518Z-020373a4", "experiment_id": "E015", "started_at": "2026-09-23T22:35:18.759879+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-astra", "reasoning_effort": "xhigh" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "6435a7db8b7ae9c7c2305a2faa9d5ed50033ae83" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "46e50f4afb0ad0faeb667c1befac89bbfb60d258", "arms": [ "control", "semantic", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.39, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1893568.0, "of": null }, "system.elapsed_s": { "value": 1491.945, "of": null }, "environment.tests_passed": { "value": 46.0, "of": 46.0 }, "environment.lines": { "value": 1682.0, "of": null }, "environment.complexity_mean": { "value": 4.8686868686868685, "of": null } }, "commands": { "all": 28, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.36, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2982774.0, "of": null }, "system.elapsed_s": { "value": 1876.401, "of": null }, "environment.tests_passed": { "value": 41.0, "of": 42.0 }, "environment.lines": { "value": 1851.0, "of": null }, "environment.complexity_mean": { "value": 4.7011494252873565, "of": null } }, "commands": { "all": 28, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.44, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2088833.0, "of": null }, "system.elapsed_s": { "value": 1519.151, "of": null }, "environment.tests_passed": { "value": 49.0, "of": 50.0 }, "environment.lines": { "value": 1796.0, "of": null }, "environment.complexity_mean": { "value": 4.517857142857143, "of": null } }, "commands": { "all": 28, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.03, "verdict": "not_supported" }, "algebraic": { "difference": 0.05, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2242095.0, "of": null }, "system.elapsed_s": { "value": 1146.518, "of": null }, "environment.tests_passed": { "value": 97.0, "of": 97.0 }, "environment.lines": { "value": 2622.0, "of": null }, "environment.complexity_mean": { "value": 4.424390243902439, "of": null } }, "commands": { "all": 40, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.48, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2491888.0, "of": null }, "system.elapsed_s": { "value": 1335.683, "of": null }, "environment.tests_passed": { "value": 86.0, "of": 87.0 }, "environment.lines": { "value": 2740.0, "of": null }, "environment.complexity_mean": { "value": 4.012269938650307, "of": null } }, "commands": { "all": 45, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.36, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2959462.0, "of": null }, "system.elapsed_s": { "value": 1285.109, "of": null }, "environment.tests_passed": { "value": 101.0, "of": 115.0 }, "environment.lines": { "value": 2888.0, "of": null }, "environment.complexity_mean": { "value": 4.036036036036036, "of": null } }, "commands": { "all": 34, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.14, "verdict": "supported" }, "algebraic": { "difference": 0.02, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.36, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3802205.0, "of": null }, "system.elapsed_s": { "value": 1541.237, "of": null }, "environment.tests_passed": { "value": 140.0, "of": 140.0 }, "environment.lines": { "value": 3448.0, "of": null }, "environment.complexity_mean": { "value": 4.017667844522968, "of": null } }, "commands": { "all": 46, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3822155.0, "of": null }, "system.elapsed_s": { "value": 1355.212, "of": null }, "environment.tests_passed": { "value": 117.0, "of": 136.0 }, "environment.lines": { "value": 3988.0, "of": null }, "environment.complexity_mean": { "value": 4.021551724137931, "of": null } }, "commands": { "all": 54, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4794219.0, "of": null }, "system.elapsed_s": { "value": 1878.542, "of": null }, "environment.tests_passed": { "value": 135.0, "of": 185.0 }, "environment.lines": { "value": 4382.0, "of": null }, "environment.complexity_mean": { "value": 4.115151515151515, "of": null } }, "commands": { "all": 52, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.05, "verdict": "not_supported" }, "algebraic": { "difference": -0.05, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.39, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4450604.0, "of": null }, "system.elapsed_s": { "value": 1138.345, "of": null }, "environment.tests_passed": { "value": 146.0, "of": 146.0 }, "environment.lines": { "value": 3903.0, "of": null }, "environment.complexity_mean": { "value": 4.110429447852761, "of": null } }, "commands": { "all": 56, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5043186.0, "of": null }, "system.elapsed_s": { "value": 1405.099, "of": null }, "environment.tests_passed": { "value": 134.0, "of": 165.0 }, "environment.lines": { "value": 4578.0, "of": null }, "environment.complexity_mean": { "value": 4.086466165413534, "of": null } }, "commands": { "all": 61, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 13259272.0, "of": null }, "system.elapsed_s": { "value": 2406.328, "of": null }, "environment.tests_passed": { "value": 149.0, "of": 220.0 }, "environment.lines": { "value": 4850.0, "of": null }, "environment.complexity_mean": { "value": 4.259562841530054, "of": null } }, "commands": { "all": 86, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.09, "verdict": "not_supported" }, "algebraic": { "difference": -0.05, "verdict": "not_supported" } } }, { "stage": 5, "name": "rust-conversion", "arms": { "control": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test acceptance`" } ], "measurements": { "system.total_tokens": { "value": 8076659.0, "of": null }, "system.elapsed_s": { "value": 2553.725, "of": null }, "environment.rust_tests_passed": { "value": 1.0, "of": 25.0 } }, "commands": { "all": 71, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 8479478.0, "of": null }, "system.elapsed_s": { "value": 2360.319, "of": null }, "environment.rust_tests_passed": { "value": 194.0, "of": 194.0 } }, "commands": { "all": 72, "z3": 0 } }, "algebraic": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test cutover`" } ], "measurements": { "system.total_tokens": { "value": 12773497.0, "of": null }, "system.elapsed_s": { "value": 3547.753, "of": null }, "environment.rust_tests_passed": { "value": 38.0, "of": 76.0 } }, "commands": { "all": 125, "z3": 0 } } }, "contrasts": { "semantic": { "difference": null, "verdict": "invalid" }, "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.39 }, { "stage": 2, "score": 0.34 }, { "stage": 3, "score": 0.36 }, { "stage": 4, "score": 0.39 }, { "stage": 5, "score": null } ], "mean_score": 0.37, "scored_stages": 4, "build_tokens": { "value": 20465131.0, "stages": 5 }, "review_tokens": { "value": 2022106.0, "stages": 5 }, "total_tokens": { "value": 22487237.0, "stages": 5 }, "build_s": { "value": 7871.814, "stages": 5 }, "review_s": { "value": 712.723, "stages": 5 }, "total_s": { "value": 8595.626, "stages": 5 }, "final_stage": 5, "final_lines": 5178.0, "final_complexity": null }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.36 }, { "stage": 2, "score": 0.48 }, { "stage": 3, "score": 0.31 }, { "stage": 4, "score": 0.3 }, { "stage": 5, "score": 0.31 } ], "mean_score": 0.352, "scored_stages": 5, "build_tokens": { "value": 22819481.0, "stages": 5 }, "review_tokens": { "value": 1948842.0, "stages": 5 }, "total_tokens": { "value": 24768323.0, "stages": 5 }, "build_s": { "value": 8332.764, "stages": 5 }, "review_s": { "value": 693.302, "stages": 5 }, "total_s": { "value": 9036.567000000001, "stages": 5 }, "final_stage": 5, "final_lines": 5062.0, "final_complexity": null }, { "arm": "algebraic", "stages": 5, "scores": [ { "stage": 1, "score": 0.44 }, { "stage": 2, "score": 0.36 }, { "stage": 3, "score": 0.31 }, { "stage": 4, "score": 0.34 }, { "stage": 5, "score": null } ], "mean_score": 0.3625, "scored_stages": 4, "build_tokens": { "value": 35875283.0, "stages": 5 }, "review_tokens": { "value": 2254332.0, "stages": 5 }, "total_tokens": { "value": 38129615.0, "stages": 5 }, "build_s": { "value": 10637.009, "stages": 5 }, "review_s": { "value": 725.435, "stages": 5 }, "total_s": { "value": 11373.323, "stages": 5 }, "final_stage": 5, "final_lines": 6425.0, "final_complexity": null } ] }, { "run_id": "run-20260923T223651Z-b952ede3", "experiment_id": "E016", "started_at": "2026-09-23T22:36:51.350183+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "6435a7db8b7ae9c7c2305a2faa9d5ed50033ae83" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "46e50f4afb0ad0faeb667c1befac89bbfb60d258", "arms": [ "control", "semantic", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1332180.0, "of": null }, "system.elapsed_s": { "value": 558.712, "of": null }, "environment.tests_passed": { "value": 4.0, "of": 4.0 }, "environment.lines": { "value": 512.0, "of": null }, "environment.complexity_mean": { "value": 5.620689655172414, "of": null } }, "commands": { "all": 28, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.4, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 956769.0, "of": null }, "system.elapsed_s": { "value": 405.148, "of": null }, "environment.tests_passed": { "value": 2.0, "of": 2.0 }, "environment.lines": { "value": 404.0, "of": null }, "environment.complexity_mean": { "value": 6.391304347826087, "of": null } }, "commands": { "all": 20, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1857162.0, "of": null }, "system.elapsed_s": { "value": 432.145, "of": null }, "environment.tests_passed": { "value": 2.0, "of": 2.0 }, "environment.lines": { "value": 511.0, "of": null }, "environment.complexity_mean": { "value": 6.36, "of": null } }, "commands": { "all": 29, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.1, "verdict": "supported" }, "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2339461.0, "of": null }, "system.elapsed_s": { "value": 714.115, "of": null }, "environment.tests_passed": { "value": 12.0, "of": 12.0 }, "environment.lines": { "value": 699.0, "of": null }, "environment.complexity_mean": { "value": 5.7555555555555555, "of": null } }, "commands": { "all": 42, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1369048.0, "of": null }, "system.elapsed_s": { "value": 397.792, "of": null }, "environment.tests_passed": { "value": 9.0, "of": 9.0 }, "environment.lines": { "value": 636.0, "of": null }, "environment.complexity_mean": { "value": 6.914285714285715, "of": null } }, "commands": { "all": 29, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.36, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2013668.0, "of": null }, "system.elapsed_s": { "value": 519.723, "of": null }, "environment.tests_passed": { "value": 11.0, "of": 11.0 }, "environment.lines": { "value": 873.0, "of": null }, "environment.complexity_mean": { "value": 5.547169811320755, "of": null } }, "commands": { "all": 38, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.06, "verdict": "not_supported" }, "algebraic": { "difference": 0.08, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.36, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2698152.0, "of": null }, "system.elapsed_s": { "value": 910.644, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1146.0, "of": null }, "environment.complexity_mean": { "value": 6.555555555555555, "of": null } }, "commands": { "all": 45, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4368996.0, "of": null }, "system.elapsed_s": { "value": 741.695, "of": null }, "environment.tests_passed": { "value": 14.0, "of": 14.0 }, "environment.lines": { "value": 980.0, "of": null }, "environment.complexity_mean": { "value": 6.767857142857143, "of": null } }, "commands": { "all": 50, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5401780.0, "of": null }, "system.elapsed_s": { "value": 1417.93, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1407.0, "of": null }, "environment.complexity_mean": { "value": 6.886075949367089, "of": null } }, "commands": { "all": 59, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.08, "verdict": "not_supported" }, "algebraic": { "difference": -0.08, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2693363.0, "of": null }, "system.elapsed_s": { "value": 455.658, "of": null }, "environment.tests_passed": { "value": 24.0, "of": 24.0 }, "environment.lines": { "value": 1285.0, "of": null }, "environment.complexity_mean": { "value": 6.353658536585366, "of": null } }, "commands": { "all": 50, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3621029.0, "of": null }, "system.elapsed_s": { "value": 626.29, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1141.0, "of": null }, "environment.complexity_mean": { "value": 7.317460317460317, "of": null } }, "commands": { "all": 55, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4604135.0, "of": null }, "system.elapsed_s": { "value": 644.502, "of": null }, "environment.tests_passed": { "value": 24.0, "of": 24.0 }, "environment.lines": { "value": 1543.0, "of": null }, "environment.complexity_mean": { "value": 7.1395348837209305, "of": null } }, "commands": { "all": 53, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.03, "verdict": "not_supported" }, "algebraic": { "difference": -0.01, "verdict": "not_supported" } } }, { "stage": 5, "name": "rust-conversion", "arms": { "control": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test cutover`" } ], "measurements": { "system.total_tokens": { "value": 5934105.0, "of": null }, "system.elapsed_s": { "value": 988.588, "of": null }, "environment.rust_tests_passed": { "value": 10.0, "of": 12.0 } }, "commands": { "all": 60, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3931779.0, "of": null }, "system.elapsed_s": { "value": 862.301, "of": null }, "environment.rust_tests_passed": { "value": 9.0, "of": 9.0 } }, "commands": { "all": 51, "z3": 0 } }, "algebraic": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test cutover`" } ], "measurements": { "system.total_tokens": { "value": 6609145.0, "of": null }, "system.elapsed_s": { "value": 1003.158, "of": null }, "environment.rust_tests_passed": { "value": 11.0, "of": 12.0 } }, "commands": { "all": 71, "z3": 0 } } }, "contrasts": { "semantic": { "difference": null, "verdict": "invalid" }, "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.3 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.36 }, { "stage": 4, "score": 0.31 }, { "stage": 5, "score": null } ], "mean_score": 0.3125, "scored_stages": 4, "build_tokens": { "value": 14997261.0, "stages": 5 }, "review_tokens": { "value": 1557742.0, "stages": 5 }, "total_tokens": { "value": 16555003.0, "stages": 5 }, "build_s": { "value": 3627.768, "stages": 5 }, "review_s": { "value": 614.658, "stages": 5 }, "total_s": { "value": 4251.777, "stages": 5 }, "final_stage": 5, "final_lines": 128.0, "final_complexity": null }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.4 }, { "stage": 2, "score": 0.34 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.28 } ], "mean_score": 0.316, "scored_stages": 5, "build_tokens": { "value": 14247621.0, "stages": 5 }, "review_tokens": { "value": 1285028.0, "stages": 5 }, "total_tokens": { "value": 15532649.0, "stages": 5 }, "build_s": { "value": 3033.276, "stages": 5 }, "review_s": { "value": 647.872, "stages": 5 }, "total_s": { "value": 3690.413, "stages": 5 }, "final_stage": 5, "final_lines": 200.0, "final_complexity": null }, { "arm": "algebraic", "stages": 5, "scores": [ { "stage": 1, "score": 0.3 }, { "stage": 2, "score": 0.36 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.3 }, { "stage": 5, "score": null } ], "mean_score": 0.31, "scored_stages": 4, "build_tokens": { "value": 20485890.0, "stages": 5 }, "review_tokens": { "value": 1537204.0, "stages": 5 }, "total_tokens": { "value": 22023094.0, "stages": 5 }, "build_s": { "value": 4017.507, "stages": 5 }, "review_s": { "value": 872.063, "stages": 5 }, "total_s": { "value": 4899.06, "stages": 5 }, "final_stage": 5, "final_lines": 144.0, "final_complexity": null } ] } ], "checks": [ { "kind": "contrast", "arms": [ "algebraic" ], "than": "control", "measurement": null, "require": "every", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T223518Z-020373a4", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.05, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.02, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.05, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.05, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260923T223651Z-b952ede3", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.08, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.08, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.01, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] }, { "kind": "difference", "arms": [ "algebraic" ], "than": "semantic", "measurement": null, "require": "every", "verdict": false, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T223518Z-020373a4", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.08, "passed": false }, { "stage": 2, "difference": -0.12, "passed": false }, { "stage": 3, "difference": 0.0, "passed": false }, { "stage": 4, "difference": 0.04, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260923T223651Z-b952ede3", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": -0.1, "passed": false }, { "stage": 2, "difference": 0.02, "passed": false }, { "stage": 3, "difference": 0.0, "passed": false }, { "stage": 4, "difference": 0.02, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] } ], "verdict": "not_supported", "readings": { "every": false, "most": false, "mean": false }, "reading_independent": true, "effect": { "algebraic": { "cells": 8, "mean": -0.005, "min": -0.08, "max": 0.08, "differences": [ 0.05, 0.02, -0.05, -0.05, 0.0, 0.08, -0.08, -0.01 ] } }, "spread": { "sets": [] } }, { "id": "E018-E025", "label": "Generation \u00b7 fresh codebase \u00b7 5 imperative stages \u00b7 z3-checked algebra", "experiments": [ "E018", "E025" ], "card": { "question": "Does an algebraic design-derivation framework produce cleaner generated code than a semantic design framework when the builder must derive the design before coding?", "hypothesis": "Deriving substrate identities, capabilities, and their composition with algebraic laws and proofs improve code generation quality by reducing inference via proofs which reuse repeated reasoning", "prediction": "The algebraic arm's quality score exceeds the semantic arm's by at least 0.1.", "falsifier": "The algebraic arm's quality score is no higher than the semantic arm's. A positive difference below 0.1 is inconclusive." }, "decision_rule": { "metric": "score", "not_supported_if_difference_at_most": 0.099, "supported_if_difference_at_least": 0.1 }, "original_decision_rule": null, "stages": [ "initial-build", "telemetry-add", "parity-substrate", "process-ontology", "rust-conversion" ], "arms": [ { "name": "control", "form": "null_form", "control": true }, { "name": "algebraic", "form": "algebraic_solver", "control": false } ], "hypothesis_arms": [ "algebraic" ], "limits": { "confidence_threshold": 0.9, "spending_limit_usd": 1.0, "time_limit_s": 600000 }, "varies": [], "runs": [ { "run_id": "run-20260923T232241Z-59b76d12", "experiment_id": "E018", "started_at": "2026-09-23T23:22:41.069354+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "e01402ca8d21cba2dcee1ef93734c00992151556" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "46e50f4afb0ad0faeb667c1befac89bbfb60d258", "arms": [ "control", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.32, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1500101.0, "of": null }, "system.elapsed_s": { "value": 558.86, "of": null }, "environment.tests_passed": { "value": 2.0, "of": 2.0 }, "environment.lines": { "value": 560.0, "of": null }, "environment.complexity_mean": { "value": 6.846153846153846, "of": null } }, "commands": { "all": 32, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.32, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5983377.0, "of": null }, "system.elapsed_s": { "value": 777.397, "of": null }, "system.solver_obligations": { "value": 62.0, "of": null }, "system.solver_established": { "value": 62.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 3.0, "of": 3.0 }, "environment.lines": { "value": 632.0, "of": null }, "environment.complexity_mean": { "value": 6.033333333333333, "of": null } }, "commands": { "all": 46, "z3": 4 } } }, "contrasts": { "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4073846.0, "of": null }, "system.elapsed_s": { "value": 734.039, "of": null }, "environment.tests_passed": { "value": 12.0, "of": 12.0 }, "environment.lines": { "value": 898.0, "of": null }, "environment.complexity_mean": { "value": 6.761904761904762, "of": null } }, "commands": { "all": 38, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3274234.0, "of": null }, "system.elapsed_s": { "value": 621.113, "of": null }, "system.solver_obligations": { "value": 82.0, "of": null }, "system.solver_established": { "value": 82.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 13.0, "of": 13.0 }, "environment.lines": { "value": 939.0, "of": null }, "environment.complexity_mean": { "value": 5.829787234042553, "of": null } }, "commands": { "all": 49, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": 0.02, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 12352256.0, "of": null }, "system.elapsed_s": { "value": 1749.87, "of": null }, "environment.tests_passed": { "value": 18.0, "of": 18.0 }, "environment.lines": { "value": 1485.0, "of": null }, "environment.complexity_mean": { "value": 7.7, "of": null } }, "commands": { "all": 77, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2024388.0, "of": null }, "system.elapsed_s": { "value": 707.499, "of": null }, "system.solver_obligations": { "value": 96.0, "of": null }, "system.solver_established": { "value": 96.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 16.0, "of": 16.0 }, "environment.lines": { "value": 1341.0, "of": null }, "environment.complexity_mean": { "value": 6.863636363636363, "of": null } }, "commands": { "all": 36, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": -0.03, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4524259.0, "of": null }, "system.elapsed_s": { "value": 704.344, "of": null }, "environment.tests_passed": { "value": 24.0, "of": 24.0 }, "environment.lines": { "value": 1709.0, "of": null }, "environment.complexity_mean": { "value": 8.08860759493671, "of": null } }, "commands": { "all": 46, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.39, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2997377.0, "of": null }, "system.elapsed_s": { "value": 577.345, "of": null }, "system.solver_obligations": { "value": 102.0, "of": null }, "system.solver_established": { "value": 102.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 21.0, "of": 21.0 }, "environment.lines": { "value": 1490.0, "of": null }, "environment.complexity_mean": { "value": 7.205479452054795, "of": null } }, "commands": { "all": 48, "z3": 2 } } }, "contrasts": { "algebraic": { "difference": 0.11, "verdict": "supported" } } }, { "stage": 5, "name": "rust-conversion", "arms": { "control": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test cutover`" } ], "measurements": { "system.total_tokens": { "value": 9074070.0, "of": null }, "system.elapsed_s": { "value": 1372.868, "of": null }, "environment.rust_tests_passed": { "value": 7.0, "of": 9.0 } }, "commands": { "all": 62, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5839999.0, "of": null }, "system.elapsed_s": { "value": 955.518, "of": null }, "system.solver_obligations": { "value": 115.0, "of": null }, "system.solver_established": { "value": 115.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.rust_tests_passed": { "value": 9.0, "of": 9.0 } }, "commands": { "all": 58, "z3": 4 } } }, "contrasts": { "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.32 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.31 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": null } ], "mean_score": 0.2975, "scored_stages": 4, "build_tokens": { "value": 31524532.0, "stages": 5 }, "review_tokens": { "value": 1061537.0, "stages": 5 }, "total_tokens": { "value": 32586069.0, "stages": 5 }, "build_s": { "value": 5120.025, "stages": 5 }, "review_s": { "value": 1130.9080000000001, "stages": 5 }, "total_s": { "value": 6260.973, "stages": 5 }, "final_stage": 5, "final_lines": 156.0, "final_complexity": null }, { "arm": "algebraic", "stages": 5, "scores": [ { "stage": 1, "score": 0.32 }, { "stage": 2, "score": 0.3 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.39 }, { "stage": 5, "score": 0.28 } ], "mean_score": 0.314, "scored_stages": 5, "build_tokens": { "value": 20119375.0, "stages": 5 }, "review_tokens": { "value": 1407854.0, "stages": 5 }, "total_tokens": { "value": 21527229.0, "stages": 5 }, "build_s": { "value": 3639.296, "stages": 5 }, "review_s": { "value": 674.076, "stages": 5 }, "total_s": { "value": 4323.321, "stages": 5 }, "final_stage": 5, "final_lines": 213.0, "final_complexity": null } ] }, { "run_id": "run-20260924T031035Z-6515aace", "experiment_id": "E025", "started_at": "2026-09-24T03:10:35.591612+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "e01402ca8d21cba2dcee1ef93734c00992151556" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1117244.0, "of": null }, "system.elapsed_s": { "value": 555.43, "of": null }, "environment.tests_passed": { "value": 4.0, "of": 4.0 }, "environment.lines": { "value": 664.0, "of": null }, "environment.complexity_mean": { "value": 5.8, "of": null } }, "commands": { "all": 18, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3693918.0, "of": null }, "system.elapsed_s": { "value": 809.117, "of": null }, "system.solver_obligations": { "value": 53.0, "of": null }, "system.solver_established": { "value": 51.0, "of": null }, "system.solver_refuted": { "value": 2.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 3.0, "of": 3.0 }, "environment.lines": { "value": 595.0, "of": null }, "environment.complexity_mean": { "value": 6.2, "of": null } }, "commands": { "all": 53, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": 0.02, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1450401.0, "of": null }, "system.elapsed_s": { "value": 471.772, "of": null }, "environment.tests_passed": { "value": 13.0, "of": 13.0 }, "environment.lines": { "value": 939.0, "of": null }, "environment.complexity_mean": { "value": 5.9787234042553195, "of": null } }, "commands": { "all": 30, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4054171.0, "of": null }, "system.elapsed_s": { "value": 765.328, "of": null }, "system.solver_obligations": { "value": 63.0, "of": null }, "system.solver_established": { "value": 61.0, "of": null }, "system.solver_refuted": { "value": 2.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 10.0, "of": 10.0 }, "environment.lines": { "value": 745.0, "of": null }, "environment.complexity_mean": { "value": 6.3, "of": null } }, "commands": { "all": 46, "z3": 0 } } }, "contrasts": { "algebraic": { "difference": 0.02, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 11459273.0, "of": null }, "system.elapsed_s": { "value": 1563.268, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 20.0 }, "environment.lines": { "value": 1318.0, "of": null }, "environment.complexity_mean": { "value": 6.214285714285714, "of": null } }, "commands": { "all": 82, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.25, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4268376.0, "of": null }, "system.elapsed_s": { "value": 1281.317, "of": null }, "system.solver_obligations": { "value": 77.0, "of": null }, "system.solver_established": { "value": 75.0, "of": null }, "system.solver_refuted": { "value": 2.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1211.0, "of": null }, "environment.complexity_mean": { "value": 6.545454545454546, "of": null } }, "commands": { "all": 44, "z3": 2 } } }, "contrasts": { "algebraic": { "difference": -0.03, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3254351.0, "of": null }, "system.elapsed_s": { "value": 530.015, "of": null }, "environment.tests_passed": { "value": 26.0, "of": 26.0 }, "environment.lines": { "value": 1470.0, "of": null }, "environment.complexity_mean": { "value": 6.576923076923077, "of": null } }, "commands": { "all": 49, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2862835.0, "of": null }, "system.elapsed_s": { "value": 498.279, "of": null }, "system.solver_obligations": { "value": 87.0, "of": null }, "system.solver_established": { "value": 85.0, "of": null }, "system.solver_refuted": { "value": 2.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 1397.0, "of": null }, "environment.complexity_mean": { "value": 6.3076923076923075, "of": null } }, "commands": { "all": 46, "z3": 2 } } }, "contrasts": { "algebraic": { "difference": 0.06, "verdict": "not_supported" } } }, { "stage": 5, "name": "rust-conversion", "arms": { "control": { "status": "completed", "score": 0.25, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5669156.0, "of": null }, "system.elapsed_s": { "value": 838.309, "of": null }, "environment.rust_tests_passed": { "value": 11.0, "of": 11.0 } }, "commands": { "all": 44, "z3": 0 } }, "algebraic": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test cutover`" } ], "measurements": { "system.total_tokens": { "value": 6410362.0, "of": null }, "system.elapsed_s": { "value": 966.629, "of": null }, "system.solver_obligations": { "value": 92.0, "of": null }, "system.solver_established": { "value": 90.0, "of": null }, "system.solver_refuted": { "value": 2.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.rust_tests_passed": { "value": 9.0, "of": 13.0 } }, "commands": { "all": 63, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.28 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.25 } ], "mean_score": 0.274, "scored_stages": 5, "build_tokens": { "value": 22950425.0, "stages": 5 }, "review_tokens": { "value": 1127795.0, "stages": 5 }, "total_tokens": { "value": 24078220.0, "stages": 5 }, "build_s": { "value": 3958.868, "stages": 5 }, "review_s": { "value": 1064.313, "stages": 5 }, "total_s": { "value": 5033.007, "stages": 5 }, "final_stage": 5, "final_lines": 215.0, "final_complexity": null }, { "arm": "algebraic", "stages": 5, "scores": [ { "stage": 1, "score": 0.3 }, { "stage": 2, "score": 0.3 }, { "stage": 3, "score": 0.25 }, { "stage": 4, "score": 0.34 }, { "stage": 5, "score": null } ], "mean_score": 0.2975, "scored_stages": 4, "build_tokens": { "value": 21289662.0, "stages": 5 }, "review_tokens": { "value": 1305575.0, "stages": 5 }, "total_tokens": { "value": 22595237.0, "stages": 5 }, "build_s": { "value": 4321.16, "stages": 5 }, "review_s": { "value": 911.933, "stages": 5 }, "total_s": { "value": 5242.388, "stages": 5 }, "final_stage": 5, "final_lines": 972.0, "final_complexity": null } ] } ], "checks": [ { "kind": "contrast", "arms": [ "algebraic" ], "than": "control", "measurement": null, "require": "every", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T232241Z-59b76d12", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.02, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.03, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.11, "verdict": "supported", "passed": true } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T031035Z-6515aace", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.02, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.02, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.03, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.06, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] }, { "kind": "difference", "arms": [ "algebraic" ], "than": "semantic", "measurement": null, "require": "every", "verdict": false, "not_run": true, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T232241Z-59b76d12", "arm": "algebraic", "not_run": true }, { "run_id": "run-20260924T031035Z-6515aace", "arm": "algebraic", "not_run": true } ] } ], "verdict": "not_supported", "readings": { "every": false, "most": false, "mean": false }, "reading_independent": true, "effect": { "algebraic": { "cells": 8, "mean": 0.0213, "min": -0.03, "max": 0.11, "differences": [ 0.0, 0.02, -0.03, 0.11, 0.02, 0.02, -0.03, 0.06 ] } }, "spread": { "sets": [ { "runs": [ "run-20260923T232241Z-59b76d12", "run-20260924T031035Z-6515aace" ], "cells": [ { "stage": 1, "arm": "control", "scores": [ 0.32, 0.28 ], "range": 0.04, "reaches_threshold": false }, { "stage": 1, "arm": "algebraic", "scores": [ 0.32, 0.3 ], "range": 0.02, "reaches_threshold": false }, { "stage": 2, "arm": "control", "scores": [ 0.28, 0.28 ], "range": 0.0, "reaches_threshold": false }, { "stage": 2, "arm": "algebraic", "scores": [ 0.3, 0.3 ], "range": 0.0, "reaches_threshold": false }, { "stage": 3, "arm": "control", "scores": [ 0.31, 0.28 ], "range": 0.03, "reaches_threshold": false }, { "stage": 3, "arm": "algebraic", "scores": [ 0.28, 0.25 ], "range": 0.03, "reaches_threshold": false }, { "stage": 4, "arm": "control", "scores": [ 0.28, 0.28 ], "range": 0.0, "reaches_threshold": false }, { "stage": 4, "arm": "algebraic", "scores": [ 0.39, 0.34 ], "range": 0.05, "reaches_threshold": false } ] } ] } }, { "id": "E020-E026", "label": "Generation \u00b7 fresh codebase \u00b7 7 declarative stages \u00b7 z3-checked algebra", "experiments": [ "E020", "E026" ], "card": { "question": "Does an algebraic design-derivation framework produce cleaner generated code than a semantic design framework when the builder must derive the design before coding?", "hypothesis": "Deriving substrate identities, capabilities, and their composition with algebraic laws and proofs improve code generation quality by reducing inference via proofs which reuse repeated reasoning", "prediction": "The algebraic arm's quality score exceeds the semantic arm's by at least 0.1.", "falsifier": "The algebraic arm's quality score is no higher than the semantic arm's. A positive difference below 0.1 is inconclusive." }, "decision_rule": { "metric": "score", "not_supported_if_difference_at_most": 0.099, "supported_if_difference_at_least": 0.1 }, "original_decision_rule": null, "stages": [ "initial-build", "telemetry-add", "parity-substrate", "process-ontology", "check-no-write", "concurrent-submissions", "rust-conversion" ], "arms": [ { "name": "control", "form": "null_form", "control": true }, { "name": "algebraic", "form": "algebraic_solver", "control": false } ], "hypothesis_arms": [ "algebraic" ], "limits": { "confidence_threshold": 0.9, "spending_limit_usd": 1.0, "time_limit_s": 600000 }, "varies": [], "runs": [ { "run_id": "run-20260923T235842Z-4979eeee", "experiment_id": "E020", "started_at": "2026-09-23T23:58:42.608614+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "e01402ca8d21cba2dcee1ef93734c00992151556" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "61310b7d8710ea3616738c3698a03b383aef5c94", "arms": [ "control", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1001791.0, "of": null }, "system.elapsed_s": { "value": 376.774, "of": null }, "environment.tests_passed": { "value": 4.0, "of": 4.0 }, "environment.lines": { "value": 379.0, "of": null }, "environment.complexity_mean": { "value": 5.25, "of": null } }, "commands": { "all": 29, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1555714.0, "of": null }, "system.elapsed_s": { "value": 457.813, "of": null }, "system.solver_obligations": { "value": 54.0, "of": null }, "system.solver_established": { "value": 54.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 4.0, "of": 4.0 }, "environment.lines": { "value": 501.0, "of": null }, "environment.complexity_mean": { "value": 7.541666666666667, "of": null } }, "commands": { "all": 28, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": -0.04, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1776764.0, "of": null }, "system.elapsed_s": { "value": 466.684, "of": null }, "environment.tests_passed": { "value": 9.0, "of": 9.0 }, "environment.lines": { "value": 613.0, "of": null }, "environment.complexity_mean": { "value": 6.0476190476190474, "of": null } }, "commands": { "all": 31, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3827071.0, "of": null }, "system.elapsed_s": { "value": 698.477, "of": null }, "system.solver_obligations": { "value": 78.0, "of": null }, "system.solver_established": { "value": 78.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 14.0, "of": 14.0 }, "environment.lines": { "value": 827.0, "of": null }, "environment.complexity_mean": { "value": 7.6, "of": null } }, "commands": { "all": 70, "z3": 7 } } }, "contrasts": { "algebraic": { "difference": -0.06, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 9069019.0, "of": null }, "system.elapsed_s": { "value": 1298.293, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1114.0, "of": null }, "environment.complexity_mean": { "value": 5.1558441558441555, "of": null } }, "commands": { "all": 91, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 6301165.0, "of": null }, "system.elapsed_s": { "value": 976.761, "of": null }, "system.solver_obligations": { "value": 96.0, "of": null }, "system.solver_established": { "value": 96.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 20.0, "of": 20.0 }, "environment.lines": { "value": 1260.0, "of": null }, "environment.complexity_mean": { "value": 8.548387096774194, "of": null } }, "commands": { "all": 73, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": -0.02, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4461895.0, "of": null }, "system.elapsed_s": { "value": 1058.996, "of": null }, "environment.tests_passed": { "value": 23.0, "of": 24.0 }, "environment.lines": { "value": 1251.0, "of": null }, "environment.complexity_mean": { "value": 5.882352941176471, "of": null } }, "commands": { "all": 57, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4122365.0, "of": null }, "system.elapsed_s": { "value": 782.235, "of": null }, "system.solver_obligations": { "value": 102.0, "of": null }, "system.solver_established": { "value": 102.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 1424.0, "of": null }, "environment.complexity_mean": { "value": 8.084507042253522, "of": null } }, "commands": { "all": 56, "z3": 0 } } }, "contrasts": { "algebraic": { "difference": -0.04, "verdict": "not_supported" } } }, { "stage": 5, "name": "check-no-write", "arms": { "control": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3956110.0, "of": null }, "system.elapsed_s": { "value": 811.722, "of": null }, "environment.tests_passed": { "value": 28.0, "of": 28.0 }, "environment.lines": { "value": 1405.0, "of": null }, "environment.complexity_mean": { "value": 5.849462365591398, "of": null } }, "commands": { "all": 45, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5949865.0, "of": null }, "system.elapsed_s": { "value": 692.514, "of": null }, "system.solver_obligations": { "value": 114.0, "of": null }, "system.solver_established": { "value": 114.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 30.0, "of": 30.0 }, "environment.lines": { "value": 1533.0, "of": null }, "environment.complexity_mean": { "value": 7.822784810126582, "of": null } }, "commands": { "all": 45, "z3": 0 } } }, "contrasts": { "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 6, "name": "concurrent-submissions", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3299270.0, "of": null }, "system.elapsed_s": { "value": 705.794, "of": null }, "environment.tests_passed": { "value": 31.0, "of": 32.0 }, "environment.lines": { "value": 1523.0, "of": null }, "environment.complexity_mean": { "value": 5.63, "of": null } }, "commands": { "all": 42, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3814377.0, "of": null }, "system.elapsed_s": { "value": 803.302, "of": null }, "system.solver_obligations": { "value": 122.0, "of": null }, "system.solver_established": { "value": 122.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 34.0, "of": 34.0 }, "environment.lines": { "value": 1658.0, "of": null }, "environment.complexity_mean": { "value": 7.635294117647059, "of": null } }, "commands": { "all": 41, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": -0.02, "verdict": "not_supported" } } }, { "stage": 7, "name": "rust-conversion", "arms": { "control": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test integration`" } ], "measurements": { "system.total_tokens": { "value": 7640862.0, "of": null }, "system.elapsed_s": { "value": 1259.718, "of": null }, "environment.rust_tests_passed": { "value": 14.0, "of": 19.0 } }, "commands": { "all": 87, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 10791260.0, "of": null }, "system.elapsed_s": { "value": 1468.378, "of": null }, "system.solver_obligations": { "value": 139.0, "of": null }, "system.solver_established": { "value": 139.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.rust_tests_passed": { "value": 12.0, "of": 12.0 } }, "commands": { "all": 69, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 7, "scores": [ { "stage": 1, "score": 0.34 }, { "stage": 2, "score": 0.34 }, { "stage": 3, "score": 0.3 }, { "stage": 4, "score": 0.34 }, { "stage": 5, "score": 0.31 }, { "stage": 6, "score": 0.3 }, { "stage": 7, "score": null } ], "mean_score": 0.3216666666666667, "scored_stages": 6, "build_tokens": { "value": 31205711.0, "stages": 7 }, "review_tokens": { "value": 2445322.0, "stages": 7 }, "total_tokens": { "value": 33651033.0, "stages": 7 }, "build_s": { "value": 5978.105, "stages": 7 }, "review_s": { "value": 1663.741, "stages": 7 }, "total_s": { "value": 7659.996, "stages": 7 }, "final_stage": 7, "final_lines": 138.0, "final_complexity": null }, { "arm": "algebraic", "stages": 7, "scores": [ { "stage": 1, "score": 0.3 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.3 }, { "stage": 5, "score": 0.31 }, { "stage": 6, "score": 0.28 }, { "stage": 7, "score": 0.28 } ], "mean_score": 0.29000000000000004, "scored_stages": 7, "build_tokens": { "value": 36361817.0, "stages": 7 }, "review_tokens": { "value": 1959216.0, "stages": 7 }, "total_tokens": { "value": 38321033.0, "stages": 7 }, "build_s": { "value": 5880.196, "stages": 7 }, "review_s": { "value": 1439.94, "stages": 7 }, "total_s": { "value": 7336.739, "stages": 7 }, "final_stage": 7, "final_lines": 551.0, "final_complexity": null } ] }, { "run_id": "run-20260924T031833Z-a6b163d1", "experiment_id": "E020", "started_at": "2026-09-24T03:18:34.305731+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "e01402ca8d21cba2dcee1ef93734c00992151556" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.41, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 895466.0, "of": null }, "system.elapsed_s": { "value": 374.586, "of": null }, "environment.tests_passed": { "value": 2.0, "of": 2.0 }, "environment.lines": { "value": 497.0, "of": null }, "environment.complexity_mean": { "value": 5.6, "of": null } }, "commands": { "all": 22, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1985509.0, "of": null }, "system.elapsed_s": { "value": 533.587, "of": null }, "system.solver_obligations": { "value": 59.0, "of": null }, "system.solver_established": { "value": 59.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 3.0, "of": 3.0 }, "environment.lines": { "value": 478.0, "of": null }, "environment.complexity_mean": { "value": 6.3, "of": null } }, "commands": { "all": 40, "z3": 2 } } }, "contrasts": { "algebraic": { "difference": -0.11, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.35, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2223308.0, "of": null }, "system.elapsed_s": { "value": 609.716, "of": null }, "environment.tests_passed": { "value": 13.0, "of": 13.0 }, "environment.lines": { "value": 817.0, "of": null }, "environment.complexity_mean": { "value": 4.95, "of": null } }, "commands": { "all": 36, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.25, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1946832.0, "of": null }, "system.elapsed_s": { "value": 504.1, "of": null }, "system.solver_obligations": { "value": 79.0, "of": null }, "system.solver_established": { "value": 79.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 9.0, "of": 9.0 }, "environment.lines": { "value": 620.0, "of": null }, "environment.complexity_mean": { "value": 5.9523809523809526, "of": null } }, "commands": { "all": 37, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": -0.1, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2119246.0, "of": null }, "system.elapsed_s": { "value": 785.262, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1441.0, "of": null }, "environment.complexity_mean": { "value": 5.698630136986301, "of": null } }, "commands": { "all": 40, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.25, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5646897.0, "of": null }, "system.elapsed_s": { "value": 1170.783, "of": null }, "system.solver_obligations": { "value": 102.0, "of": null }, "system.solver_established": { "value": 102.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 19.0, "of": 19.0 }, "environment.lines": { "value": 1270.0, "of": null }, "environment.complexity_mean": { "value": 6.567901234567901, "of": null } }, "commands": { "all": 72, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": -0.03, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.35, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4032380.0, "of": null }, "system.elapsed_s": { "value": 755.015, "of": null }, "environment.tests_passed": { "value": 24.0, "of": 24.0 }, "environment.lines": { "value": 1587.0, "of": null }, "environment.complexity_mean": { "value": 5.6875, "of": null } }, "commands": { "all": 75, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4224980.0, "of": null }, "system.elapsed_s": { "value": 639.776, "of": null }, "system.solver_obligations": { "value": 107.0, "of": null }, "system.solver_established": { "value": 107.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 24.0, "of": 24.0 }, "environment.lines": { "value": 1410.0, "of": null }, "environment.complexity_mean": { "value": 6.351648351648351, "of": null } }, "commands": { "all": 49, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": -0.07, "verdict": "not_supported" } } }, { "stage": 5, "name": "check-no-write", "arms": { "control": { "status": "completed", "score": 0.39, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3334146.0, "of": null }, "system.elapsed_s": { "value": 731.292, "of": null }, "environment.tests_passed": { "value": 28.0, "of": 28.0 }, "environment.lines": { "value": 1728.0, "of": null }, "environment.complexity_mean": { "value": 5.655172413793103, "of": null } }, "commands": { "all": 46, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2247946.0, "of": null }, "system.elapsed_s": { "value": 461.668, "of": null }, "system.solver_obligations": { "value": 117.0, "of": null }, "system.solver_established": { "value": 117.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 27.0, "of": 27.0 }, "environment.lines": { "value": 1507.0, "of": null }, "environment.complexity_mean": { "value": 6.553191489361702, "of": null } }, "commands": { "all": 22, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": -0.11, "verdict": "not_supported" } } }, { "stage": 6, "name": "concurrent-submissions", "arms": { "control": { "status": "completed", "score": 0.35, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3603076.0, "of": null }, "system.elapsed_s": { "value": 898.292, "of": null }, "environment.tests_passed": { "value": 30.0, "of": 30.0 }, "environment.lines": { "value": 1809.0, "of": null }, "environment.complexity_mean": { "value": 5.711111111111111, "of": null } }, "commands": { "all": 55, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5103858.0, "of": null }, "system.elapsed_s": { "value": 733.567, "of": null }, "system.solver_obligations": { "value": 134.0, "of": null }, "system.solver_established": { "value": 134.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 29.0, "of": 30.0 }, "environment.lines": { "value": 1604.0, "of": null }, "environment.complexity_mean": { "value": 6.484848484848484, "of": null } }, "commands": { "all": 45, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": -0.07, "verdict": "not_supported" } } }, { "stage": 7, "name": "rust-conversion", "arms": { "control": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test workflow`" } ], "measurements": { "system.total_tokens": { "value": 5820932.0, "of": null }, "system.elapsed_s": { "value": 868.994, "of": null }, "environment.rust_tests_passed": { "value": 10.0, "of": 12.0 } }, "commands": { "all": 55, "z3": 0 } }, "algebraic": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test behavior`" } ], "measurements": { "system.total_tokens": { "value": 5415466.0, "of": null }, "system.elapsed_s": { "value": 815.367, "of": null }, "system.solver_obligations": { "value": 142.0, "of": null }, "system.solver_established": { "value": 142.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.rust_tests_passed": { "value": 0.0, "of": 6.0 } }, "commands": { "all": 61, "z3": 5 } } }, "contrasts": { "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 7, "scores": [ { "stage": 1, "score": 0.41 }, { "stage": 2, "score": 0.35 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.35 }, { "stage": 5, "score": 0.39 }, { "stage": 6, "score": 0.35 }, { "stage": 7, "score": null } ], "mean_score": 0.355, "scored_stages": 6, "build_tokens": { "value": 22028554.0, "stages": 7 }, "review_tokens": { "value": 2045340.0, "stages": 7 }, "total_tokens": { "value": 24073894.0, "stages": 7 }, "build_s": { "value": 5023.222, "stages": 7 }, "review_s": { "value": 1020.125, "stages": 7 }, "total_s": { "value": 6056.388, "stages": 7 }, "final_stage": 7, "final_lines": 212.0, "final_complexity": null }, { "arm": "algebraic", "stages": 7, "scores": [ { "stage": 1, "score": 0.3 }, { "stage": 2, "score": 0.25 }, { "stage": 3, "score": 0.25 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.28 }, { "stage": 6, "score": 0.28 }, { "stage": 7, "score": null } ], "mean_score": 0.2733333333333334, "scored_stages": 6, "build_tokens": { "value": 26571488.0, "stages": 7 }, "review_tokens": { "value": 1733919.0, "stages": 7 }, "total_tokens": { "value": 28305407.0, "stages": 7 }, "build_s": { "value": 4859.53, "stages": 7 }, "review_s": { "value": 927.643, "stages": 7 }, "total_s": { "value": 5802.13, "stages": 7 }, "final_stage": 7, "final_lines": 288.0, "final_complexity": null } ] }, { "run_id": "run-20260924T033220Z-8386576e", "experiment_id": "E026", "started_at": "2026-09-24T03:32:21.208027+00:00", "status": "failed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "e01402ca8d21cba2dcee1ef93734c00992151556" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "algebraic" ], "stages": [ { "stage": 1, "name": "initial-build", "arms": { "control": { "status": "completed", "score": 0.32, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 988720.0, "of": null }, "system.elapsed_s": { "value": 487.009, "of": null }, "environment.tests_passed": { "value": 2.0, "of": 2.0 }, "environment.lines": { "value": 519.0, "of": null }, "environment.complexity_mean": { "value": 5.785714285714286, "of": null } }, "commands": { "all": 30, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.41, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1863574.0, "of": null }, "system.elapsed_s": { "value": 600.873, "of": null }, "system.solver_obligations": { "value": 65.0, "of": null }, "system.solver_established": { "value": 64.0, "of": null }, "system.solver_refuted": { "value": 1.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 5.0, "of": 5.0 }, "environment.lines": { "value": 624.0, "of": null }, "environment.complexity_mean": { "value": 5.966666666666667, "of": null } }, "commands": { "all": 30, "z3": 3 } } }, "contrasts": { "algebraic": { "difference": 0.09, "verdict": "not_supported" } } }, { "stage": 2, "name": "telemetry-add", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1599864.0, "of": null }, "system.elapsed_s": { "value": 432.083, "of": null }, "environment.tests_passed": { "value": 10.0, "of": 10.0 }, "environment.lines": { "value": 850.0, "of": null }, "environment.complexity_mean": { "value": 6.086956521739131, "of": null } }, "commands": { "all": 27, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2671781.0, "of": null }, "system.elapsed_s": { "value": 652.527, "of": null }, "system.solver_obligations": { "value": 78.0, "of": null }, "system.solver_established": { "value": 77.0, "of": null }, "system.solver_refuted": { "value": 1.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 17.0, "of": 17.0 }, "environment.lines": { "value": 968.0, "of": null }, "environment.complexity_mean": { "value": 5.795918367346939, "of": null } }, "commands": { "all": 32, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 3, "name": "parity-substrate", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4336600.0, "of": null }, "system.elapsed_s": { "value": 1156.752, "of": null }, "environment.tests_passed": { "value": 18.0, "of": 18.0 }, "environment.lines": { "value": 1340.0, "of": null }, "environment.complexity_mean": { "value": 6.626666666666667, "of": null } }, "commands": { "all": 41, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5141874.0, "of": null }, "system.elapsed_s": { "value": 1067.68, "of": null }, "system.solver_obligations": { "value": 104.0, "of": null }, "system.solver_established": { "value": 101.0, "of": null }, "system.solver_refuted": { "value": 3.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 1616.0, "of": null }, "environment.complexity_mean": { "value": 6.35, "of": null } }, "commands": { "all": 44, "z3": 2 } } }, "contrasts": { "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 4, "name": "process-ontology", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 7348258.0, "of": null }, "system.elapsed_s": { "value": 1521.465, "of": null }, "environment.tests_passed": { "value": 24.0, "of": 24.0 }, "environment.lines": { "value": 1574.0, "of": null }, "environment.complexity_mean": { "value": 7.247058823529412, "of": null } }, "commands": { "all": 75, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3135856.0, "of": null }, "system.elapsed_s": { "value": 592.363, "of": null }, "system.solver_obligations": { "value": 112.0, "of": null }, "system.solver_established": { "value": 108.0, "of": null }, "system.solver_refuted": { "value": 4.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 30.0, "of": 30.0 }, "environment.lines": { "value": 1764.0, "of": null }, "environment.complexity_mean": { "value": 6.620689655172414, "of": null } }, "commands": { "all": 40, "z3": 0 } } }, "contrasts": { "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 5, "name": "check-no-write", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2291750.0, "of": null }, "system.elapsed_s": { "value": 616.454, "of": null }, "environment.tests_passed": { "value": 30.0, "of": 30.0 }, "environment.lines": { "value": 1712.0, "of": null }, "environment.complexity_mean": { "value": 6.935483870967742, "of": null } }, "commands": { "all": 26, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 7180077.0, "of": null }, "system.elapsed_s": { "value": 1122.472, "of": null }, "system.solver_obligations": { "value": 124.0, "of": null }, "system.solver_established": { "value": 120.0, "of": null }, "system.solver_refuted": { "value": 4.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 34.0, "of": 34.0 }, "environment.lines": { "value": 1912.0, "of": null }, "environment.complexity_mean": { "value": 6.364583333333333, "of": null } }, "commands": { "all": 68, "z3": 1 } } }, "contrasts": { "algebraic": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 6, "name": "concurrent-submissions", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2678584.0, "of": null }, "system.elapsed_s": { "value": 668.828, "of": null }, "environment.tests_passed": { "value": 35.0, "of": 35.0 }, "environment.lines": { "value": 1892.0, "of": null }, "environment.complexity_mean": { "value": 6.609523809523809, "of": null } }, "commands": { "all": 32, "z3": 0 } }, "algebraic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4537398.0, "of": null }, "system.elapsed_s": { "value": 730.838, "of": null }, "system.solver_obligations": { "value": 141.0, "of": null }, "system.solver_established": { "value": 137.0, "of": null }, "system.solver_refuted": { "value": 4.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 39.0, "of": 39.0 }, "environment.lines": { "value": 2061.0, "of": null }, "environment.complexity_mean": { "value": 6.085714285714285, "of": null } }, "commands": { "all": 35, "z3": 0 } } }, "contrasts": { "algebraic": { "difference": -0.06, "verdict": "not_supported" } } }, { "stage": 7, "name": "rust-conversion", "arms": { "control": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--bin ontology_app`" } ], "measurements": { "system.total_tokens": { "value": 5190121.0, "of": null }, "system.elapsed_s": { "value": 942.254, "of": null }, "environment.rust_tests_passed": { "value": 9.0, "of": 13.0 } }, "commands": { "all": 51, "z3": 0 } }, "algebraic": { "status": "failed", "score": null, "failures": [ "the environment's grade has status failed" ], "errors": [ { "measurement": "rust_tests_passed", "first": "cargo test exited 101", "last": "error: test failed, to rerun pass `--test rust_behavior`" } ], "measurements": { "system.total_tokens": { "value": 3611411.0, "of": null }, "system.elapsed_s": { "value": 888.914, "of": null }, "system.solver_obligations": { "value": 159.0, "of": null }, "system.solver_established": { "value": 155.0, "of": null }, "system.solver_refuted": { "value": 4.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.rust_tests_passed": { "value": 12.0, "of": 13.0 } }, "commands": { "all": 42, "z3": 6 } } }, "contrasts": { "algebraic": { "difference": null, "verdict": "invalid" } } } ], "totals": [ { "arm": "control", "stages": 7, "scores": [ { "stage": 1, "score": 0.32 }, { "stage": 2, "score": 0.3 }, { "stage": 3, "score": 0.3 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.28 }, { "stage": 6, "score": 0.34 }, { "stage": 7, "score": null } ], "mean_score": 0.30333333333333334, "scored_stages": 6, "build_tokens": { "value": 24433897.0, "stages": 7 }, "review_tokens": { "value": 1791458.0, "stages": 7 }, "total_tokens": { "value": 26225355.0, "stages": 7 }, "build_s": { "value": 5824.911, "stages": 7 }, "review_s": { "value": 1370.97, "stages": 7 }, "total_s": { "value": 7209.128, "stages": 7 }, "final_stage": 7, "final_lines": 101.0, "final_complexity": null }, { "arm": "algebraic", "stages": 7, "scores": [ { "stage": 1, "score": 0.41 }, { "stage": 2, "score": 0.3 }, { "stage": 3, "score": 0.3 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.28 }, { "stage": 6, "score": 0.28 }, { "stage": 7, "score": null } ], "mean_score": 0.30833333333333335, "scored_stages": 6, "build_tokens": { "value": 28141971.0, "stages": 7 }, "review_tokens": { "value": 1915591.0, "stages": 7 }, "total_tokens": { "value": 30057562.0, "stages": 7 }, "build_s": { "value": 5656.396, "stages": 7 }, "review_s": { "value": 826.5129999999999, "stages": 7 }, "total_s": { "value": 6496.71, "stages": 7 }, "final_stage": 7, "final_lines": 321.0, "final_complexity": null } ] } ], "checks": [ { "kind": "contrast", "arms": [ "algebraic" ], "than": "control", "measurement": null, "require": "every", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T235842Z-4979eeee", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": -0.04, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": -0.06, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.02, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.04, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 6, "difference": -0.02, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T031833Z-a6b163d1", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": -0.11, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": -0.1, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.03, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": -0.11, "verdict": "not_supported", "passed": false }, { "stage": 6, "difference": -0.07, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T033220Z-8386576e", "arm": "algebraic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.09, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 6, "difference": -0.06, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] }, { "kind": "difference", "arms": [ "algebraic" ], "than": "semantic", "measurement": null, "require": "every", "verdict": false, "not_run": true, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260923T235842Z-4979eeee", "arm": "algebraic", "not_run": true }, { "run_id": "run-20260924T031833Z-a6b163d1", "arm": "algebraic", "not_run": true }, { "run_id": "run-20260924T033220Z-8386576e", "arm": "algebraic", "not_run": true } ] } ], "verdict": "not_supported", "readings": { "every": false, "most": false, "mean": false }, "reading_independent": true, "effect": { "algebraic": { "cells": 18, "mean": -0.0356, "min": -0.11, "max": 0.09, "differences": [ -0.04, -0.06, -0.02, -0.04, 0.0, -0.02, -0.11, -0.1, -0.03, -0.07, -0.11, -0.07, 0.09, 0.0, 0.0, 0.0, 0.0, -0.06 ] } }, "spread": { "sets": [ { "runs": [ "run-20260923T235842Z-4979eeee", "run-20260924T031833Z-a6b163d1", "run-20260924T033220Z-8386576e" ], "cells": [ { "stage": 1, "arm": "control", "scores": [ 0.34, 0.41, 0.32 ], "range": 0.09, "reaches_threshold": false }, { "stage": 1, "arm": "algebraic", "scores": [ 0.3, 0.3, 0.41 ], "range": 0.11, "reaches_threshold": true }, { "stage": 2, "arm": "control", "scores": [ 0.34, 0.35, 0.3 ], "range": 0.05, "reaches_threshold": false }, { "stage": 2, "arm": "algebraic", "scores": [ 0.28, 0.25, 0.3 ], "range": 0.05, "reaches_threshold": false }, { "stage": 3, "arm": "control", "scores": [ 0.3, 0.28, 0.3 ], "range": 0.02, "reaches_threshold": false }, { "stage": 3, "arm": "algebraic", "scores": [ 0.28, 0.25, 0.3 ], "range": 0.05, "reaches_threshold": false }, { "stage": 4, "arm": "control", "scores": [ 0.34, 0.35, 0.28 ], "range": 0.07, "reaches_threshold": false }, { "stage": 4, "arm": "algebraic", "scores": [ 0.3, 0.28, 0.28 ], "range": 0.02, "reaches_threshold": false }, { "stage": 5, "arm": "control", "scores": [ 0.31, 0.39, 0.28 ], "range": 0.11, "reaches_threshold": true }, { "stage": 5, "arm": "algebraic", "scores": [ 0.31, 0.28, 0.28 ], "range": 0.03, "reaches_threshold": false }, { "stage": 6, "arm": "control", "scores": [ 0.3, 0.35, 0.34 ], "range": 0.05, "reaches_threshold": false }, { "stage": 6, "arm": "algebraic", "scores": [ 0.28, 0.28, 0.28 ], "range": 0.0, "reaches_threshold": false } ] } ] } }, { "id": "E022-E023", "label": "Retrieval \u00b7 400k-line codebase \u00b7 5 questions \u00b7 z3-checked algebra", "experiments": [ "E022", "E023" ], "card": { "question": "In a codebase of about 400,000 lines, does a finished design tree let an agent explain one part of the application, for a planned change, with fewer tokens and less time than the code alone, without a loss of correctness? Does an algebraic tree with solver files do so more than a prose tree?", "hypothesis": "A short design tree works as an index: the agent reads the records instead of searching the code, and an algebraic tree states identity, responsibilities, and invariants more directly than prose.", "prediction": "Each design arm uses fewer total_tokens and less elapsed_s than the control in most stages, and its answer correctness is no more than 0.1 below the control's. The algebraic arm uses fewer tokens than the semantic arm.", "falsifier": "A design arm uses as many or more tokens than the control, or its correctness falls more than 0.1 below the control's. The harness verdict tests only the correctness floor: supported means the arm's score is at most 0.1 below the control's. Tokens and time are compared from the system measurements in the observer." }, "decision_rule": { "metric": "score", "supported_if_difference_at_least": 0.1, "not_supported_if_difference_at_most": 0.0999 }, "original_decision_rule": { "metric": "score", "not_supported_if_difference_at_most": -0.1001, "supported_if_difference_at_least": -0.1 }, "stages": [ "checks", "ontology", "operation", "telemetry", "composition" ], "arms": [ { "name": "control", "form": "retrieval_null", "control": true }, { "name": "semantic", "form": "retrieval_semantic", "control": false }, { "name": "algebraic_solver", "form": "retrieval_algebraic_solver", "control": false } ], "hypothesis_arms": [ "algebraic_solver", "semantic" ], "limits": { "confidence_threshold": 0.9, "spending_limit_usd": 1.0, "time_limit_s": 3600 }, "varies": [ "arms.algebraic_solver.controls.model_id", "arms.algebraic_solver.model.id", "arms.control.controls.model_id", "arms.control.model.id", "arms.semantic.controls.model_id", "arms.semantic.model.id" ], "runs": [ { "run_id": "run-20260924T023130Z-ccbee4e6", "experiment_id": "E022", "started_at": "2026-09-24T02:31:30.965567+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "1326bbd5d19c6abb55ef8850b2e4f11a693ba3d6" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "c7ac0d7811b106c4af8d045d331363dfe735e4e9" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "semantic", "algebraic_solver" ], "stages": [ { "stage": 1, "name": "checks", "arms": { "control": { "status": "completed", "score": 0.96, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 255199.0, "of": null }, "system.elapsed_s": { "value": 111.888, "of": null } }, "commands": { "all": 16, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.93, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 278048.0, "of": null }, "system.elapsed_s": { "value": 95.494, "of": null } }, "commands": { "all": 13, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.89, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 236544.0, "of": null }, "system.elapsed_s": { "value": 85.384, "of": null } }, "commands": { "all": 14, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.03, "verdict": "supported" }, "algebraic_solver": { "difference": -0.07, "verdict": "supported" } } }, { "stage": 2, "name": "ontology", "arms": { "control": { "status": "completed", "score": 0.89, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 194763.0, "of": null }, "system.elapsed_s": { "value": 93.193, "of": null } }, "commands": { "all": 12, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.89, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 317943.0, "of": null }, "system.elapsed_s": { "value": 119.607, "of": null } }, "commands": { "all": 11, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.82, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 265836.0, "of": null }, "system.elapsed_s": { "value": 103.862, "of": null } }, "commands": { "all": 11, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.0, "verdict": "supported" }, "algebraic_solver": { "difference": -0.07, "verdict": "supported" } } }, { "stage": 3, "name": "operation", "arms": { "control": { "status": "completed", "score": 1.0, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 251817.0, "of": null }, "system.elapsed_s": { "value": 114.144, "of": null } }, "commands": { "all": 15, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.89, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 337475.0, "of": null }, "system.elapsed_s": { "value": 140.396, "of": null } }, "commands": { "all": 14, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.96, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 297417.0, "of": null }, "system.elapsed_s": { "value": 115.402, "of": null } }, "commands": { "all": 17, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.11, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.04, "verdict": "supported" } } }, { "stage": 4, "name": "telemetry", "arms": { "control": { "status": "completed", "score": 1.0, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 253712.0, "of": null }, "system.elapsed_s": { "value": 123.478, "of": null } }, "commands": { "all": 14, "z3": 0 } }, "semantic": { "status": "completed", "score": 1.0, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 275342.0, "of": null }, "system.elapsed_s": { "value": 99.14, "of": null } }, "commands": { "all": 24, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.93, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 415084.0, "of": null }, "system.elapsed_s": { "value": 162.854, "of": null } }, "commands": { "all": 19, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.0, "verdict": "supported" }, "algebraic_solver": { "difference": -0.07, "verdict": "supported" } } }, { "stage": 5, "name": "composition", "arms": { "control": { "status": "completed", "score": 0.82, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 278797.0, "of": null }, "system.elapsed_s": { "value": 128.251, "of": null } }, "commands": { "all": 18, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.82, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 348639.0, "of": null }, "system.elapsed_s": { "value": 117.258, "of": null } }, "commands": { "all": 14, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.86, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 294483.0, "of": null }, "system.elapsed_s": { "value": 123.16, "of": null } }, "commands": { "all": 14, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.0, "verdict": "supported" }, "algebraic_solver": { "difference": 0.04, "verdict": "supported" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.96 }, { "stage": 2, "score": 0.89 }, { "stage": 3, "score": 1.0 }, { "stage": 4, "score": 1.0 }, { "stage": 5, "score": 0.82 } ], "mean_score": 0.9339999999999999, "scored_stages": 5, "build_tokens": { "value": 1234288.0, "stages": 5 }, "review_tokens": { "value": 1362868.0, "stages": 5 }, "total_tokens": { "value": 2597156.0, "stages": 5 }, "build_s": { "value": 571.192, "stages": 5 }, "review_s": { "value": 437.27, "stages": 5 }, "total_s": { "value": 1032.733, "stages": 5 }, "final_stage": 5, "final_lines": null, "final_complexity": null }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.93 }, { "stage": 2, "score": 0.89 }, { "stage": 3, "score": 0.89 }, { "stage": 4, "score": 1.0 }, { "stage": 5, "score": 0.82 } ], "mean_score": 0.906, "scored_stages": 5, "build_tokens": { "value": 1557447.0, "stages": 5 }, "review_tokens": { "value": 1658232.0, "stages": 5 }, "total_tokens": { "value": 3215679.0, "stages": 5 }, "build_s": { "value": 572.1370000000001, "stages": 5 }, "review_s": { "value": 514.845, "stages": 5 }, "total_s": { "value": 1109.876, "stages": 5 }, "final_stage": 5, "final_lines": null, "final_complexity": null }, { "arm": "algebraic_solver", "stages": 5, "scores": [ { "stage": 1, "score": 0.89 }, { "stage": 2, "score": 0.82 }, { "stage": 3, "score": 0.96 }, { "stage": 4, "score": 0.93 }, { "stage": 5, "score": 0.86 } ], "mean_score": 0.892, "scored_stages": 5, "build_tokens": { "value": 1509364.0, "stages": 5 }, "review_tokens": { "value": 1854895.0, "stages": 5 }, "total_tokens": { "value": 3364259.0, "stages": 5 }, "build_s": { "value": 590.879, "stages": 5 }, "review_s": { "value": 504.788, "stages": 5 }, "total_s": { "value": 1118.162, "stages": 5 }, "final_stage": 5, "final_lines": null, "final_complexity": null } ] }, { "run_id": "run-20260924T023152Z-aa033b06", "experiment_id": "E023", "started_at": "2026-09-24T02:31:52.639507+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-luna", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "1326bbd5d19c6abb55ef8850b2e4f11a693ba3d6" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "c7ac0d7811b106c4af8d045d331363dfe735e4e9" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "semantic", "algebraic_solver" ], "stages": [ { "stage": 1, "name": "checks", "arms": { "control": { "status": "completed", "score": 0.86, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 219684.0, "of": null }, "system.elapsed_s": { "value": 56.105, "of": null } }, "commands": { "all": 6, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.89, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 193711.0, "of": null }, "system.elapsed_s": { "value": 70.9, "of": null } }, "commands": { "all": 5, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.82, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 175155.0, "of": null }, "system.elapsed_s": { "value": 73.737, "of": null } }, "commands": { "all": 5, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.03, "verdict": "supported" }, "algebraic_solver": { "difference": -0.04, "verdict": "supported" } } }, { "stage": 2, "name": "ontology", "arms": { "control": { "status": "completed", "score": 0.82, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 269422.0, "of": null }, "system.elapsed_s": { "value": 77.865, "of": null } }, "commands": { "all": 7, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.75, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 154494.0, "of": null }, "system.elapsed_s": { "value": 102.594, "of": null } }, "commands": { "all": 4, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.75, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 311870.0, "of": null }, "system.elapsed_s": { "value": 81.686, "of": null } }, "commands": { "all": 8, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.07, "verdict": "supported" }, "algebraic_solver": { "difference": -0.07, "verdict": "supported" } } }, { "stage": 3, "name": "operation", "arms": { "control": { "status": "completed", "score": 0.93, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 319229.0, "of": null }, "system.elapsed_s": { "value": 130.797, "of": null } }, "commands": { "all": 8, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.82, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 252872.0, "of": null }, "system.elapsed_s": { "value": 92.094, "of": null } }, "commands": { "all": 7, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.75, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 215427.0, "of": null }, "system.elapsed_s": { "value": 81.878, "of": null } }, "commands": { "all": 5, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.11, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.18, "verdict": "not_supported" } } }, { "stage": 4, "name": "telemetry", "arms": { "control": { "status": "completed", "score": 0.71, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 203868.0, "of": null }, "system.elapsed_s": { "value": 72.901, "of": null } }, "commands": { "all": 7, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.93, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 117945.0, "of": null }, "system.elapsed_s": { "value": 69.12, "of": null } }, "commands": { "all": 8, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.64, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 206384.0, "of": null }, "system.elapsed_s": { "value": 74.375, "of": null } }, "commands": { "all": 14, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.22, "verdict": "supported" }, "algebraic_solver": { "difference": -0.07, "verdict": "supported" } } }, { "stage": 5, "name": "composition", "arms": { "control": { "status": "completed", "score": 0.68, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 332929.0, "of": null }, "system.elapsed_s": { "value": 96.427, "of": null } }, "commands": { "all": 8, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.71, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 326170.0, "of": null }, "system.elapsed_s": { "value": 138.304, "of": null } }, "commands": { "all": 6, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.64, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 324954.0, "of": null }, "system.elapsed_s": { "value": 104.165, "of": null } }, "commands": { "all": 8, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.03, "verdict": "supported" }, "algebraic_solver": { "difference": -0.04, "verdict": "supported" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.86 }, { "stage": 2, "score": 0.82 }, { "stage": 3, "score": 0.93 }, { "stage": 4, "score": 0.71 }, { "stage": 5, "score": 0.68 } ], "mean_score": 0.8, "scored_stages": 5, "build_tokens": { "value": 1345132.0, "stages": 5 }, "review_tokens": { "value": 1455415.0, "stages": 5 }, "total_tokens": { "value": 2800547.0, "stages": 5 }, "build_s": { "value": 434.279, "stages": 5 }, "review_s": { "value": 483.379, "stages": 5 }, "total_s": { "value": 940.2860000000001, "stages": 5 }, "final_stage": 5, "final_lines": null, "final_complexity": null }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.89 }, { "stage": 2, "score": 0.75 }, { "stage": 3, "score": 0.82 }, { "stage": 4, "score": 0.93 }, { "stage": 5, "score": 0.71 } ], "mean_score": 0.82, "scored_stages": 5, "build_tokens": { "value": 1045192.0, "stages": 5 }, "review_tokens": { "value": 1603922.0, "stages": 5 }, "total_tokens": { "value": 2649114.0, "stages": 5 }, "build_s": { "value": 473.213, "stages": 5 }, "review_s": { "value": 515.044, "stages": 5 }, "total_s": { "value": 1012.557, "stages": 5 }, "final_stage": 5, "final_lines": null, "final_complexity": null }, { "arm": "algebraic_solver", "stages": 5, "scores": [ { "stage": 1, "score": 0.82 }, { "stage": 2, "score": 0.75 }, { "stage": 3, "score": 0.75 }, { "stage": 4, "score": 0.64 }, { "stage": 5, "score": 0.64 } ], "mean_score": 0.72, "scored_stages": 5, "build_tokens": { "value": 1233790.0, "stages": 5 }, "review_tokens": { "value": 1802171.0, "stages": 5 }, "total_tokens": { "value": 3035961.0, "stages": 5 }, "build_s": { "value": 416.058, "stages": 5 }, "review_s": { "value": 507.13599999999997, "stages": 5 }, "total_s": { "value": 947.054, "stages": 5 }, "final_stage": 5, "final_lines": null, "final_complexity": null } ] } ], "checks": [ { "kind": "contrast", "arms": [ "algebraic_solver", "semantic" ], "than": "control", "measurement": null, "require": "every", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.04, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": 0.04, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "semantic", "not_run": false, "stages": [ { "stage": 1, "difference": -0.03, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.11, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": 0.0, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "difference": -0.04, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.18, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": -0.04, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "semantic", "not_run": false, "stages": [ { "stage": 1, "difference": 0.03, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": -0.11, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.22, "verdict": "supported", "passed": true }, { "stage": 5, "difference": 0.03, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] }, { "kind": "fewer", "arms": [ "algebraic_solver", "semantic" ], "than": "control", "measurement": "system.total_tokens", "require": "most", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "value": 236544.0, "than": 255199.0, "passed": true }, { "stage": 2, "value": 265836.0, "than": 194763.0, "passed": false }, { "stage": 3, "value": 297417.0, "than": 251817.0, "passed": false }, { "stage": 4, "value": 415084.0, "than": 253712.0, "passed": false }, { "stage": 5, "value": 294483.0, "than": 278797.0, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "semantic", "not_run": false, "stages": [ { "stage": 1, "value": 278048.0, "than": 255199.0, "passed": false }, { "stage": 2, "value": 317943.0, "than": 194763.0, "passed": false }, { "stage": 3, "value": 337475.0, "than": 251817.0, "passed": false }, { "stage": 4, "value": 275342.0, "than": 253712.0, "passed": false }, { "stage": 5, "value": 348639.0, "than": 278797.0, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "value": 175155.0, "than": 219684.0, "passed": true }, { "stage": 2, "value": 311870.0, "than": 269422.0, "passed": false }, { "stage": 3, "value": 215427.0, "than": 319229.0, "passed": true }, { "stage": 4, "value": 206384.0, "than": 203868.0, "passed": false }, { "stage": 5, "value": 324954.0, "than": 332929.0, "passed": true } ], "passed": { "every": false, "most": true, "mean": true } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "semantic", "not_run": false, "stages": [ { "stage": 1, "value": 193711.0, "than": 219684.0, "passed": true }, { "stage": 2, "value": 154494.0, "than": 269422.0, "passed": true }, { "stage": 3, "value": 252872.0, "than": 319229.0, "passed": true }, { "stage": 4, "value": 117945.0, "than": 203868.0, "passed": true }, { "stage": 5, "value": 326170.0, "than": 332929.0, "passed": true } ], "passed": { "every": true, "most": true, "mean": true } } ] }, { "kind": "fewer", "arms": [ "algebraic_solver", "semantic" ], "than": "control", "measurement": "system.elapsed_s", "require": "most", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "value": 85.384, "than": 111.888, "passed": true }, { "stage": 2, "value": 103.862, "than": 93.193, "passed": false }, { "stage": 3, "value": 115.402, "than": 114.144, "passed": false }, { "stage": 4, "value": 162.854, "than": 123.478, "passed": false }, { "stage": 5, "value": 123.16, "than": 128.251, "passed": true } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "semantic", "not_run": false, "stages": [ { "stage": 1, "value": 95.494, "than": 111.888, "passed": true }, { "stage": 2, "value": 119.607, "than": 93.193, "passed": false }, { "stage": 3, "value": 140.396, "than": 114.144, "passed": false }, { "stage": 4, "value": 99.14, "than": 123.478, "passed": true }, { "stage": 5, "value": 117.258, "than": 128.251, "passed": true } ], "passed": { "every": false, "most": true, "mean": false } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "value": 73.737, "than": 56.105, "passed": false }, { "stage": 2, "value": 81.686, "than": 77.865, "passed": false }, { "stage": 3, "value": 81.878, "than": 130.797, "passed": true }, { "stage": 4, "value": 74.375, "than": 72.901, "passed": false }, { "stage": 5, "value": 104.165, "than": 96.427, "passed": false } ], "passed": { "every": false, "most": false, "mean": true } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "semantic", "not_run": false, "stages": [ { "stage": 1, "value": 70.9, "than": 56.105, "passed": false }, { "stage": 2, "value": 102.594, "than": 77.865, "passed": false }, { "stage": 3, "value": 92.094, "than": 130.797, "passed": true }, { "stage": 4, "value": 69.12, "than": 72.901, "passed": true }, { "stage": 5, "value": 138.304, "than": 96.427, "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] }, { "kind": "fewer", "arms": [ "algebraic_solver" ], "than": "semantic", "measurement": "system.total_tokens", "require": "most", "verdict": true, "not_run": false, "passed": true, "readings": { "every": false, "most": true, "mean": false }, "results": [ { "run_id": "run-20260924T023130Z-ccbee4e6", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "value": 236544.0, "than": 278048.0, "passed": true }, { "stage": 2, "value": 265836.0, "than": 317943.0, "passed": true }, { "stage": 3, "value": 297417.0, "than": 337475.0, "passed": true }, { "stage": 4, "value": 415084.0, "than": 275342.0, "passed": false }, { "stage": 5, "value": 294483.0, "than": 348639.0, "passed": true } ], "passed": { "every": false, "most": true, "mean": true } }, { "run_id": "run-20260924T023152Z-aa033b06", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "value": 175155.0, "than": 193711.0, "passed": true }, { "stage": 2, "value": 311870.0, "than": 154494.0, "passed": false }, { "stage": 3, "value": 215427.0, "than": 252872.0, "passed": true }, { "stage": 4, "value": 206384.0, "than": 117945.0, "passed": false }, { "stage": 5, "value": 324954.0, "than": 326170.0, "passed": true } ], "passed": { "every": false, "most": true, "mean": false } } ] } ], "verdict": "not_supported", "readings": { "every": false, "most": false, "mean": false }, "reading_independent": true, "effect": { "algebraic_solver": { "cells": 10, "mean": -0.061, "min": -0.18, "max": 0.04, "differences": [ -0.07, -0.07, -0.04, -0.07, 0.04, -0.04, -0.07, -0.18, -0.07, -0.04 ] }, "semantic": { "cells": 10, "mean": -0.004, "min": -0.11, "max": 0.22, "differences": [ -0.03, 0.0, -0.11, 0.0, 0.0, 0.03, -0.07, -0.11, 0.22, 0.03 ] } }, "spread": { "sets": [] } }, { "id": "E024", "label": "Generation \u00b7 400k-line codebase \u00b7 5 imperative stages \u00b7 z3-checked algebra", "experiments": [ "E024" ], "card": { "question": "In a codebase of about 400,000 lines, does an algebraic design-derivation framework with solver-checked proofs produce cleaner code than a semantic framework or none, when the builder extends an application its design records already describe?", "hypothesis": "At a larger codebase size, the algebraic laws and proofs of the existing substrates let the builder place each change with less inference, so its changes keep ownership, boundaries, and contracts intact.", "prediction": "The algebraic_solver arm's quality score exceeds the control's by at least 0.1 in each stage; tokens and time are recorded as secondary measures.", "falsifier": "The algebraic_solver arm's quality score is no higher than the control's. A positive difference below 0.1 is inconclusive." }, "decision_rule": { "metric": "score", "not_supported_if_difference_at_most": 0.099, "supported_if_difference_at_least": 0.1 }, "original_decision_rule": null, "stages": [ "duplicate-name-check", "entry-supersession", "revise-information", "agent-token-telemetry", "ontology-audit" ], "arms": [ { "name": "control", "form": "null_form", "control": true }, { "name": "semantic", "form": "semantic_finished", "control": false }, { "name": "algebraic_solver", "form": "algebraic_solver_finished", "control": false } ], "hypothesis_arms": [ "algebraic_solver" ], "limits": { "confidence_threshold": 0.9, "spending_limit_usd": 1.0, "time_limit_s": 600000 }, "varies": [], "runs": [ { "run_id": "run-20260924T013909Z-79df517b", "experiment_id": "E024", "started_at": "2026-09-24T01:39:09.672587+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "3c3ed0d2ea393873dd2586533ce85643f7123c59" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "semantic", "algebraic_solver" ], "stages": [ { "stage": 1, "name": "duplicate-name-check", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1553478.0, "of": null }, "system.elapsed_s": { "value": 546.017, "of": null }, "environment.tests_passed": { "value": 27.0, "of": 27.0 }, "environment.lines": { "value": 401041.0, "of": null }, "environment.complexity_mean": { "value": 3.265105897009967, "of": null } }, "commands": { "all": 24, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2616917.0, "of": null }, "system.elapsed_s": { "value": 636.192, "of": null }, "environment.tests_passed": { "value": 26.0, "of": 26.0 }, "environment.lines": { "value": 401010.0, "of": null }, "environment.complexity_mean": { "value": 3.265368639667705, "of": null } }, "commands": { "all": 42, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.32, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3620133.0, "of": null }, "system.elapsed_s": { "value": 904.824, "of": null }, "system.solver_obligations": { "value": 118.0, "of": null }, "system.solver_established": { "value": 118.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 401026.0, "of": null }, "environment.complexity_mean": { "value": 3.26521287642783, "of": null } }, "commands": { "all": 54, "z3": 0 } } }, "contrasts": { "semantic": { "difference": -0.03, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.02, "verdict": "not_supported" } } }, { "stage": 2, "name": "entry-supersession", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3747078.0, "of": null }, "system.elapsed_s": { "value": 959.5, "of": null }, "environment.tests_passed": { "value": 45.0, "of": 45.0 }, "environment.lines": { "value": 401330.0, "of": null }, "environment.complexity_mean": { "value": 3.2662553147360778, "of": null } }, "commands": { "all": 55, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 5107176.0, "of": null }, "system.elapsed_s": { "value": 871.829, "of": null }, "environment.tests_passed": { "value": 45.0, "of": 45.0 }, "environment.lines": { "value": 401334.0, "of": null }, "environment.complexity_mean": { "value": 3.2655431682654914, "of": null } }, "commands": { "all": 47, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3499895.0, "of": null }, "system.elapsed_s": { "value": 814.186, "of": null }, "system.solver_obligations": { "value": 140.0, "of": null }, "system.solver_established": { "value": 140.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 42.0, "of": 42.0 }, "environment.lines": { "value": 401307.0, "of": null }, "environment.complexity_mean": { "value": 3.266075503007675, "of": null } }, "commands": { "all": 57, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.0, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.02, "verdict": "not_supported" } } }, { "stage": 3, "name": "revise-information", "arms": { "control": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1596482.0, "of": null }, "system.elapsed_s": { "value": 457.612, "of": null }, "environment.tests_passed": { "value": 50.0, "of": 50.0 }, "environment.lines": { "value": 401487.0, "of": null }, "environment.complexity_mean": { "value": 3.2663488444398383, "of": null } }, "commands": { "all": 33, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.39, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2881128.0, "of": null }, "system.elapsed_s": { "value": 579.829, "of": null }, "environment.tests_passed": { "value": 50.0, "of": 50.0 }, "environment.lines": { "value": 401471.0, "of": null }, "environment.complexity_mean": { "value": 3.2663280116110305, "of": null } }, "commands": { "all": 45, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3222377.0, "of": null }, "system.elapsed_s": { "value": 651.605, "of": null }, "system.solver_obligations": { "value": 156.0, "of": null }, "system.solver_established": { "value": 156.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 46.0, "of": 46.0 }, "environment.lines": { "value": 401448.0, "of": null }, "environment.complexity_mean": { "value": 3.266587186398507, "of": null } }, "commands": { "all": 55, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.08, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.03, "verdict": "not_supported" } } }, { "stage": 4, "name": "agent-token-telemetry", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1551241.0, "of": null }, "system.elapsed_s": { "value": 569.517, "of": null }, "environment.tests_passed": { "value": 54.0, "of": 54.0 }, "environment.lines": { "value": 401630.0, "of": null }, "environment.complexity_mean": { "value": 3.2677785259232404, "of": null } }, "commands": { "all": 18, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3517972.0, "of": null }, "system.elapsed_s": { "value": 576.041, "of": null }, "environment.tests_passed": { "value": 56.0, "of": 56.0 }, "environment.lines": { "value": 401619.0, "of": null }, "environment.complexity_mean": { "value": 3.268134715025907, "of": null } }, "commands": { "all": 27, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2057233.0, "of": null }, "system.elapsed_s": { "value": 627.744, "of": null }, "system.solver_obligations": { "value": 172.0, "of": null }, "system.solver_established": { "value": 172.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 50.0, "of": 50.0 }, "environment.lines": { "value": 401588.0, "of": null }, "environment.complexity_mean": { "value": 3.267720207253886, "of": null } }, "commands": { "all": 37, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.0, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 5, "name": "ontology-audit", "arms": { "control": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1566009.0, "of": null }, "system.elapsed_s": { "value": 497.923, "of": null }, "environment.tests_passed": { "value": 66.0, "of": 66.0 }, "environment.lines": { "value": 401979.0, "of": null }, "environment.complexity_mean": { "value": 3.269764072847682, "of": null } }, "commands": { "all": 35, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3034095.0, "of": null }, "system.elapsed_s": { "value": 679.612, "of": null }, "environment.tests_passed": { "value": 70.0, "of": 70.0 }, "environment.lines": { "value": 402029.0, "of": null }, "environment.complexity_mean": { "value": 3.2714145230578127, "of": null } }, "commands": { "all": 42, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2526886.0, "of": null }, "system.elapsed_s": { "value": 579.3, "of": null }, "system.solver_obligations": { "value": 222.0, "of": null }, "system.solver_established": { "value": 222.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 60.0, "of": 60.0 }, "environment.lines": { "value": 401941.0, "of": null }, "environment.complexity_mean": { "value": 3.2702660730924524, "of": null } }, "commands": { "all": 45, "z3": 2 } } }, "contrasts": { "semantic": { "difference": -0.04, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.06, "verdict": "not_supported" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.34 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.31 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.34 } ], "mean_score": 0.31, "scored_stages": 5, "build_tokens": { "value": 10014288.0, "stages": 5 }, "review_tokens": { "value": 1969435.0, "stages": 5 }, "total_tokens": { "value": 11983723.0, "stages": 5 }, "build_s": { "value": 3030.799, "stages": 5 }, "review_s": { "value": 912.826, "stages": 5 }, "total_s": { "value": 3963.808, "stages": 5 }, "final_stage": 5, "final_lines": 401979.0, "final_complexity": 3.269764072847682 }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.31 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.39 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.3 } ], "mean_score": 0.312, "scored_stages": 5, "build_tokens": { "value": 17157288.0, "stages": 5 }, "review_tokens": { "value": 2033956.0, "stages": 5 }, "total_tokens": { "value": 19191244.0, "stages": 5 }, "build_s": { "value": 3343.709, "stages": 5 }, "review_s": { "value": 873.933, "stages": 5 }, "total_s": { "value": 4237.572, "stages": 5 }, "final_stage": 5, "final_lines": 402029.0, "final_complexity": 3.2714145230578127 }, { "arm": "algebraic_solver", "stages": 5, "scores": [ { "stage": 1, "score": 0.32 }, { "stage": 2, "score": 0.3 }, { "stage": 3, "score": 0.34 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.28 } ], "mean_score": 0.304, "scored_stages": 5, "build_tokens": { "value": 14926524.0, "stages": 5 }, "review_tokens": { "value": 1605995.0, "stages": 5 }, "total_tokens": { "value": 16532519.0, "stages": 5 }, "build_s": { "value": 3578.382, "stages": 5 }, "review_s": { "value": 840.29, "stages": 5 }, "total_s": { "value": 4439.804, "stages": 5 }, "final_stage": 5, "final_lines": 401941.0, "final_complexity": 3.2702660730924524 } ] }, { "run_id": "run-20260924T013934Z-a085b261", "experiment_id": "E024", "started_at": "2026-09-24T01:39:34.796394+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "3c3ed0d2ea393873dd2586533ce85643f7123c59" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "semantic", "algebraic_solver" ], "stages": [ { "stage": 1, "name": "duplicate-name-check", "arms": { "control": { "status": "completed", "score": 0.36, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1073163.0, "of": null }, "system.elapsed_s": { "value": 469.084, "of": null }, "environment.tests_passed": { "value": 23.0, "of": 25.0 }, "environment.lines": { "value": 401022.0, "of": null }, "environment.complexity_mean": { "value": 3.265070875954099, "of": null } }, "commands": { "all": 31, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.41, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2574634.0, "of": null }, "system.elapsed_s": { "value": 490.831, "of": null }, "environment.tests_passed": { "value": 23.0, "of": 25.0 }, "environment.lines": { "value": 401032.0, "of": null }, "environment.complexity_mean": { "value": 3.264821929187, "of": null } }, "commands": { "all": 25, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.35, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3655861.0, "of": null }, "system.elapsed_s": { "value": 868.605, "of": null }, "system.solver_obligations": { "value": 120.0, "of": null }, "system.solver_established": { "value": 120.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 400973.0, "of": null }, "environment.complexity_mean": { "value": 3.265240419565895, "of": null } }, "commands": { "all": 39, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.05, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.01, "verdict": "not_supported" } } }, { "stage": 2, "name": "entry-supersession", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3723228.0, "of": null }, "system.elapsed_s": { "value": 747.399, "of": null }, "environment.tests_passed": { "value": 42.0, "of": 42.0 }, "environment.lines": { "value": 401328.0, "of": null }, "environment.complexity_mean": { "value": 3.265791930297687, "of": null } }, "commands": { "all": 35, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3675250.0, "of": null }, "system.elapsed_s": { "value": 870.127, "of": null }, "environment.tests_passed": { "value": 39.0, "of": 39.0 }, "environment.lines": { "value": 401252.0, "of": null }, "environment.complexity_mean": { "value": 3.266455729031589, "of": null } }, "commands": { "all": 53, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3457638.0, "of": null }, "system.elapsed_s": { "value": 686.056, "of": null }, "system.solver_obligations": { "value": 140.0, "of": null }, "system.solver_established": { "value": 140.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 41.0, "of": 41.0 }, "environment.lines": { "value": 401285.0, "of": null }, "environment.complexity_mean": { "value": 3.2661549631780935, "of": null } }, "commands": { "all": 32, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.01, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.01, "verdict": "not_supported" } } }, { "stage": 3, "name": "revise-information", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1760862.0, "of": null }, "system.elapsed_s": { "value": 523.173, "of": null }, "environment.tests_passed": { "value": 46.0, "of": 46.0 }, "environment.lines": { "value": 401454.0, "of": null }, "environment.complexity_mean": { "value": 3.266590626296143, "of": null } }, "commands": { "all": 35, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1942337.0, "of": null }, "system.elapsed_s": { "value": 537.06, "of": null }, "environment.tests_passed": { "value": 48.0, "of": 48.0 }, "environment.lines": { "value": 401422.0, "of": null }, "environment.complexity_mean": { "value": 3.266818700114025, "of": null } }, "commands": { "all": 27, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2834320.0, "of": null }, "system.elapsed_s": { "value": 557.8, "of": null }, "system.solver_obligations": { "value": 158.0, "of": null }, "system.solver_established": { "value": 158.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 45.0, "of": 45.0 }, "environment.lines": { "value": 401425.0, "of": null }, "environment.complexity_mean": { "value": 3.2661068781423315, "of": null } }, "commands": { "all": 27, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.06, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.06, "verdict": "not_supported" } } }, { "stage": 4, "name": "agent-token-telemetry", "arms": { "control": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2504895.0, "of": null }, "system.elapsed_s": { "value": 634.807, "of": null }, "environment.tests_passed": { "value": 52.0, "of": 52.0 }, "environment.lines": { "value": 401650.0, "of": null }, "environment.complexity_mean": { "value": 3.2685732048492384, "of": null } }, "commands": { "all": 36, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.39, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2617657.0, "of": null }, "system.elapsed_s": { "value": 658.017, "of": null }, "environment.tests_passed": { "value": 55.0, "of": 55.0 }, "environment.lines": { "value": 401609.0, "of": null }, "environment.complexity_mean": { "value": 3.2688043928719437, "of": null } }, "commands": { "all": 39, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2730721.0, "of": null }, "system.elapsed_s": { "value": 726.975, "of": null }, "system.solver_obligations": { "value": 168.0, "of": null }, "system.solver_established": { "value": 168.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 50.0, "of": 50.0 }, "environment.lines": { "value": 401572.0, "of": null }, "environment.complexity_mean": { "value": 3.268017201181286, "of": null } }, "commands": { "all": 32, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.08, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.03, "verdict": "not_supported" } } }, { "stage": 5, "name": "ontology-audit", "arms": { "control": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2992177.0, "of": null }, "system.elapsed_s": { "value": 634.276, "of": null }, "environment.tests_passed": { "value": 62.0, "of": 62.0 }, "environment.lines": { "value": 402004.0, "of": null }, "environment.complexity_mean": { "value": 3.2713250517598342, "of": null } }, "commands": { "all": 42, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1381531.0, "of": null }, "system.elapsed_s": { "value": 368.477, "of": null }, "environment.tests_passed": { "value": 65.0, "of": 65.0 }, "environment.lines": { "value": 401897.0, "of": null }, "environment.complexity_mean": { "value": 3.2714189573950407, "of": null } }, "commands": { "all": 32, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3931528.0, "of": null }, "system.elapsed_s": { "value": 830.965, "of": null }, "system.solver_obligations": { "value": 201.0, "of": null }, "system.solver_established": { "value": 201.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 63.0, "of": 63.0 }, "environment.lines": { "value": 401993.0, "of": null }, "environment.complexity_mean": { "value": 3.270026909542538, "of": null } }, "commands": { "all": 56, "z3": 12 } } }, "contrasts": { "semantic": { "difference": -0.02, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.01, "verdict": "not_supported" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.36 }, { "stage": 2, "score": 0.3 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.31 }, { "stage": 5, "score": 0.3 } ], "mean_score": 0.31, "scored_stages": 5, "build_tokens": { "value": 12054325.0, "stages": 5 }, "review_tokens": { "value": 1927932.0, "stages": 5 }, "total_tokens": { "value": 13982257.0, "stages": 5 }, "build_s": { "value": 3008.891, "stages": 5 }, "review_s": { "value": 864.9449999999999, "stages": 5 }, "total_s": { "value": 3895.0769999999998, "stages": 5 }, "final_stage": 5, "final_lines": 402004.0, "final_complexity": 3.2713250517598342 }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.41 }, { "stage": 2, "score": 0.31 }, { "stage": 3, "score": 0.34 }, { "stage": 4, "score": 0.39 }, { "stage": 5, "score": 0.28 } ], "mean_score": 0.346, "scored_stages": 5, "build_tokens": { "value": 12191409.0, "stages": 5 }, "review_tokens": { "value": 2220984.0, "stages": 5 }, "total_tokens": { "value": 14412393.0, "stages": 5 }, "build_s": { "value": 2924.709, "stages": 5 }, "review_s": { "value": 867.569, "stages": 5 }, "total_s": { "value": 3814.842, "stages": 5 }, "final_stage": 5, "final_lines": 401897.0, "final_complexity": 3.2714189573950407 }, { "arm": "algebraic_solver", "stages": 5, "scores": [ { "stage": 1, "score": 0.35 }, { "stage": 2, "score": 0.31 }, { "stage": 3, "score": 0.34 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.31 } ], "mean_score": 0.318, "scored_stages": 5, "build_tokens": { "value": 16610068.0, "stages": 5 }, "review_tokens": { "value": 2097119.0, "stages": 5 }, "total_tokens": { "value": 18707187.0, "stages": 5 }, "build_s": { "value": 3671.0640000000003, "stages": 5 }, "review_s": { "value": 870.8919999999999, "stages": 5 }, "total_s": { "value": 4563.246, "stages": 5 }, "final_stage": 5, "final_lines": 401993.0, "final_complexity": 3.270026909542538 } ] }, { "run_id": "run-20260924T014000Z-e4983936", "experiment_id": "E024", "started_at": "2026-09-24T01:40:00.633491+00:00", "status": "completed", "harness_failures": [], "builder": { "agent_harness": "codex-cli", "model_id": "gpt-6-sol", "reasoning_effort": "medium" }, "judge": [ "claude-opus-5-5-high" ], "seed": 1, "system": { "repository": "ag-prompt-validation-application", "commit": "3c3ed0d2ea393873dd2586533ce85643f7123c59" }, "environment": { "repository": "ag-prompt-validation-application-review", "commit": "d665e0cd964c86e12145f87cb5fc0ffe010c34e4" }, "harness_commit": "68e737e8096aff7872dde6ed1c2f6e5ac467d773", "arms": [ "control", "semantic", "algebraic_solver" ], "stages": [ { "stage": 1, "name": "duplicate-name-check", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1327506.0, "of": null }, "system.elapsed_s": { "value": 507.319, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 401038.0, "of": null }, "environment.complexity_mean": { "value": 3.2653029437723897, "of": null } }, "commands": { "all": 30, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2053891.0, "of": null }, "system.elapsed_s": { "value": 586.356, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 401041.0, "of": null }, "environment.complexity_mean": { "value": 3.2650295919426853, "of": null } }, "commands": { "all": 26, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2063320.0, "of": null }, "system.elapsed_s": { "value": 517.771, "of": null }, "system.solver_obligations": { "value": 122.0, "of": null }, "system.solver_established": { "value": 122.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 25.0, "of": 25.0 }, "environment.lines": { "value": 400994.0, "of": null }, "environment.complexity_mean": { "value": 3.2653961989822413, "of": null } }, "commands": { "all": 37, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.03, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 2, "name": "entry-supersession", "arms": { "control": { "status": "completed", "score": 0.35, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2803813.0, "of": null }, "system.elapsed_s": { "value": 782.41, "of": null }, "environment.tests_passed": { "value": 46.0, "of": 46.0 }, "environment.lines": { "value": 401354.0, "of": null }, "environment.complexity_mean": { "value": 3.2659960593176396, "of": null } }, "commands": { "all": 42, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.31, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2928793.0, "of": null }, "system.elapsed_s": { "value": 729.088, "of": null }, "environment.tests_passed": { "value": 38.0, "of": 38.0 }, "environment.lines": { "value": 401279.0, "of": null }, "environment.complexity_mean": { "value": 3.2668291670988485, "of": null } }, "commands": { "all": 31, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4490045.0, "of": null }, "system.elapsed_s": { "value": 778.875, "of": null }, "system.solver_obligations": { "value": 138.0, "of": null }, "system.solver_established": { "value": 138.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 37.0, "of": 37.0 }, "environment.lines": { "value": 401264.0, "of": null }, "environment.complexity_mean": { "value": 3.2662758728017844, "of": null } }, "commands": { "all": 66, "z3": 3 } } }, "contrasts": { "semantic": { "difference": -0.04, "verdict": "not_supported" }, "algebraic_solver": { "difference": -0.07, "verdict": "not_supported" } } }, { "stage": 3, "name": "revise-information", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1951977.0, "of": null }, "system.elapsed_s": { "value": 559.055, "of": null }, "environment.tests_passed": { "value": 51.0, "of": 51.0 }, "environment.lines": { "value": 401531.0, "of": null }, "environment.complexity_mean": { "value": 3.266103539410271, "of": null } }, "commands": { "all": 21, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.34, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2432526.0, "of": null }, "system.elapsed_s": { "value": 547.063, "of": null }, "environment.tests_passed": { "value": 42.0, "of": 42.0 }, "environment.lines": { "value": 401429.0, "of": null }, "environment.complexity_mean": { "value": 3.267209205888451, "of": null } }, "commands": { "all": 29, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.32, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2324000.0, "of": null }, "system.elapsed_s": { "value": 592.43, "of": null }, "system.solver_obligations": { "value": 147.0, "of": null }, "system.solver_established": { "value": 147.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 41.0, "of": 41.0 }, "environment.lines": { "value": 401431.0, "of": null }, "environment.complexity_mean": { "value": 3.2670572376607216, "of": null } }, "commands": { "all": 39, "z3": 0 } } }, "contrasts": { "semantic": { "difference": 0.06, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.04, "verdict": "not_supported" } } }, { "stage": 4, "name": "agent-token-telemetry", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2281053.0, "of": null }, "system.elapsed_s": { "value": 746.414, "of": null }, "environment.tests_passed": { "value": 58.0, "of": 58.0 }, "environment.lines": { "value": 401749.0, "of": null }, "environment.complexity_mean": { "value": 3.26811331503444, "of": null } }, "commands": { "all": 40, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3982354.0, "of": null }, "system.elapsed_s": { "value": 1001.52, "of": null }, "environment.tests_passed": { "value": 46.0, "of": 46.0 }, "environment.lines": { "value": 401572.0, "of": null }, "environment.complexity_mean": { "value": 3.2697030934245297, "of": null } }, "commands": { "all": 48, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 1861861.0, "of": null }, "system.elapsed_s": { "value": 536.596, "of": null }, "system.solver_obligations": { "value": 159.0, "of": null }, "system.solver_established": { "value": 159.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 45.0, "of": 45.0 }, "environment.lines": { "value": 401548.0, "of": null }, "environment.complexity_mean": { "value": 3.268618813164032, "of": null } }, "commands": { "all": 27, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.02, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.0, "verdict": "not_supported" } } }, { "stage": 5, "name": "ontology-audit", "arms": { "control": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 3846623.0, "of": null }, "system.elapsed_s": { "value": 612.991, "of": null }, "environment.tests_passed": { "value": 73.0, "of": 73.0 }, "environment.lines": { "value": 402224.0, "of": null }, "environment.complexity_mean": { "value": 3.271359122879603, "of": null } }, "commands": { "all": 30, "z3": 0 } }, "semantic": { "status": "completed", "score": 0.28, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 2667855.0, "of": null }, "system.elapsed_s": { "value": 599.922, "of": null }, "environment.tests_passed": { "value": 56.0, "of": 56.0 }, "environment.lines": { "value": 401933.0, "of": null }, "environment.complexity_mean": { "value": 3.2723507791064863, "of": null } }, "commands": { "all": 29, "z3": 0 } }, "algebraic_solver": { "status": "completed", "score": 0.3, "failures": [], "errors": [], "measurements": { "system.total_tokens": { "value": 4596747.0, "of": null }, "system.elapsed_s": { "value": 844.791, "of": null }, "system.solver_obligations": { "value": 187.0, "of": null }, "system.solver_established": { "value": 187.0, "of": null }, "system.solver_refuted": { "value": 0.0, "of": null }, "system.solver_unestablished": { "value": 0.0, "of": null }, "environment.tests_passed": { "value": 57.0, "of": 57.0 }, "environment.lines": { "value": 401956.0, "of": null }, "environment.complexity_mean": { "value": 3.2706661835498734, "of": null } }, "commands": { "all": 48, "z3": 1 } } }, "contrasts": { "semantic": { "difference": 0.0, "verdict": "not_supported" }, "algebraic_solver": { "difference": 0.02, "verdict": "not_supported" } } } ], "totals": [ { "arm": "control", "stages": 5, "scores": [ { "stage": 1, "score": 0.28 }, { "stage": 2, "score": 0.35 }, { "stage": 3, "score": 0.28 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.28 } ], "mean_score": 0.29400000000000004, "scored_stages": 5, "build_tokens": { "value": 12210972.0, "stages": 5 }, "review_tokens": { "value": 2214711.0, "stages": 5 }, "total_tokens": { "value": 14425683.0, "stages": 5 }, "build_s": { "value": 3208.389, "stages": 5 }, "review_s": { "value": 859.885, "stages": 5 }, "total_s": { "value": 4090.935, "stages": 5 }, "final_stage": 5, "final_lines": 402224.0, "final_complexity": 3.271359122879603 }, { "arm": "semantic", "stages": 5, "scores": [ { "stage": 1, "score": 0.31 }, { "stage": 2, "score": 0.31 }, { "stage": 3, "score": 0.34 }, { "stage": 4, "score": 0.3 }, { "stage": 5, "score": 0.28 } ], "mean_score": 0.308, "scored_stages": 5, "build_tokens": { "value": 14065419.0, "stages": 5 }, "review_tokens": { "value": 1828371.0, "stages": 5 }, "total_tokens": { "value": 15893790.0, "stages": 5 }, "build_s": { "value": 3464.169, "stages": 5 }, "review_s": { "value": 898.379, "stages": 5 }, "total_s": { "value": 4384.769, "stages": 5 }, "final_stage": 5, "final_lines": 401933.0, "final_complexity": 3.2723507791064863 }, { "arm": "algebraic_solver", "stages": 5, "scores": [ { "stage": 1, "score": 0.28 }, { "stage": 2, "score": 0.28 }, { "stage": 3, "score": 0.32 }, { "stage": 4, "score": 0.28 }, { "stage": 5, "score": 0.3 } ], "mean_score": 0.292, "scored_stages": 5, "build_tokens": { "value": 15335973.0, "stages": 5 }, "review_tokens": { "value": 1646884.0, "stages": 5 }, "total_tokens": { "value": 16982857.0, "stages": 5 }, "build_s": { "value": 3271.204, "stages": 5 }, "review_s": { "value": 851.276, "stages": 5 }, "total_s": { "value": 4148.963, "stages": 5 }, "final_stage": 5, "final_lines": 401956.0, "final_complexity": 3.2706661835498734 } ] } ], "checks": [ { "kind": "contrast", "arms": [ "algebraic_solver" ], "than": "control", "measurement": null, "require": "every", "verdict": true, "not_run": false, "passed": false, "readings": { "every": false, "most": false, "mean": false }, "results": [ { "run_id": "run-20260924T013909Z-79df517b", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "difference": -0.02, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.02, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": 0.03, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": -0.06, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T013934Z-a085b261", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "difference": -0.01, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": 0.01, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": 0.06, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": -0.03, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": 0.01, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } }, { "run_id": "run-20260924T014000Z-e4983936", "arm": "algebraic_solver", "not_run": false, "stages": [ { "stage": 1, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 2, "difference": -0.07, "verdict": "not_supported", "passed": false }, { "stage": 3, "difference": 0.04, "verdict": "not_supported", "passed": false }, { "stage": 4, "difference": 0.0, "verdict": "not_supported", "passed": false }, { "stage": 5, "difference": 0.02, "verdict": "not_supported", "passed": false } ], "passed": { "every": false, "most": false, "mean": false } } ] } ], "verdict": "not_supported", "readings": { "every": false, "most": false, "mean": false }, "reading_independent": true, "effect": { "algebraic_solver": { "cells": 15, "mean": -0.0, "min": -0.07, "max": 0.06, "differences": [ -0.02, 0.02, 0.03, 0.0, -0.06, -0.01, 0.01, 0.06, -0.03, 0.01, 0.0, -0.07, 0.04, 0.0, 0.02 ] } }, "spread": { "sets": [ { "runs": [ "run-20260924T013909Z-79df517b", "run-20260924T013934Z-a085b261", "run-20260924T014000Z-e4983936" ], "cells": [ { "stage": 1, "arm": "control", "scores": [ 0.34, 0.36, 0.28 ], "range": 0.08, "reaches_threshold": false }, { "stage": 1, "arm": "semantic", "scores": [ 0.31, 0.41, 0.31 ], "range": 0.1, "reaches_threshold": true }, { "stage": 1, "arm": "algebraic_solver", "scores": [ 0.32, 0.35, 0.28 ], "range": 0.07, "reaches_threshold": false }, { "stage": 2, "arm": "control", "scores": [ 0.28, 0.3, 0.35 ], "range": 0.07, "reaches_threshold": false }, { "stage": 2, "arm": "semantic", "scores": [ 0.28, 0.31, 0.31 ], "range": 0.03, "reaches_threshold": false }, { "stage": 2, "arm": "algebraic_solver", "scores": [ 0.3, 0.31, 0.28 ], "range": 0.03, "reaches_threshold": false }, { "stage": 3, "arm": "control", "scores": [ 0.31, 0.28, 0.28 ], "range": 0.03, "reaches_threshold": false }, { "stage": 3, "arm": "semantic", "scores": [ 0.39, 0.34, 0.34 ], "range": 0.05, "reaches_threshold": false }, { "stage": 3, "arm": "algebraic_solver", "scores": [ 0.34, 0.34, 0.32 ], "range": 0.02, "reaches_threshold": false }, { "stage": 4, "arm": "control", "scores": [ 0.28, 0.31, 0.28 ], "range": 0.03, "reaches_threshold": false }, { "stage": 4, "arm": "semantic", "scores": [ 0.28, 0.39, 0.3 ], "range": 0.11, "reaches_threshold": true }, { "stage": 4, "arm": "algebraic_solver", "scores": [ 0.28, 0.28, 0.28 ], "range": 0.0, "reaches_threshold": false }, { "stage": 5, "arm": "control", "scores": [ 0.34, 0.3, 0.28 ], "range": 0.06, "reaches_threshold": false }, { "stage": 5, "arm": "semantic", "scores": [ 0.3, 0.28, 0.28 ], "range": 0.02, "reaches_threshold": false }, { "stage": 5, "arm": "algebraic_solver", "scores": [ 0.28, 0.31, 0.3 ], "range": 0.03, "reaches_threshold": false } ] } ] } } ], "excluded": [ { "run_id": "run-20260923T163249Z-6db45ead", "experiment_id": "E007", "status": "completed", "started_at": "2026-09-23T16:32:49.820205+00:00", "reason": "test", "condition": null, "by": null, "note": "E007, a test run of the design" }, { "run_id": "run-20260923T165218Z-7cd21cda", "experiment_id": "E008", "status": "completed", "started_at": "2026-09-23T16:52:18.443584+00:00", "reason": "test", "condition": null, "by": null, "note": "E008, a test run of the design" }, { "run_id": "run-20260923T205309Z-9b867390", "experiment_id": "E011", "status": "failed", "started_at": "2026-09-23T20:53:09.802422+00:00", "reason": "archived", "condition": null, "by": null, "note": null }, { "run_id": "run-20260923T220828Z-cf54672c", "experiment_id": "E012", "status": "stopped", "started_at": "2026-09-23T22:08:28.828831+00:00", "reason": "archived", "condition": null, "by": null, "note": null }, { "run_id": "run-20260923T221740Z-122c26b5", "experiment_id": "E013", "status": "stopped", "started_at": "2026-09-23T22:17:40.145511+00:00", "reason": "archived", "condition": null, "by": null, "note": null }, { "run_id": "run-20260923T221810Z-3cfeceb9", "experiment_id": "E014", "status": "stopped", "started_at": "2026-09-23T22:18:10.098242+00:00", "reason": "archived", "condition": null, "by": null, "note": null }, { "run_id": "run-20260923T235751Z-d7ff2736", "experiment_id": "E019", "status": "stopped", "started_at": "2026-09-23T23:57:51.198364+00:00", "reason": "archived", "condition": null, "by": null, "note": null }, { "run_id": "run-20260923T223713Z-9f2e0c92", "experiment_id": "E017", "status": "stopped", "started_at": "2026-09-23T22:37:13.997165+00:00", "reason": "stopped", "condition": "E015-E017", "by": null, "note": null }, { "run_id": "run-20260924T000529Z-b65f821d", "experiment_id": "E021", "status": "stopped", "started_at": "2026-09-24T00:05:29.360339+00:00", "reason": "stopped", "condition": "E020-E026", "by": null, "note": null }, { "run_id": "run-20260924T010931Z-b793741b", "experiment_id": "E022", "status": "completed", "started_at": "2026-09-24T01:09:31.713312+00:00", "reason": "superseded", "condition": "E022-E023", "by": "run-20260924T023130Z-ccbee4e6", "note": "The judge's copy of the workspace left out docs/design, so it scored the design arms' citations as missing. Environment c7ac0d7 fixed the grader; the run was repeated." }, { "run_id": "run-20260924T010951Z-924d890f", "experiment_id": "E023", "status": "completed", "started_at": "2026-09-24T01:09:51.704807+00:00", "reason": "superseded", "condition": "E022-E023", "by": "run-20260924T023152Z-aa033b06", "note": "The same grader defect as E022's first run; repeated on environment c7ac0d7." } ], "spread": { "cells": 35, "replicate_sets": 3, "median_range": 0.03, "max_range": 0.11, "pooled_sd": 0.0301, "reaching_threshold": 4 }, "source": { "harness_commit": "44866678543d014566c569ce39f47a74dc6c4186", "spec": "notebooks/findings/G001.yaml", "store": "tracking.db" } }