{ "date": "2026-10-16", "n": 8, "selection": "First CSV-ordered case per language in the disclosed 24-task development cohort; no score-based selection", "selected_sha256": "3f271fffc1394591c485e4eee5e2501e39e07e0e6146263392ee8ab0e889f568", "population": "All selected tasks; scoring failures contribute zero", "metric_scope": "Official cumulative observed trajectory; no Pass@1", "metrics": { "recall": { "baseline": 0.4113320539329563, "candidate": 0.41667420613064926, "difference": 0.005342152197692983, "paired_bootstrap_95_interval": [ -0.0203472865542274, 0.033622934782938145 ] }, "precision": { "baseline": 0.33942422425132535, "candidate": 0.2932575950369542, "difference": -0.04616662921437112, "paired_bootstrap_95_interval": [ -0.1379765141495276, 0.008191522751061005 ] }, "f1": { "baseline": 0.33666878117993204, "candidate": 0.3190974702230495, "difference": -0.01757131095688256, "paired_bootstrap_95_interval": [ -0.04943844041229954, 0.01036602904413432 ] } }, "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "protocol": { "backend": "claude-agent-sdk", "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5", "max_turns": 20, "completion_tokens": 2048, "max_meter_usd": 15.0, "case_meter_usd": 3.0, "case_timeout": 600.0, "tools": [ "mcp__source__read", "mcp__source__grep", "mcp__source__glob" ], "implementation": { "scripts/benchmark_context_agent.py": "a156e5ce9ae0caa60112f1b7887a713b40d0cf6eb5448be7c514ee89a82ec657", "scripts/benchmark_context_claude.py": "d7f5933498fdb0871ec5321aec4d462d688c8205287f11d99a1600d49b1785b5", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" } }, "actual_models": [ "claude-opus-5-5" ], "dataset_revision": "c2855792b006af41c67202d33883fb9d46362853", "evaluator_revision": "1436c28a8eb95496da4ea69ad458b9f8a8eb7d61", "arms": { "native": { "source_policy": "native", "statuses": { "ok": 8 }, "scoring_failures": 0, "sdk_reported_cost_usd": 2.153231, "actual_bill_usd": null }, "functions": { "source_policy": "functions", "statuses": { "ok": 8 }, "scoring_failures": 0, "sdk_reported_cost_usd": 2.4067818, "actual_bill_usd": null } }, "conclusion": "Independent-32 and later development policies remain rejected. Source-bound C++ fixes and TypeScript/Java availability recovery improve graph readiness, but fixed queries add no annotated source and justify no new online batch. Official scores, sealed validation tasks and API spending remain unchanged.", "cost_scope": "SDK metering estimates are not account bills. Historical top-level arms use the subscription route; the latest online grep/Jev control uses explicit direct Anthropic API sessions. Per-experiment scopes and current_program_cost distinguish these routes and Jev decision receipts.", "artifact_scope": "Sanitized development metrics only; not a full leaderboard submission bundle.", "implementation_commit": "a853d8d1", "aggregation": "Macro mean of per-task line recall, precision and F1; all selected tasks retained. The evaluator CLI summary uses separate pooled-count micro aggregates.", "directed_graph_trials": [ { "n": 8, "contrast": "directed_graph_over_function_completion", "population": "All selected tasks; scoring failures contribute zero", "metric_scope": "Official cumulative observed trajectory; no Pass@1", "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "metrics": { "recall": { "baseline": 0.40225208733729645, "candidate": 0.44564829582496857, "difference": 0.043396208487672144, "paired_bootstrap_95_interval": [ -0.0005920672800379771, 0.11875505772151156 ] }, "precision": { "baseline": 0.2819475699668096, "candidate": 0.3213650055170624, "difference": 0.039417435550252816, "paired_bootstrap_95_interval": [ -0.010599561790323637, 0.09682554772623696 ] }, "f1": { "baseline": 0.3110870341379826, "candidate": 0.34282392448513155, "difference": 0.03173689034714895, "paired_bootstrap_95_interval": [ -0.008101353070228686, 0.07461544211656611 ] } }, "policy": "optional_reference_hop", "implementation_commit": "359357a2", "arms": { "control": { "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 }, "scoring_failures": 0, "sdk_reported_cost_usd": 2.6822363, "actual_bill_usd": null, "missing_metering_cases": 0, "tools": [ "mcp__source__read", "mcp__source__grep", "mcp__source__glob" ], "per_case": { "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.4594594594594595, "precision": 0.245224171539961, "f1": 0.31977630910015253, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.6683185 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5476190476190477, "f1": 0.5418848167539269, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.1506482 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.16473072861668428, "precision": 0.23636363636363636, "f1": 0.19415059116365901, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.2948599 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.6186186186186187, "precision": 0.12576312576312576, "f1": 0.20903094875697614, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.5618663999999999 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.01020408163265306, "precision": 0.0029069767441860465, "f1": 0.004524886877828054, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.2650199 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.48333333333333334, "precision": 0.4522417153996101, "f1": 0.4672708962739174, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.1684652 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.3769363166953528, "precision": 0.1697674418604651, "f1": 0.23409941207910212, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.4668738 }, "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.5684647302904564, "precision": 0.4756944444444444, "f1": 0.5179584120982986, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.10618440000000001 } } }, "candidate": { "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 }, "scoring_failures": 0, "sdk_reported_cost_usd": 2.4828441000000003, "actual_bill_usd": null, "missing_metering_cases": 0, "tools": [ "mcp__source__read", "mcp__source__grep", "mcp__source__glob", "mcp__source__graph_neighbors" ], "per_case": { "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.5025566106647188, "precision": 0.3926940639269406, "f1": 0.4408843319448894, "status": "ok", "graph_available": false, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.4448314 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5490716180371353, "f1": 0.5425950196592398, "status": "ok", "graph_available": false, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.1478053 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.1816261879619852, "precision": 0.27129337539432175, "f1": 0.21758380771663505, "status": "ok", "graph_available": false, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.27610579999999996 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.6126126126126126, "precision": 0.11056910569105691, "f1": 0.18732782369146003, "status": "ok", "graph_available": true, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.5382674000000001 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.30612244897959184, "precision": 0.06952491309385864, "f1": 0.11331444759206799, "status": "ok", "graph_available": true, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.30520210000000003 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.48333333333333334, "precision": 0.3979416809605489, "f1": 0.43650047036688616, "status": "ok", "graph_available": true, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.173739 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.37005163511187605, "precision": 0.1349654739485248, "f1": 0.19779208831646733, "status": "ok", "graph_available": true, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.5050728999999999 }, "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.5726141078838174, "precision": 0.6448598130841121, "f1": 0.6065934065934065, "status": "ok", "graph_available": true, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.0918202 } } } }, "protocol": { "backend": "claude-agent-sdk", "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5", "max_turns": 20, "source_policy": "functions", "completion_tokens": 2048, "max_meter_usd": 15.0, "case_meter_usd": 3.0, "case_timeout": 600.0, "implementation": { "scripts/benchmark_context_agent.py": "ff70cc368d09751efb6f0b1e4070c4fa72298d7df9b3af6d0608e382c50920ee", "scripts/benchmark_context_claude.py": "0f183363ffee367b1bb530bc078a4d0468238f85d72fc62c13ccdd2f02a2cfda", "scripts/contextbench_graph_navigation.py": "90b6e9c415c3d2d1ebf057afb064519c0701f3ffe9a7112c720fa07ed1b5272e", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "source_manifest_sha256": "73233491d39de65c36aff7ab290610f47f06ec2b3b294ed77e117c7fc27345bc", "graph_manifest_sha256": "883db1e8dbef3b81597e23bbc9ff4286ce611e66fb9d524cd6d7bbf2898aaa92", "graph_tokens": 2048 } }, { "n": 8, "contrast": "directed_graph_over_function_completion", "population": "All selected tasks; scoring failures contribute zero", "metric_scope": "Official cumulative observed trajectory; no Pass@1", "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "metrics": { "recall": { "baseline": 0.3743427274870472, "candidate": 0.41289431183458625, "difference": 0.03855158434753907, "paired_bootstrap_95_interval": [ -0.005057292318419334, 0.11819952431003419 ] }, "precision": { "baseline": 0.2839737331350818, "candidate": 0.32159343682426644, "difference": 0.03761970368918465, "paired_bootstrap_95_interval": [ -0.021400895280340417, 0.10924826376143723 ] }, "f1": { "baseline": 0.30179977366013355, "candidate": 0.32854617509582895, "difference": 0.026746401435695406, "paired_bootstrap_95_interval": [ -0.011059746243171006, 0.0683726801589936 ] } }, "policy": "targeted_reference_hop_before_handoff", "implementation_commit": "3f04993f", "arms": { "control": { "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 }, "scoring_failures": 0, "sdk_reported_cost_usd": 2.4976613, "actual_bill_usd": null, "missing_metering_cases": 0, "tools": [ "mcp__source__read", "mcp__source__grep", "mcp__source__glob" ], "per_case": { "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.32213294375456536, "precision": 0.311660777385159, "f1": 0.31681034482758613, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.4835439 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5240506329113924, "f1": 0.530089628681178, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.1545109 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.1678986272439282, "precision": 0.2527821939586645, "f1": 0.20177664974619286, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.3011871 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.5795795795795796, "precision": 0.12664041994750655, "f1": 0.20786214324178784, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.5244481999999999 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.0, "precision": 0.0, "f1": 0, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.2932394 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.4479166666666667, "precision": 0.5070754716981132, "f1": 0.4756637168141593, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.1450923 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.3683304647160069, "precision": 0.14488828706838186, "f1": 0.2079689018464529, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.4729049 }, "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.5726141078838174, "precision": 0.4046920821114369, "f1": 0.4742268041237113, "status": "ok", "graph_available": null, "graph_calls": 0, "graph_response_records": {}, "sdk_reported_cost_usd": 0.1227346 } } }, "candidate": { "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 }, "scoring_failures": 0, "sdk_reported_cost_usd": 2.702899, "actual_bill_usd": null, "missing_metering_cases": 0, "tools": [ "mcp__source__read", "mcp__source__grep", "mcp__source__glob", "mcp__source__graph_neighbors" ], "per_case": { "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.30533235938641345, "precision": 0.23443634324172744, "f1": 0.26522842639593913, "status": "ok", "graph_available": false, "graph_calls": 1, "graph_response_records": { "graph_unavailable": 1 }, "sdk_reported_cost_usd": 0.5992711000000001 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5240506329113924, "f1": 0.530089628681178, "status": "ok", "graph_available": false, "graph_calls": 1, "graph_response_records": { "graph_unavailable": 1 }, "sdk_reported_cost_usd": 0.17042839999999998 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.1795142555438226, "precision": 0.20581113801452786, "f1": 0.19176536943034406, "status": "ok", "graph_available": false, "graph_calls": 1, "graph_response_records": { "graph_unavailable": 1 }, "sdk_reported_cost_usd": 0.33667870000000005 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.5795795795795796, "precision": 0.1111111111111111, "f1": 0.18647342995169083, "status": "ok", "graph_available": true, "graph_calls": 1, "graph_response_records": { "source_returned": 1 }, "sdk_reported_cost_usd": 0.48835259999999997 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.3112244897959184, "precision": 0.06149193548387097, "f1": 0.1026936026936027, "status": "ok", "graph_available": true, "graph_calls": 1, "graph_response_records": { "source_not_returned": 1 }, "sdk_reported_cost_usd": 0.38124450000000004 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.44583333333333336, "precision": 0.6312684365781711, "f1": 0.5225885225885226, "status": "ok", "graph_available": true, "graph_calls": 1, "graph_response_records": { "source_returned": 1 }, "sdk_reported_cost_usd": 0.15453859999999997 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.3769363166953528, "precision": 0.16138540899042003, "f1": 0.22600619195046437, "status": "ok", "graph_available": true, "graph_calls": 1, "graph_response_records": { "no_definition_at_line": 1 }, "sdk_reported_cost_usd": 0.43596709999999994 }, "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.5684647302904564, "precision": 0.6431924882629108, "f1": 0.6035242290748899, "status": "ok", "graph_available": true, "graph_calls": 1, "graph_response_records": { "source_returned": 3 }, "sdk_reported_cost_usd": 0.136418 } } } }, "protocol": { "backend": "claude-agent-sdk", "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5", "max_turns": 20, "source_policy": "functions", "completion_tokens": 2048, "max_meter_usd": 15.0, "case_meter_usd": 3.0, "case_timeout": 600.0, "implementation": { "scripts/benchmark_context_agent.py": "60eb7f4120ef34f9116fee3de6143f13687fa74ba246b2604c8103abf12293df", "scripts/benchmark_context_claude.py": "0f183363ffee367b1bb530bc078a4d0468238f85d72fc62c13ccdd2f02a2cfda", "scripts/contextbench_graph_navigation.py": "826b0ca787f9806c67339555e3fe62588abed0781209e5684e35bc35e2d4a535", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "source_manifest_sha256": "73233491d39de65c36aff7ab290610f47f06ec2b3b294ed77e117c7fc27345bc", "graph_manifest_sha256": "883db1e8dbef3b81597e23bbc9ff4286ce611e66fb9d524cd6d7bbf2898aaa92", "graph_tokens": 2048 } } ], "explicit_graph_integration_probe": { "scope": "Synthetic explicit-call instruction; excluded from quality scores.", "graph_calls": 1, "status": "ok", "sdk_reported_cost_usd": 0.0280536, "actual_bill_usd": null }, "matched_solver_development_check": { "scope": "Repeated disclosed Python development task; not independent tasks or population Pass@1. No grading feedback entered either solver.", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "implementation_commit": "3f04993f", "protocol": { "context_budget": 16384, "source_commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "source_tree": "05a6b93fcb973fb72fe368b7d43eee00ed6635da", "upstream_commit": "1436c28a8eb95496da4ea69ad458b9f8a8eb7d61", "image_id": "sha256:b5fac9ce8ae7da0c6ad028bce73932b257eb44656985e4f7c7d73d7dfc60dcf7", "model_requested": "claude-opus-5-5", "max_turns": 30, "max_meter_usd": 3.0, "sdk_version": "0.2.165" }, "native_harness_version": "4.1.0", "native_dataset_revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", "arms": { "control": { "status": "Submitted", "actual_models": [ "claude-opus-5-5" ], "sdk_model_calls": 9, "sdk_reported_cost_usd": 0.8264703, "missing_metering_calls": 0, "actual_bill_usd": null, "patch_successfully_applied": true, "resolved": false, "required_tests_passed": 0, "required_tests_failed": 21, "failure": "Native test collection: generated production patch omits IS_PYPY imported by the reference test patch." }, "candidate": { "status": "Submitted", "actual_models": [ "claude-opus-5-5" ], "sdk_model_calls": 6, "sdk_reported_cost_usd": 0.442726, "missing_metering_calls": 0, "actual_bill_usd": null, "patch_successfully_applied": true, "resolved": false, "required_tests_passed": 0, "required_tests_failed": 21, "failure": "Native test collection: generated production patch omits IS_PYPY imported by the reference test patch." } }, "reference_patch_harness_check": { "scope": "Reference-patch harness check only; excluded from agent scores and solver attempts.", "dataset_sha256": "d92ef4dadc6482fc80a961e1a3a313713c78567721bb11b96dda6dab621ed71f", "native_harness_version": "4.1.0", "image_id": "sha256:b5fac9ce8ae7da0c6ad028bce73932b257eb44656985e4f7c7d73d7dfc60dcf7", "model_calls": 0, "completed": true, "resolved": true } }, "targeted_hop_development_gate": { "all_cases_scored": true, "all_available_graphs_called": true, "graph_returned_source": true, "recall_point_increased": true, "f1_point_not_lower": true, "decision": "Advance unchanged policy to independent validation; no promotion or SOTA claim.", "implementation_commit": "3f04993f15d098df4ee798dc93565ade70fb36ed", "comparison_sha256": "18151151ad01eff475da706613df338494bb954af98f4eaaacefb7b8abd9d152" }, "development_source_attribution": { "scope": "Development-only physical-line attribution in existing trajectories. Count each source line once, at its first actual observation; separate original tool output from completion. Annotation overlap is diagnostic, includes physical blank/comment lines, and is not the official score or a counterfactual online ablation. Repeated traces cover the same eight tasks. Non-annotated source may still help solving.", "new_model_calls": 0, "new_api_spend_usd": 0, "heldout_task_overlap": 0, "analysis_sha256": "2162bdf7955e815ddecedb7dd385a3bc4c3360d65454c014e8f6787fa9b1584c", "arms": { "development-functions": { "n_tasks": 8, "totals": { "completed_grep": { "new_physical_lines": 1901, "annotated_lines": 381, "responses_adding_source": 26, "annotated_fraction": 0.20042083114150447 }, "completed_read": { "new_physical_lines": 584, "annotated_lines": 108, "responses_adding_source": 18, "annotated_fraction": 0.18493150684931506 }, "original_grep": { "new_physical_lines": 451, "annotated_lines": 80, "responses_adding_source": 32, "annotated_fraction": 0.17738359201773837 }, "original_read": { "new_physical_lines": 4409, "annotated_lines": 1035, "responses_adding_source": 39, "annotated_fraction": 0.23474710818779768 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "counts": { "original_grep": { "new_physical_lines": 15, "annotated_lines": 14, "responses_adding_source": 1 }, "completed_grep": { "new_physical_lines": 64, "annotated_lines": 51, "responses_adding_source": 1 }, "original_read": { "new_physical_lines": 123, "annotated_lines": 37, "responses_adding_source": 2 }, "completed_read": { "new_physical_lines": 35, "annotated_lines": 35, "responses_adding_source": 1 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "counts": { "original_grep": { "new_physical_lines": 65, "annotated_lines": 0, "responses_adding_source": 5 }, "completed_grep": { "new_physical_lines": 158, "annotated_lines": 0, "responses_adding_source": 3 }, "original_read": { "new_physical_lines": 611, "annotated_lines": 60, "responses_adding_source": 7 }, "completed_read": { "new_physical_lines": 98, "annotated_lines": 0, "responses_adding_source": 2 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "counts": { "original_grep": { "new_physical_lines": 105, "annotated_lines": 17, "responses_adding_source": 6 }, "completed_grep": { "new_physical_lines": 420, "annotated_lines": 41, "responses_adding_source": 5 }, "original_read": { "new_physical_lines": 989, "annotated_lines": 328, "responses_adding_source": 7 }, "completed_read": { "new_physical_lines": 89, "annotated_lines": 10, "responses_adding_source": 4 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "counts": { "original_grep": { "new_physical_lines": 49, "annotated_lines": 11, "responses_adding_source": 1 }, "completed_grep": { "new_physical_lines": 35, "annotated_lines": 0, "responses_adding_source": 1 }, "original_read": { "new_physical_lines": 310, "annotated_lines": 196, "responses_adding_source": 1 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "counts": { "original_grep": { "new_physical_lines": 12, "annotated_lines": 6, "responses_adding_source": 3 }, "completed_grep": { "new_physical_lines": 250, "annotated_lines": 77, "responses_adding_source": 3 }, "original_read": { "new_physical_lines": 227, "annotated_lines": 100, "responses_adding_source": 3 }, "completed_read": { "new_physical_lines": 77, "annotated_lines": 59, "responses_adding_source": 2 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "counts": { "original_grep": { "new_physical_lines": 53, "annotated_lines": 18, "responses_adding_source": 4 }, "completed_grep": { "new_physical_lines": 224, "annotated_lines": 85, "responses_adding_source": 4 }, "original_read": { "new_physical_lines": 364, "annotated_lines": 43, "responses_adding_source": 4 }, "completed_read": { "new_physical_lines": 16, "annotated_lines": 0, "responses_adding_source": 2 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "counts": { "original_read": { "new_physical_lines": 954, "annotated_lines": 167, "responses_adding_source": 10 }, "original_grep": { "new_physical_lines": 81, "annotated_lines": 8, "responses_adding_source": 4 }, "completed_grep": { "new_physical_lines": 217, "annotated_lines": 35, "responses_adding_source": 2 }, "completed_read": { "new_physical_lines": 235, "annotated_lines": 4, "responses_adding_source": 5 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "counts": { "original_grep": { "new_physical_lines": 71, "annotated_lines": 6, "responses_adding_source": 8 }, "completed_grep": { "new_physical_lines": 533, "annotated_lines": 92, "responses_adding_source": 7 }, "original_read": { "new_physical_lines": 831, "annotated_lines": 104, "responses_adding_source": 5 }, "completed_read": { "new_physical_lines": 34, "annotated_lines": 0, "responses_adding_source": 2 } } } ] }, "required-hop-control-functions": { "n_tasks": 8, "totals": { "completed_grep": { "new_physical_lines": 1931, "annotated_lines": 335, "responses_adding_source": 29, "annotated_fraction": 0.17348524080787156 }, "completed_read": { "new_physical_lines": 640, "annotated_lines": 102, "responses_adding_source": 19, "annotated_fraction": 0.159375 }, "original_grep": { "new_physical_lines": 542, "annotated_lines": 72, "responses_adding_source": 35, "annotated_fraction": 0.13284132841328414 }, "original_read": { "new_physical_lines": 3952, "annotated_lines": 1058, "responses_adding_source": 37, "annotated_fraction": 0.26771255060728744 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "counts": { "original_grep": { "new_physical_lines": 15, "annotated_lines": 14, "responses_adding_source": 1 }, "completed_grep": { "new_physical_lines": 64, "annotated_lines": 51, "responses_adding_source": 1 }, "original_read": { "new_physical_lines": 152, "annotated_lines": 43, "responses_adding_source": 3 }, "completed_read": { "new_physical_lines": 110, "annotated_lines": 30, "responses_adding_source": 2 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "counts": { "original_grep": { "new_physical_lines": 68, "annotated_lines": 0, "responses_adding_source": 4 }, "completed_grep": { "new_physical_lines": 217, "annotated_lines": 0, "responses_adding_source": 3 }, "original_read": { "new_physical_lines": 479, "annotated_lines": 0, "responses_adding_source": 6 }, "completed_read": { "new_physical_lines": 96, "annotated_lines": 0, "responses_adding_source": 1 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "counts": { "original_grep": { "new_physical_lines": 97, "annotated_lines": 12, "responses_adding_source": 8 }, "completed_grep": { "new_physical_lines": 336, "annotated_lines": 41, "responses_adding_source": 5 }, "original_read": { "new_physical_lines": 889, "annotated_lines": 377, "responses_adding_source": 6 }, "completed_read": { "new_physical_lines": 93, "annotated_lines": 11, "responses_adding_source": 4 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "counts": { "original_grep": { "new_physical_lines": 50, "annotated_lines": 11, "responses_adding_source": 2 }, "completed_grep": { "new_physical_lines": 35, "annotated_lines": 0, "responses_adding_source": 1 }, "original_read": { "new_physical_lines": 310, "annotated_lines": 196, "responses_adding_source": 1 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "counts": { "original_grep": { "new_physical_lines": 11, "annotated_lines": 6, "responses_adding_source": 4 }, "completed_grep": { "new_physical_lines": 280, "annotated_lines": 76, "responses_adding_source": 4 }, "original_read": { "new_physical_lines": 79, "annotated_lines": 79, "responses_adding_source": 1 }, "completed_read": { "new_physical_lines": 54, "annotated_lines": 54, "responses_adding_source": 1 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "counts": { "original_grep": { "new_physical_lines": 56, "annotated_lines": 11, "responses_adding_source": 7 }, "completed_grep": { "new_physical_lines": 197, "annotated_lines": 37, "responses_adding_source": 7 }, "original_read": { "new_physical_lines": 358, "annotated_lines": 109, "responses_adding_source": 4 }, "completed_read": { "new_physical_lines": 18, "annotated_lines": 2, "responses_adding_source": 3 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "counts": { "original_read": { "new_physical_lines": 928, "annotated_lines": 171, "responses_adding_source": 10 }, "completed_read": { "new_physical_lines": 213, "annotated_lines": 0, "responses_adding_source": 5 }, "original_grep": { "new_physical_lines": 118, "annotated_lines": 8, "responses_adding_source": 3 }, "completed_grep": { "new_physical_lines": 218, "annotated_lines": 35, "responses_adding_source": 2 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "counts": { "original_grep": { "new_physical_lines": 127, "annotated_lines": 10, "responses_adding_source": 6 }, "completed_grep": { "new_physical_lines": 584, "annotated_lines": 95, "responses_adding_source": 6 }, "original_read": { "new_physical_lines": 757, "annotated_lines": 83, "responses_adding_source": 6 }, "completed_read": { "new_physical_lines": 56, "annotated_lines": 5, "responses_adding_source": 3 } } } ] }, "required-hop-candidate": { "n_tasks": 8, "totals": { "completed_grep": { "new_physical_lines": 2123, "annotated_lines": 278, "responses_adding_source": 30, "annotated_fraction": 0.13094677343382008 }, "completed_read": { "new_physical_lines": 382, "annotated_lines": 115, "responses_adding_source": 16, "annotated_fraction": 0.3010471204188482 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0, "annotated_fraction": 0 }, "original_grep": { "new_physical_lines": 642, "annotated_lines": 96, "responses_adding_source": 40, "annotated_fraction": 0.14953271028037382 }, "original_read": { "new_physical_lines": 4495, "annotated_lines": 1130, "responses_adding_source": 39, "annotated_fraction": 0.25139043381535037 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "counts": { "original_grep": { "new_physical_lines": 17, "annotated_lines": 14, "responses_adding_source": 2 }, "completed_grep": { "new_physical_lines": 72, "annotated_lines": 51, "responses_adding_source": 2 }, "original_read": { "new_physical_lines": 48, "annotated_lines": 42, "responses_adding_source": 1 }, "completed_read": { "new_physical_lines": 76, "annotated_lines": 30, "responses_adding_source": 1 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "counts": { "original_grep": { "new_physical_lines": 132, "annotated_lines": 2, "responses_adding_source": 6 }, "completed_grep": { "new_physical_lines": 339, "annotated_lines": 0, "responses_adding_source": 4 }, "original_read": { "new_physical_lines": 418, "annotated_lines": 59, "responses_adding_source": 7 }, "completed_read": { "new_physical_lines": 103, "annotated_lines": 0, "responses_adding_source": 2 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "counts": { "original_grep": { "new_physical_lines": 145, "annotated_lines": 22, "responses_adding_source": 9 }, "completed_grep": { "new_physical_lines": 481, "annotated_lines": 0, "responses_adding_source": 6 }, "original_read": { "new_physical_lines": 1110, "annotated_lines": 384, "responses_adding_source": 8 }, "completed_read": { "new_physical_lines": 47, "annotated_lines": 12, "responses_adding_source": 4 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "counts": { "original_grep": { "new_physical_lines": 50, "annotated_lines": 11, "responses_adding_source": 2 }, "completed_grep": { "new_physical_lines": 35, "annotated_lines": 0, "responses_adding_source": 1 }, "original_read": { "new_physical_lines": 310, "annotated_lines": 196, "responses_adding_source": 1 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "counts": { "original_grep": { "new_physical_lines": 9, "annotated_lines": 6, "responses_adding_source": 3 }, "completed_grep": { "new_physical_lines": 198, "annotated_lines": 76, "responses_adding_source": 3 }, "original_read": { "new_physical_lines": 79, "annotated_lines": 79, "responses_adding_source": 1 }, "completed_read": { "new_physical_lines": 53, "annotated_lines": 53, "responses_adding_source": 1 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "counts": { "original_grep": { "new_physical_lines": 60, "annotated_lines": 20, "responses_adding_source": 5 }, "completed_grep": { "new_physical_lines": 229, "annotated_lines": 30, "responses_adding_source": 5 }, "original_read": { "new_physical_lines": 513, "annotated_lines": 100, "responses_adding_source": 5 }, "completed_read": { "new_physical_lines": 24, "annotated_lines": 20, "responses_adding_source": 2 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "counts": { "original_grep": { "new_physical_lines": 103, "annotated_lines": 16, "responses_adding_source": 6 }, "completed_grep": { "new_physical_lines": 237, "annotated_lines": 28, "responses_adding_source": 2 }, "original_read": { "new_physical_lines": 983, "annotated_lines": 175, "responses_adding_source": 10 }, "completed_read": { "new_physical_lines": 34, "annotated_lines": 0, "responses_adding_source": 3 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "counts": { "original_grep": { "new_physical_lines": 126, "annotated_lines": 5, "responses_adding_source": 7 }, "completed_grep": { "new_physical_lines": 532, "annotated_lines": 93, "responses_adding_source": 7 }, "original_read": { "new_physical_lines": 1034, "annotated_lines": 95, "responses_adding_source": 6 }, "completed_read": { "new_physical_lines": 45, "annotated_lines": 0, "responses_adding_source": 3 }, "graph": { "new_physical_lines": 0, "annotated_lines": 0, "responses_adding_source": 0 } } } ] } }, "targeted_graph_line_novelty": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "graph_physical_lines_returned": 39, "graph_new_physical_lines": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "graph_physical_lines_returned": 0, "graph_new_physical_lines": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "graph_physical_lines_returned": 0, "graph_new_physical_lines": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "graph_physical_lines_returned": 0, "graph_new_physical_lines": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "graph_physical_lines_returned": 133, "graph_new_physical_lines": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "graph_physical_lines_returned": 0, "graph_new_physical_lines": 0 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "graph_physical_lines_returned": 0, "graph_new_physical_lines": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "graph_physical_lines_returned": 48, "graph_new_physical_lines": 0 } ], "conclusion": "All 220 graph-returned physical lines were already observed in the targeted-hop development trial. Grep completion contributes most new completion lines and most newly observed annotated lines; no causal claim that removing it improves an online agent. Next development gates must measure useful novel source. Do not tune on the completed independent 32.", "annotation_correction_protocol_sha256": "12abf234264f904a4b54d4df6615fae1d2b9b6210eed671fdb926644ec96bf43" }, "early_unseen_development_trial": { "n": 8, "scope": "Development selection only; eight disclosed tasks, no use of the completed independent 32 for tuning.", "implementation_commit": "768785a421d2630ff31a75c3b387df319726bd75", "protocol": { "scope": "Development selection only; eight disclosed tasks, no use of the completed independent 32 for tuning.", "implementation_commit": "768785a421d2630ff31a75c3b387df319726bd75", "selection_sha256": "3f271fffc1394591c485e4eee5e2501e39e07e0e6146263392ee8ab0e889f568", "arms": { "native": { "source_policy": "native", "graph_navigation": false }, "functions": { "source_policy": "functions", "graph_navigation": false }, "functions_graph": { "source_policy": "functions", "graph_navigation": true } }, "graph_source_policy": "unseen", "graph_policy": "early_reference_hop_before_expanding_reads", "model": "claude-opus-5-5", "max_turns": 20, "completion_tokens": 2048, "graph_tokens": 2048, "max_meter_usd_per_arm": 10, "case_meter_usd": 3, "case_timeout_seconds": 600, "billing_route": "existing_claude_max", "additional_API_spending_ceiling_usd": 100, "automatic_API_fallback": false, "primary_contrast": "functions_graph minus native, complete route; both completion and graph policy change.", "secondary_contrasts": [ "functions minus native", "functions_graph minus functions" ], "metrics": "All eight selected tasks retained, official macro per-task cumulative observed line Recall, precision, F1.", "uncertainty": { "method": "paired task bootstrap", "draws": 10000, "seed": 20261016, "secondary_multiplicity": "Exploratory; no adjusted significance claim." }, "development_gate": { "all_tasks_complete_and_scored": true, "graph_supplies_at_least_one_previously_unseen_annotated_line": true, "primary_recall_point_increased": true, "primary_f1_point_not_lower": true }, "scope_of_passing": "Development candidate selection only; requires further development breadth and a new independent validation protocol before promotion.", "policy_changes_after_start": "None. No benchmark labels, reference patches or score feedback enter model prompts." }, "protocol_sha256": "c11ee7dc2fd58dab22c548929d42edeb5a89d02a89205689141bff70dec24a4b", "implementation": { "scripts/benchmark_context_agent.py": "c1691033bd8d5654e2d4187b9442a1c0da0df02ce66d346c6cd94988e641a8b6", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "48c509b47f82933ab59186ec35ad13221926baecd5dda9ba203ab5944cc54292", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "source_manifest_sha256": "73233491d39de65c36aff7ab290610f47f06ec2b3b294ed77e117c7fc27345bc", "graph_manifest_sha256": "883db1e8dbef3b81597e23bbc9ff4286ce611e66fb9d524cd6d7bbf2898aaa92", "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "arms": { "native": { "scores": { "trajectory": { "recall": 0.3915030969331417, "precision": 0.35742942488383506, "f1": 0.3426867221995677, "official_valid": 8 }, "served_16384": { "recall": 0.3908931172102317, "precision": 0.3570917327364308, "f1": 0.34220720930451654, "official_valid": 8 }, "served_32768": { "recall": 0.3908931172102317, "precision": 0.3570917327364308, "f1": 0.34220720930451654, "official_valid": 8 } }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 2.0107074, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 8 }, "cases": 8 }, "values": { "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.44398340248962653, "precision": 0.5095238095238095, "f1": 0.4745011086474501 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.25510204081632654, "precision": 0.07363770250368189, "f1": 0.11428571428571428 }, "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.3323593864134405, "precision": 0.3415915915915916, "f1": 0.33691225472047387 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5240506329113924, "f1": 0.530089628681178 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.4791666666666667, "precision": 0.8582089552238806, "f1": 0.6149732620320856 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.15100316789862725, "precision": 0.20634920634920634, "f1": 0.174390243902439 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.35456110154905335, "precision": 0.18181818181818182, "f1": 0.24037339556592766 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.5795795795795796, "precision": 0.16425531914893618, "f1": 0.25596816976127323 } }, "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 } }, "functions": { "scores": { "trajectory": { "recall": 0.43971001609480276, "precision": 0.3157302037855129, "f1": 0.3402071174334975, "official_valid": 8 }, "served_16384": { "recall": 0.4348301362149229, "precision": 0.3282325517377529, "f1": 0.3480307588668886, "official_valid": 8 }, "served_32768": { "recall": 0.43971001609480276, "precision": 0.3157607276933164, "f1": 0.3402397946124174, "official_valid": 8 } }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 2.6665744, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 8 }, "cases": 8 }, "values": { "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.5684647302904564, "precision": 0.5805084745762712, "f1": 0.5744234800838575 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.30612244897959184, "precision": 0.0664451827242525, "f1": 0.10919017288444041 }, "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.293644996347699, "precision": 0.21157894736842106, "f1": 0.245946772713368 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5240506329113924, "f1": 0.530089628681178 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.5041666666666667, "precision": 0.42086956521739133, "f1": 0.4587677725118484 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.2787750791974657, "precision": 0.4012158054711246, "f1": 0.32897196261682243 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.35456110154905335, "precision": 0.17591801878736124, "f1": 0.2351598173515982 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.6756756756756757, "precision": 0.14525500322788895, "f1": 0.23910733262486714 } }, "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 } }, "functions_graph": { "scores": { "trajectory": { "recall": 0.4403295826632678, "precision": 0.3641481052549322, "f1": 0.36600887131592075, "official_valid": 8 }, "served_16384": { "recall": 0.4390387048663659, "precision": 0.3789737299241462, "f1": 0.37570271984183523, "official_valid": 8 }, "served_32768": { "recall": 0.4403295826632678, "precision": 0.36421733552978586, "f1": 0.3660735713640091, "official_valid": 8 } }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 2.5889262000000004, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 8 }, "cases": 8 }, "values": { "SWE-Bench-Verified__python__maintenance__bugfix__1409977d": { "recall": 0.5684647302904564, "precision": 0.9013157894736842, "f1": 0.6972010178117047 }, "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b": { "recall": 0.20408163265306123, "precision": 0.04561003420752566, "f1": 0.07455731593662629 }, "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8": { "recall": 0.506208911614317, "precision": 0.414721723518851, "f1": 0.45592105263157895 }, "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e": { "recall": 0.5362694300518135, "precision": 0.5348837209302325, "f1": 0.5355756791720568 }, "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92": { "recall": 0.45625, "precision": 0.36622073578595316, "f1": 0.40630797773654914 }, "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b": { "recall": 0.27983104540654696, "precision": 0.3566621803499327, "f1": 0.31360946745562124 }, "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647": { "recall": 0.3769363166953528, "precision": 0.1543340380549683, "f1": 0.219 }, "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163": { "recall": 0.5945945945945946, "precision": 0.13943661971830987, "f1": 0.22589845978322876 } }, "actual_models": [ "claude-opus-5-5" ], "statuses": { "ok": 8 } } }, "comparisons": { "primary": { "n": 8, "contrast": "functions_and_directed_graph_over_native", "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "metrics": { "recall": { "baseline": 0.3915030969331417, "candidate": 0.4403295826632678, "difference": 0.048826485730126074, "paired_bootstrap_95_interval": [ -0.0011230693365482658, 0.1045021972724606 ] }, "precision": { "baseline": 0.35742942488383506, "candidate": 0.3641481052549322, "difference": 0.006718680371097125, "paired_bootstrap_95_interval": [ -0.1621506413003429, 0.1613752447606018 ] }, "f1": { "baseline": 0.3426867221995677, "candidate": 0.36600887131592075, "difference": 0.023322149116353025, "paired_bootstrap_95_interval": [ -0.06443609944957052, 0.11033728202662997 ] } } }, "functions": { "n": 8, "contrast": "online_function_completion", "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "metrics": { "recall": { "baseline": 0.3915030969331417, "candidate": 0.43971001609480276, "difference": 0.04820691916166102, "paired_bootstrap_95_interval": [ 0.008583414495576633, 0.08894867264493111 ] }, "precision": { "baseline": 0.35742942488383506, "candidate": 0.3157302037855129, "difference": -0.04169922109832213, "paired_bootstrap_95_interval": [ -0.17407796356605812, 0.06318871158103542 ] }, "f1": { "baseline": 0.3426867221995677, "candidate": 0.3402071174334975, "difference": -0.002479604766070209, "paired_bootstrap_95_interval": [ -0.06517423287420498, 0.061700439910588045 ] } } }, "graph": { "n": 8, "contrast": "directed_graph_over_function_completion", "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "metrics": { "recall": { "baseline": 0.43971001609480276, "candidate": 0.4403295826632678, "difference": 0.0006195665684650548, "paired_bootstrap_95_interval": [ -0.05348209344063262, 0.07001725962972317 ] }, "precision": { "baseline": 0.3157302037855129, "candidate": 0.3641481052549322, "difference": 0.04841790146941925, "paired_bootstrap_95_interval": [ -0.027301128762629214, 0.14744926064798108 ] }, "f1": { "baseline": 0.3402071174334975, "candidate": 0.36600887131592075, "difference": 0.025801753882423232, "paired_bootstrap_95_interval": [ -0.024539212355341955, 0.09151877074392979 ] } } } }, "development_gate": { "protocol_sha256": "c11ee7dc2fd58dab22c548929d42edeb5a89d02a89205689141bff70dec24a4b", "analysis_sha256": "738022ae4f72fee6a02f65bd3642b1982c7b90eb3e3a5144c2b964dd66d0890d", "gates": { "all_tasks_complete_and_scored": true, "graph_supplies_at_least_one_previously_unseen_annotated_line": false, "primary_recall_point_increased": true, "primary_f1_point_not_lower": true }, "passed": false, "graph_adoption": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": {} } ], "primary_metrics": { "recall": { "baseline": 0.3915030969331417, "candidate": 0.4403295826632678, "difference": 0.048826485730126074, "paired_bootstrap_95_interval": [ -0.0011230693365482658, 0.1045021972724606 ] }, "precision": { "baseline": 0.35742942488383506, "candidate": 0.3641481052549322, "difference": 0.006718680371097125, "paired_bootstrap_95_interval": [ -0.1621506413003429, 0.1613752447606018 ] }, "f1": { "baseline": 0.3426867221995677, "candidate": 0.36600887131592075, "difference": 0.023322149116353025, "paired_bootstrap_95_interval": [ -0.06443609944957052, 0.11033728202662997 ] } }, "scope": "Development selection gate only; no independent quality or leaderboard claim." }, "solver_pair": { "protocol": { "scope": "Fresh paired development solver and model-input receipt check. Same disclosed Python development task; not independent evidence or best-of selection.", "selection": "First CSV-ordered development task, also the only Verified task in this eight-task cohort. Selected before new retrieval scores.", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "original_inst_id": "pylint-dev__pylint-4604", "arms": [ "native", "functions_graph" ], "implementation_commit": "54a8f15f8f23c66cf2540dc0ce030ccaa3c5826c", "image_id": "sha256:b5fac9ce8ae7da0c6ad028bce73932b257eb44656985e4f7c7d73d7dfc60dcf7", "source_tree": "05a6b93fcb973fb72fe368b7d43eee00ed6635da", "source_commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "reference_check_sha256": "8b5ccd94047435d0ef0fbb0b7fbfb5aa400254ddaa30c5ac657ed2f5e6e5e0e7", "model": "claude-opus-5-5", "context_budget": 16384, "max_turns": 30, "max_meter_usd_per_arm": 3, "native_dataset_sha256": "d92ef4dadc6482fc80a961e1a3a313713c78567721bb11b96dda6dab621ed71f", "failure_handling": "One attempt per arm, no grading feedback or reference patch supplied; stop on unknown metering or SDK errors. Keep all failures.", "billing": "Existing Claude Max only; no automatic API fallback.", "observation_gate": "Every actual model request has a verified content hash; confirmed inputs export after native output formatting. Arbitrary shell-to-source line attribution remains open." }, "arms": { "native": { "status": "Submitted", "resolved": false, "tasks": 1, "model_calls": 7, "known_sdk_reported_cost_usd": 0.4858567, "missing_metering_calls": 0, "container_removed": true, "all_model_inputs_confirmed": true, "confirmed_model_inputs": 7, "model_observations_sha256": "29577410422e8031dc21d1769c76f7f05e3a53a9adc50edd2ecb244641f00652", "source_attribution": "Pending; no shell source-line credit inferred." }, "functions_graph": { "status": "Submitted", "resolved": false, "tasks": 1, "model_calls": 11, "known_sdk_reported_cost_usd": 0.8819372000000001, "missing_metering_calls": 0, "container_removed": true, "all_model_inputs_confirmed": true, "confirmed_model_inputs": 11, "model_observations_sha256": "18d33f64a748c976c7ac5bda2c15804559916647cba3ae816ea0cb169013e5f2", "source_attribution": "Pending; no shell source-line credit inferred." } }, "implementation": { "scripts/benchmark_context_solver.py": "42f6097d6e7ba927852f460775ae8a10cadea1f1fe4974ae1358d264a278c15e", "mini/agents/default.py": "faf4442506b1cdaad886f146182dff98ed740d45536729617efbdcb6d5fb5524", "mini/environments/docker.py": "e3c42a76d0aee03edb0e14e693c963d4fc2e28b678ca36a72f774c90dc89dbd4", "mini/config/extra/swebench.yaml": "81bb47b91311e19b73ff0f8bc670898dc5b807f774ff687e11556038e5878162" }, "scope": "Repeated development task, 0/1 resolved in each new arm; not independent evidence or a population Pass@1 estimate." }, "cost": { "retrieval_sdk_estimate_usd": 7.266208, "solver_sdk_estimate_usd": 1.3677939000000001, "cumulative_program_known_sdk_estimate_usd": 50.25750360000001, "cumulative_program_unmetered_attempts": [ "solver-smoke-functions-v2" ], "standalone_api_key_inference_spend_usd": 0, "actual_subscription_bill_usd": null, "scope": "SDK estimates, not invoices. Program total includes previous development and the completed independent 32; one earlier aborted solver request remains unmetered.", "snapshot_scope": "Historical snapshot at early/unseen trial completion; current_program_cost contains subsequent mechanism probes." }, "conclusion": "The gate fails: all eight graph-enabled tasks make zero graph calls. Positive development point differences cannot be attributed to graph-returned source and both primary intervals cross zero. Require an adoption/novelty mechanism check before another quality cohort; no new independent validation or full-500 run." }, "first_callable_graph_mechanism": { "scope": "Development mechanism checks only, excluded from quality/leaderboard scores. No independent-validation tasks.", "offline": { "selection_sha256": "3f271fffc1394591c485e4eee5e2501e39e07e0e6146263392ee8ab0e889f568", "anchor_rule": "First actual read response intersecting a compiler callable. Choose maximum count of returned physical lines in a callable, then smallest callable span and lexical name; use its earliest observed line. This rule is fixed before annotation overlap is measured.", "observations": "Actual source-checked message prefix through that read; both directions use the same prefix independently.", "annotation_use": "Only compute overlap after anchor selection and graph responses. Do not select a better anchor from annotations.", "directions": [ "incoming", "outgoing" ], "limit_per_call": 5, "graph_tokens_per_call": 2048, "neighbor_order_change": "Functions/methods precede other defined symbols within the same 1000-raw-edge scan; stable edge order within groups.", "protocol_sha256": "ff24ad2ec7b18d48e3135f48fd8da4701771195af8a59f35571fc72d42a8feaf", "baseline_summary_sha256": "9399c750f331ae9d6a028f659cc4de79760776f9309173b11fb27660e9529d58", "analysis_sha256": "3233844bcdd387affa280dbaef012fd7fe40dd94c8b7b5108cfcec8806b1ffa8", "implementation": { "scripts/benchmark_context_agent.py": "a5c4a762721142fc3c910e79e2ed6e7a60f2e991ce80b535afa27d5f2a077b26", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "09729d03dd77d4a4c1504f7c77a31c0a0b72fabe9fd70031946e14f57cc20d0e", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "model_calls": 0, "new_physical_lines_union_per_task": { "before": 256, "after": 412 }, "new_annotated_physical_lines_union_per_task": { "before": 1, "after": 16 }, "interpretation": "Only the Go anchor adds annotated source; its previous annotated line is displaced. Physical blank/comment lines are counted. This is not official Recall/F1 or an online quality score.", "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "graph_available": true, "directions": { "incoming": { "before_new_lines": 27, "after_new_lines": 27, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 27, "removed_annotated_lines": 0, "statuses": [ "source_returned" ], "node_types": [ "method" ] }, "outgoing": { "before_new_lines": 59, "after_new_lines": 64, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 58, "removed_annotated_lines": 0, "statuses": [ "source_returned", "source_returned", "source_returned", "source_returned", "source_returned" ], "node_types": [ "method", "function", "function", "function", "class" ] } }, "union_directions": { "before_new_lines": 86, "after_new_lines": 91, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "graph_available": true, "directions": { "incoming": { "before_new_lines": 5, "after_new_lines": 5, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 5, "removed_annotated_lines": 0, "statuses": [ "source_not_returned", "source_returned" ], "node_types": [ "function", "function" ] }, "outgoing": { "before_new_lines": 5, "after_new_lines": 128, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 0, "removed_annotated_lines": 0, "statuses": [ "source_returned", "source_returned", "source_returned", "source_returned", "source_returned" ], "node_types": [ "function", "function", "function", "function", "method" ] } }, "union_directions": { "before_new_lines": 10, "after_new_lines": 133, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "graph_available": false, "directions": {} }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "graph_available": false, "directions": {} }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "graph_available": true, "directions": { "incoming": { "before_new_lines": 0, "after_new_lines": 0, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 0, "removed_annotated_lines": 0, "statuses": [ "no_returned_neighbors" ], "node_types": [] }, "outgoing": { "before_new_lines": 5, "after_new_lines": 101, "before_new_annotated_lines": 1, "after_new_annotated_lines": 16, "retained_new_lines": 0, "removed_annotated_lines": 1, "statuses": [ "source_returned", "source_returned", "source_returned", "source_returned", "source_returned" ], "node_types": [ "function", "function", "method", "function", "function" ] } }, "union_directions": { "before_new_lines": 5, "after_new_lines": 101, "before_new_annotated_lines": 1, "after_new_annotated_lines": 16 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "graph_available": false, "directions": {} }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "graph_available": true, "directions": { "incoming": { "before_new_lines": 0, "after_new_lines": 0, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 0, "removed_annotated_lines": 0, "statuses": [ "no_returned_neighbors" ], "node_types": [] }, "outgoing": { "before_new_lines": 25, "after_new_lines": 25, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 25, "removed_annotated_lines": 0, "statuses": [ "source_returned", "source_returned" ], "node_types": [ "function", "field" ] } }, "union_directions": { "before_new_lines": 25, "after_new_lines": 25, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "graph_available": true, "directions": { "incoming": { "before_new_lines": 16, "after_new_lines": 16, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 16, "removed_annotated_lines": 0, "statuses": [ "source_returned" ], "node_types": [ "method" ] }, "outgoing": { "before_new_lines": 114, "after_new_lines": 46, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0, "retained_new_lines": 7, "removed_annotated_lines": 0, "statuses": [ "source_returned", "source_returned", "source_returned", "source_returned", "source_returned" ], "node_types": [ "method", "method", "method", "method", "function" ] } }, "union_directions": { "before_new_lines": 130, "after_new_lines": 62, "before_new_annotated_lines": 0, "after_new_annotated_lines": 0 } } ] }, "online_versions": [ { "scope": "Single development mechanism probe on the original issue; no quality or leaderboard score. No task-specific source anchors supplied to model.", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "protocol_sha256": "02d858fdef7fa2a3c69a3358c97c37c8dbacdd85204dd86ecf5b7399c08da21a", "result_sha256": "c82bfb91ab783b66942255342fced1786dd71cff20f329dcadd0ce4306661683", "analysis_sha256": "8edd91d9748c91c9da229dc09b887ce1aae80c15919540bdbd05a940a219ca33", "implementation": { "scripts/benchmark_context_agent.py": "a5c4a762721142fc3c910e79e2ed6e7a60f2e991ce80b535afa27d5f2a077b26", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "09729d03dd77d4a4c1504f7c77a31c0a0b72fabe9fd70031946e14f57cc20d0e", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "model_requested": "claude-opus-5-5", "actual_models": [ "claude-opus-5-5" ], "graph_policy": "first_callable_read_reference_hop", "first_callable_read": { "message_index": 8, "tool": "read", "next_tools": [ "grep" ] }, "first_callable_source": { "message_index": 3, "tool": "grep", "next_tools": [ "graph_neighbors" ] }, "confirmed_new_graph_physical_lines": 132, "confirmed_new_graph_annotated_physical_lines": 132, "strict_first_read_timing_passed": false, "first_source_timing_passed": true, "predeclared_timing_passed": false, "predeclared_gate_passed": false, "status": "ok", "sdk_reported_cost_usd": 0.201015, "sdk_metering_is_actual_billing": false, "actual_bill_usd": null, "missing_metering_calls": 0, "quality_score": null, "quality_claim": false, "tool_sequence": [ "grep", "graph_neighbors", "read", "grep", "grep" ], "name": "online-probe", "graph_calls": [ { "message_index": 6, "following_assistant_confirms_delivery": true, "returned_physical_lines": 132, "new_physical_lines": 132, "statuses": [ "source_returned" ] } ], "limits": { "max_turns": 8, "case_timeout_seconds": 180, "sdk_meter_guard_usd": 1.0, "source_policy": "functions", "completion_tokens": 2048, "graph_source_policy": "unseen", "graph_tokens": 2048 }, "predeclared_gate": "The first new assistant tool-call turn after a callable-bearing read calls graph_neighbors before further source tools, and a delivered graph response adds previously unseen physical source.", "selected_source_anchor_in_prompt": false, "attempts_for_this_version": 1 }, { "scope": "Second development mechanism version on the same original issue; first version retained separately; no quality or leaderboard score. No task-specific source anchors supplied to model.", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "protocol_sha256": "3429ba8b216af5593479f50629ce33729863db4774a82979c4c3aa281cba44a0", "result_sha256": "51a41236b11494403da6348cf69e688cc7b02c3ef1d64e1c0283aadc95309d9a", "analysis_sha256": "8edd91d9748c91c9da229dc09b887ce1aae80c15919540bdbd05a940a219ca33", "implementation": { "scripts/benchmark_context_agent.py": "3a0b3c1604ed09532eabd1173349c60f9497019b83131023e85de752d13a922b", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "26ee7bc5e203b4c028f4b4acc5d019b106b04ab3c8365ec6a3fb42bc9b220bc9", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "model_requested": "claude-opus-5-5", "actual_models": [ "claude-opus-5-5" ], "graph_policy": "first_callable_source_reference_hop", "first_callable_read": null, "first_callable_source": { "message_index": 3, "tool": "grep", "next_tools": [ "graph_neighbors" ] }, "confirmed_new_graph_physical_lines": 132, "confirmed_new_graph_annotated_physical_lines": 132, "strict_first_read_timing_passed": false, "first_source_timing_passed": true, "predeclared_timing_passed": true, "predeclared_gate_passed": true, "status": "ok", "sdk_reported_cost_usd": 0.1261007, "sdk_metering_is_actual_billing": false, "actual_bill_usd": null, "missing_metering_calls": 0, "quality_score": null, "quality_claim": false, "tool_sequence": [ "grep", "graph_neighbors", "grep", "grep" ], "name": "online-probe-source-trigger", "graph_calls": [ { "message_index": 6, "following_assistant_confirms_delivery": true, "returned_physical_lines": 132, "new_physical_lines": 132, "statuses": [ "source_returned" ] } ], "limits": { "max_turns": 8, "case_timeout_seconds": 180, "sdk_meter_guard_usd": 1.0, "source_policy": "functions", "completion_tokens": 2048, "graph_source_policy": "unseen", "graph_tokens": 2048 }, "predeclared_gate": "The first new assistant tool-call turn after the first callable-bearing source reply (read or function-completed grep) calls graph_neighbors before further source tools, and a delivered graph response adds previously unseen physical source.", "selected_source_anchor_in_prompt": false, "attempts_for_this_version": 1 } ], "conclusion": "The revised source-observation trigger passes a one-task mechanism gate with 132 new annotated physical lines. This establishes no Recall/F1 or Pass@1 gain. Preserve both versions and the earlier negative independent validation. Broader fresh matched development evidence is required before a new independent quality selection." }, "current_program_cost": { "known_sdk_reported_estimate_usd": 182.31928416, "mechanism_probes_sdk_estimate_usd": 0.3271157, "unmetered_solver_attempts": [ "solver-smoke-functions-v2" ], "standalone_api_key_inference_calls": 1012, "standalone_api_key_inference_cost_usd": 0.36997736999999997, "authorized_new_api_spend_usd": 100, "sdk_metering_is_actual_billing": false, "actual_bill_usd": null, "scope": "As-of verified500_context_execution:knownfreshMax nominal includesoriginal36/quota65/capacity53(allknownincloneSDKrejectedaction) pluscurrentcases once. Oneoriginalunknownterminal retained,notzero. AddtopriorMax113.75041130/mixed162.32272866,pendingexcluded. API59.09398873unchanged,no newBatch/count/repair. Maxnominalnotcash;historical2Maxgaps plusoriginal1gapremain;billing/creditbalanceunverified.", "unmetered_retrieval_cases": [ { "run": "first-source-24/functions", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "reason": "Unexpected SDK tool attempt ended stream before terminal usage." } ], "first_source_24_retrieval_sdk_estimate_usd": 12.8310857, "post_fix_reliability_sdk_estimate_usd": 0.42661920000000003, "multi_fmt_development_solver_sdk_estimate_usd": 1.3218016000000001, "late_graph_8_retrieval_sdk_estimate_usd": 6.796706500000001, "reader_adoption_v1_sdk_estimate_usd": 0.6959919, "reader_adoption_v2_sdk_estimate_usd": 0.93285, "multi_gson_development_solver_sdk_estimate_usd": 1.0885025000000002, "multi_ponyc_development_solver_sdk_estimate_usd": 8.1156106, "ponyc_terminal_metering_correction": { "original_aggregate_missing_calls": 1, "recovered_from_preserved_terminal_event_usd": 0.3218848, "remaining_unmetered_calls_in_this_comparison": 0, "original_failed_attempt_unchanged": true, "reconciliation_protocol_sha256": "955db06b47399b76fec281ab182e7c17d77cb48cb2c67e519487b7941ad9694e" }, "late_opposite_followup_8_retrieval_sdk_estimate_usd": 5.0872571, "solver_search_adoption_sdk_estimate_usd": 2.2842187, "jev_rerank_api_cost_usd": 0.025743857999999994, "unmetered_api_calls": 0, "jev_grep_api_cost_usd": 0.012341112, "known_claude_max_sdk_estimate_usd": 133.7469668, "known_anthropic_api_sdk_estimate_usd": 48.57231736, "standalone_api_call_count_scope": "1012 Jev decisions;216 retrieval SDK sessions,36 packet-review SDK sessions and55 historical repair SDK queries.112 Messages inference requests and160 free token counts are separate; SDK sessions are not HTTP request counts.", "anthropic_api_sdk_sessions": 307, "known_combined_api_usage_value_usd": 59.09398873000001, "online_grep_jev_retrieval_api_sdk_estimate_usd": 5.4908492, "online_grep_jev_decision_api_cost_usd": 0.00552048, "online_grep_jev_repair_max_sdk_estimate_usd": 7.2426413, "online_grep_hints_retrieval_api_sdk_estimate_usd": 4.8388315, "online_grep_hints_decision_api_cost_usd": 0.009593262, "online_grep_hints_repair_max_sdk_estimate_usd": 6.0355762, "retrieval_notes_repair_max_sdk_estimate_usd": 9.058375, "haiku_repair_api_sdk_estimate_usd": 0.26029216, "file_map_retrieval_api_sdk_estimate_usd": 1.496507, "grep_neighbors_pilot_api_sdk_estimate_usd": 1.7605027, "grep_neighbors_replication_api_sdk_estimate_usd": 1.8852270999999998, "grep_neighbors_total_api_sdk_estimate_usd": 3.6457298, "neighbors_independent32_api_sdk_estimate_usd": 12.630568599999995, "neighbors_independent32_sdk_sessions": 64, "unmetered_anthropic_independent32_sessions": 0, "novel_source_jev_probe_cost_usd": 0.020709527999999998, "file_window_jev_probe_cost_usd": 0.024354119999999993, "file_source_online_pilot_api_sdk_estimate_usd": 1.6432091, "file_source_online_pilot_sdk_sessions": 8, "all_file_source_online_api_sdk_estimate_usd": 9.8611063, "all_file_source_online_sdk_sessions": 48, "late_source_jev_probe_cost_usd": 0.27171501, "late_packet_input_api_sdk_estimate_usd": 4.631439, "late_packet_input_sdk_sessions": 36, "remaining20_late_packet_input_api_sdk_estimate_usd": 2.8706633000000004, "remaining20_late_packet_input_sdk_sessions": 24, "known_direct_anthropic_messages_usage_estimate_usd": 9.721476000000001, "direct_anthropic_messages_requests": 112, "direct_anthropic_free_token_count_requests": 168, "all_packet_delivery_api_usage_usd": 14.352915000000003, "unmetered_direct_messages_requests": 0, "late_source_amended_independent32_native_sdk_usd": 5.716993799999999, "late_source_amended_independent32_messages_api_usd": 5.466732000000001, "late_source_amended_independent32_combined_api_usd": 11.183725800000001, "known_direct_anthropic_batch_usage_estimate_usd": 0.430218, "direct_anthropic_batch_message_requests": 8, "direct_anthropic_batch_create_posts": 1, "batch_free_token_count_requests": 8, "max_batch_development_Max_sdk_sessions": 4, "max_batch_development_Max_nominal_sdk_usd": 1.2485557, "unmetered_batch_requests": 0, "verified500_native_checkpoint": "2026-10-17T19:25:39+00:00", "verified500_new_Max_nominal_sdk_usd": 19.996555500000007, "verified500_new_Max_sdk_terminal_sessions": 158, "verified500_unknown_Max_terminal_cases": 1 }, "first_callable_source_24_development": { "scope": "Fresh three-arm development attempt on all 24 previously disclosed tasks. The primary pair is complete; function-only ablations are incomplete. Not independent validation or a leaderboard result.", "protocol": { "scope": "Three fresh arms on the disclosed 24 development tasks, three per language. This is development selection, not independent validation or a leaderboard score.", "implementation_commit": "63e035eb79635f45c578c718a0750785dace3c39", "selection_sha256": "122a70d4c61abcea5c27f87781fc204db768d933dd0d3bb7516e20117866f513", "selection": "Reuse every task in the previously disclosed 24-task selection; no new score-based filtering. Includes all eight earlier pilot tasks and 16 additional development tasks.", "n": 24, "arms": { "native": { "source_policy": "native", "graph_navigation": false }, "functions": { "source_policy": "functions", "graph_navigation": false }, "functions_graph": { "source_policy": "functions", "graph_navigation": true } }, "graph_source_policy": "unseen", "graph_policy": "first_callable_source_reference_hop", "neighbor_order": "Functions/methods before other defined symbols inside the unchanged raw-edge budget.", "model": "claude-opus-5-5", "max_turns": 20, "completion_tokens": 2048, "graph_tokens": 2048, "max_meter_usd_per_arm": 12, "case_meter_usd": 3, "case_timeout_seconds": 600, "billing_route": "existing_claude_max", "additional_API_spending_ceiling_usd": 100, "automatic_API_fallback": false, "primary_contrast": "functions_graph minus native, complete route.", "secondary_contrasts": [ "functions minus native", "functions_graph minus functions" ], "metrics": "Official per-task macro cumulative observed line Recall, precision and F1 over all 24 selected tasks; failed or missing scores remain zero.", "uncertainty": { "method": "paired task bootstrap", "draws": 10000, "seed": 20261016, "secondary_multiplicity": "Exploratory; no adjusted significance claim." }, "development_gate": { "all_selected_tasks_finish_and_score": true, "every_available_graph_called": true, "at_least_two_languages_receive_previously_unseen_annotated_graph_source": true, "primary_recall_point_increased": true, "primary_f1_point_not_lower": true }, "graph_ablation_interpretation": "Report the graph-versus-functions contrast separately; positive complete-route scores do not establish marginal graph benefit.", "source_delivery": "Count graph novelty only from actual source-checked tool replies followed by an assistant response; requested ranges and metadata earn no source credit.", "policy_changes_after_start": "None. No annotation or reference patch enters model prompts. All arms retain their first attempt; no best retries.", "scope_of_passing": "Candidate selection only. A new independent validation protocol and broader native solver evidence remain required before the official 500." }, "execution": { "protocol_sha256": "3def16e46468b5a98c9e4901fe8f68a768b87857cc21c4aef28f237b441edc96", "driver_sha256": "59aeeab8840a89143885a476d393349e7bf9d2cbd9829ee766fdc4dd039cf171", "graph_manifest_sha256": "1d8de298ffb44da0aa511ce439b4dab60ac85d0d3517cbcefdede2dffcfdec11", "source_manifest_sha256": "3574d68ad8832273efc58cae36db0b2361ec80a764e1714a16e5a89f5daaa43f", "state": "finished", "arms": { "functions": { "process_exit_code": 0 }, "functions_graph": { "process_exit_code": 0 }, "native": { "process_exit_code": 0 } } }, "dataset_revision": "c2855792b006af41c67202d33883fb9d46362853", "evaluator_revision": "1436c28a8eb95496da4ea69ad458b9f8a8eb7d61", "population_sources": { "Multi": 16, "Poly": 2, "Pro": 3, "Verified": 3 }, "arms": { "native": { "models": [ "claude-opus-5-5" ], "statuses": { "ok": 24 }, "scoring_failures": 0, "summary": { "n": 24, "arms": { "trajectory": { "recall": 0.5335256710568391, "precision": 0.31624050372742957, "f1": 0.31176405157915105, "official_valid": 24 }, "served_16384": { "recall": 0.5290730444172403, "precision": 0.32156232477720387, "f1": 0.31700750295252067, "official_valid": 24 }, "served_32768": { "recall": 0.5333242389837503, "precision": 0.3168462409239863, "f1": 0.31234242488057157, "official_valid": 24 } }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 5.4937837, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 24 }, "cases": 24 } }, "implementation": { "scripts/benchmark_context_agent.py": "3a0b3c1604ed09532eabd1173349c60f9497019b83131023e85de752d13a922b", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "26ee7bc5e203b4c028f4b4acc5d019b106b04ab3c8365ec6a3fb42bc9b220bc9", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "original_inst_id": "pylint-dev__pylint-4604", "base_commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "status": "ok", "trajectory": { "recall": 0.3651452282157676, "precision": 0.704, "f1": 0.48087431693989063 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "original_inst_id": "vuejs__core-10250", "base_commit": "d276a4f3e914aaccc291f7b2513e5d978919d0f9", "status": "ok", "trajectory": { "recall": 0.40816326530612246, "precision": 0.10854816824966079, "f1": 0.17148981779206862 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "original_inst_id": "microsoft__vscode-135197", "base_commit": "a993ada9300a531c09dfa6a1be772b1abc1492ad", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "original_inst_id": "django__django-15863", "base_commit": "37c5b8c07be104fd5288cd87f101e48cb7a40298", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.3169642857142857, "f1": 0.48135593220338985 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "original_inst_id": "ponylang__ponyc-2205", "base_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "status": "ok", "trajectory": { "recall": 0.36303871439006574, "precision": 0.3066008636644047, "f1": 0.3324414715719064 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "original_inst_id": "jqlang__jq-2919", "base_commit": "7f547827e47b5ade563a293329deb4226496d72f", "status": "ok", "trajectory": { "recall": 0.8706140350877193, "precision": 0.5251322751322751, "f1": 0.6551155115511551 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "original_inst_id": "fmtlib__fmt-2394", "base_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "status": "ok", "trajectory": { "recall": 0.5362694300518135, "precision": 0.5036496350364964, "f1": 0.519447929736512 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "original_inst_id": "instance_future-architect__vuls-b8db2e0b74f60cb7d45f710f255e061f054b6afc", "base_commit": "43b46cb324f64076e4d9e807c0b60c4b9ce11a82", "status": "ok", "trajectory": { "recall": 0.425, "precision": 0.8326530612244898, "f1": 0.5627586206896551 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "original_inst_id": "fasterxml__jackson-databind-4469", "base_commit": "7c9e7c1e2acd9c55926b9d2592fc65234d9c3ff7", "status": "ok", "trajectory": { "recall": 0.14889123548046462, "precision": 0.2256, "f1": 0.17938931297709926 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "original_inst_id": "darkreader__darkreader-7241", "base_commit": "991883df4d5910851130e3dc0e21fcbce604ea7d", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "original_inst_id": "nlohmann__json-3601", "base_commit": "2d48a4d9c5e8f0a5ce914922eb2a45dc0ec93ee3", "status": "ok", "trajectory": { "recall": 0.29678638941398866, "precision": 0.48606811145510836, "f1": 0.3685446009389672 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "original_inst_id": "catchorg__Catch2-1608", "base_commit": "979bbf03bb00bc55ca09783791b5091a2247df68", "status": "ok", "trajectory": { "recall": 0.5, "precision": 0.23148148148148148, "f1": 0.3164556962025316 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "original_inst_id": "facebook__zstd-1733", "base_commit": "a505463710aa34bccafd268c44064c129cdfb3e2", "status": "ok", "trajectory": { "recall": 0.2047244094488189, "precision": 0.19030192131747484, "f1": 0.19724988146040778 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "original_inst_id": "instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan", "base_commit": "1e137b07052bc3ea0da44ed201702c94055b8ad2", "status": "ok", "trajectory": { "recall": 0.3528399311531842, "precision": 0.2074898785425101, "f1": 0.26131293817718293 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "original_inst_id": "clap-rs__clap-3420", "base_commit": "135b15467eda2394b017f2a7d25cda1417c0feec", "status": "ok", "trajectory": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "original_inst_id": "tokio-rs__bytes-721", "base_commit": "9965a04b5684079bb614addd750340ffc165a9f5", "status": "ok", "trajectory": { "recall": 0.5652173913043478, "precision": 0.03117505995203837, "f1": 0.05909090909090909 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "original_inst_id": "serverless__serverless-3457", "base_commit": "fea269947104889165e3ebb0f64638921caf2052", "status": "ok", "trajectory": { "recall": 0.8996282527881041, "precision": 0.36890243902439024, "f1": 0.5232432432432432 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "original_inst_id": "nushell__nushell-13357", "base_commit": "5417c89387b67c3192ae9043473b556cd669ee15", "status": "ok", "trajectory": { "recall": 0.7195767195767195, "precision": 0.1348314606741573, "f1": 0.22710826607291956 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "original_inst_id": "google__gson-1391", "base_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "status": "ok", "trajectory": { "recall": 0.46819787985865724, "precision": 0.828125, "f1": 0.5981941309255079 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "original_inst_id": "cli__cli-4253", "base_commit": "e6ff77ce73c201b0ee36d2b802ea45e9e1ad1822", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "original_inst_id": "alibaba__fastjson2-2775", "base_commit": "12b40c7ba3e7c30e35977195770c80beb34715c5", "status": "ok", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "original_inst_id": "instance_flipt-io__flipt-c154dd1a3590954dfd3b901555fc6267f646a289", "base_commit": "e432032cf2d11e3bb6f558748a5ee41a90640daa", "status": "ok", "trajectory": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.17806935332708526 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "original_inst_id": "axios__axios-5661", "base_commit": "0abc70564746496eb211bbd951041b4655aec268", "status": "ok", "trajectory": { "recall": 0.23621323529411764, "precision": 0.7427745664739884, "f1": 0.3584379358437936 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "original_inst_id": "pydata__xarray-2905", "base_commit": "7c4e2ac83f7b4306296ff9b7b51aaf016e5ad614", "status": "ok", "trajectory": { "recall": 0.5901639344262295, "precision": 0.4044943820224719, "f1": 0.48 } } ] }, "functions": { "models": [ "claude-opus-5-5" ], "statuses": { "ok": 4, "error": 1, "not_run": 19 }, "scoring_failures": 19, "summary": { "n": 24, "arms": { "trajectory": { "recall": 0.12064738088464838, "precision": 0.042582924380788985, "f1": 0.054296528980621366, "official_valid": 5 }, "served_16384": { "recall": 0.12064738088464838, "precision": 0.042845524435860816, "f1": 0.054760142399322434, "official_valid": 5 }, "served_32768": { "recall": 0.12064738088464838, "precision": 0.042582924380788985, "f1": 0.054296528980621366, "official_valid": 5 } }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 0.961724, "missing_metering_cases": 1, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 4, "error": 1, "not_run": 19 }, "cases": 24 } }, "implementation": { "scripts/benchmark_context_agent.py": "3a0b3c1604ed09532eabd1173349c60f9497019b83131023e85de752d13a922b", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "26ee7bc5e203b4c028f4b4acc5d019b106b04ab3c8365ec6a3fb42bc9b220bc9", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "original_inst_id": "pylint-dev__pylint-4604", "base_commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "status": "ok", "trajectory": { "recall": 0.5726141078838174, "precision": 0.4046920821114369, "f1": 0.4742268041237113 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "original_inst_id": "vuejs__core-10250", "base_commit": "d276a4f3e914aaccc291f7b2513e5d978919d0f9", "status": "ok", "trajectory": { "recall": 0.30612244897959184, "precision": 0.09345794392523364, "f1": 0.1431980906921241 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "original_inst_id": "microsoft__vscode-135197", "base_commit": "a993ada9300a531c09dfa6a1be772b1abc1492ad", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.061203319502074686, "f1": 0.11534701857282502 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "original_inst_id": "django__django-15863", "base_commit": "37c5b8c07be104fd5288cd87f101e48cb7a40298", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.3717277486910995, "f1": 0.5419847328244275 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "original_inst_id": "ponylang__ponyc-2205", "base_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "status": "error", "trajectory": { "recall": 0.016800584368151936, "precision": 0.09090909090909091, "f1": 0.02836004932182491 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "original_inst_id": "jqlang__jq-2919", "base_commit": "7f547827e47b5ade563a293329deb4226496d72f", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "original_inst_id": "fmtlib__fmt-2394", "base_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "original_inst_id": "instance_future-architect__vuls-b8db2e0b74f60cb7d45f710f255e061f054b6afc", "base_commit": "43b46cb324f64076e4d9e807c0b60c4b9ce11a82", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "original_inst_id": "fasterxml__jackson-databind-4469", "base_commit": "7c9e7c1e2acd9c55926b9d2592fc65234d9c3ff7", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "original_inst_id": "darkreader__darkreader-7241", "base_commit": "991883df4d5910851130e3dc0e21fcbce604ea7d", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "original_inst_id": "nlohmann__json-3601", "base_commit": "2d48a4d9c5e8f0a5ce914922eb2a45dc0ec93ee3", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "original_inst_id": "catchorg__Catch2-1608", "base_commit": "979bbf03bb00bc55ca09783791b5091a2247df68", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "original_inst_id": "facebook__zstd-1733", "base_commit": "a505463710aa34bccafd268c44064c129cdfb3e2", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "original_inst_id": "instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan", "base_commit": "1e137b07052bc3ea0da44ed201702c94055b8ad2", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "original_inst_id": "clap-rs__clap-3420", "base_commit": "135b15467eda2394b017f2a7d25cda1417c0feec", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "original_inst_id": "tokio-rs__bytes-721", "base_commit": "9965a04b5684079bb614addd750340ffc165a9f5", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "original_inst_id": "serverless__serverless-3457", "base_commit": "fea269947104889165e3ebb0f64638921caf2052", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "original_inst_id": "nushell__nushell-13357", "base_commit": "5417c89387b67c3192ae9043473b556cd669ee15", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "original_inst_id": "google__gson-1391", "base_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "original_inst_id": "cli__cli-4253", "base_commit": "e6ff77ce73c201b0ee36d2b802ea45e9e1ad1822", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "original_inst_id": "alibaba__fastjson2-2775", "base_commit": "12b40c7ba3e7c30e35977195770c80beb34715c5", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "original_inst_id": "instance_flipt-io__flipt-c154dd1a3590954dfd3b901555fc6267f646a289", "base_commit": "e432032cf2d11e3bb6f558748a5ee41a90640daa", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "original_inst_id": "axios__axios-5661", "base_commit": "0abc70564746496eb211bbd951041b4655aec268", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "original_inst_id": "pydata__xarray-2905", "base_commit": "7c4e2ac83f7b4306296ff9b7b51aaf016e5ad614", "status": "not_run", "trajectory": { "recall": 0.0, "precision": 0.0, "f1": 0 } } ] }, "functions_graph": { "models": [ "claude-opus-5-5" ], "statuses": { "ok": 24 }, "scoring_failures": 0, "summary": { "n": 24, "arms": { "trajectory": { "recall": 0.5401686329564104, "precision": 0.2860920178829965, "f1": 0.3014815297089224, "official_valid": 24 }, "served_16384": { "recall": 0.5258478761889225, "precision": 0.2877248025695819, "f1": 0.3012754427265953, "official_valid": 24 }, "served_32768": { "recall": 0.5386683435889694, "precision": 0.2866860354099939, "f1": 0.3021406647114891, "official_valid": 24 } }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 6.375578, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 24 }, "cases": 24 } }, "implementation": { "scripts/benchmark_context_agent.py": "3a0b3c1604ed09532eabd1173349c60f9497019b83131023e85de752d13a922b", "scripts/benchmark_context_claude.py": "0ba94a80c5bf60c61f64ad3964d02cfb22682dfa8e13878e93b4d2fd0fb2595b", "scripts/contextbench_graph_navigation.py": "26ee7bc5e203b4c028f4b4acc5d019b106b04ab3c8365ec6a3fb42bc9b220bc9", "scripts/benchmark_context_serving.py": "87f93c9152007e923de6b94fad57d65dce805f6c47e22ede6347a8948141b9ec", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "2274a7b66faa89492a905eaeed66c046cdfeeb0482769b1bc97793119d633269", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "original_inst_id": "pylint-dev__pylint-4604", "base_commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "status": "ok", "trajectory": { "recall": 0.5684647302904564, "precision": 0.6116071428571429, "f1": 0.589247311827957 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "original_inst_id": "vuejs__core-10250", "base_commit": "d276a4f3e914aaccc291f7b2513e5d978919d0f9", "status": "ok", "trajectory": { "recall": 0.25510204081632654, "precision": 0.06297229219143577, "f1": 0.101010101010101 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "original_inst_id": "microsoft__vscode-135197", "base_commit": "a993ada9300a531c09dfa6a1be772b1abc1492ad", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.09104938271604938, "f1": 0.1669024045261669 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "original_inst_id": "django__django-15863", "base_commit": "37c5b8c07be104fd5288cd87f101e48cb7a40298", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.3641025641025641, "f1": 0.5338345864661653 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "original_inst_id": "ponylang__ponyc-2205", "base_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "status": "ok", "trajectory": { "recall": 0.4616508400292184, "precision": 0.36155606407322655, "f1": 0.4055181264035932 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "original_inst_id": "jqlang__jq-2919", "base_commit": "7f547827e47b5ade563a293329deb4226496d72f", "status": "ok", "trajectory": { "recall": 0.5460526315789473, "precision": 0.41089108910891087, "f1": 0.4689265536723164 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "original_inst_id": "fmtlib__fmt-2394", "base_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "status": "ok", "trajectory": { "recall": 0.5362694300518135, "precision": 0.552, "f1": 0.5440210249671484 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "original_inst_id": "instance_future-architect__vuls-b8db2e0b74f60cb7d45f710f255e061f054b6afc", "base_commit": "43b46cb324f64076e4d9e807c0b60c4b9ce11a82", "status": "ok", "trajectory": { "recall": 0.4875, "precision": 0.45792563600782776, "f1": 0.47225025227043393 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "original_inst_id": "fasterxml__jackson-databind-4469", "base_commit": "7c9e7c1e2acd9c55926b9d2592fc65234d9c3ff7", "status": "ok", "trajectory": { "recall": 0.19640971488912354, "precision": 0.3089700996677741, "f1": 0.24015493867010976 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "original_inst_id": "darkreader__darkreader-7241", "base_commit": "991883df4d5910851130e3dc0e21fcbce604ea7d", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.047619047619047616, "f1": 0.0909090909090909 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "original_inst_id": "nlohmann__json-3601", "base_commit": "2d48a4d9c5e8f0a5ce914922eb2a45dc0ec93ee3", "status": "ok", "trajectory": { "recall": 0.2495274102079395, "precision": 0.5076923076923077, "f1": 0.3346007604562738 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "original_inst_id": "catchorg__Catch2-1608", "base_commit": "979bbf03bb00bc55ca09783791b5091a2247df68", "status": "ok", "trajectory": { "recall": 0.44, "precision": 0.16356877323420074, "f1": 0.2384823848238482 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "original_inst_id": "facebook__zstd-1733", "base_commit": "a505463710aa34bccafd268c44064c129cdfb3e2", "status": "ok", "trajectory": { "recall": 0.17716535433070865, "precision": 0.14551333872271624, "f1": 0.15978695073235685 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "original_inst_id": "instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan", "base_commit": "1e137b07052bc3ea0da44ed201702c94055b8ad2", "status": "ok", "trajectory": { "recall": 0.36316695352839934, "precision": 0.18672566371681415, "f1": 0.24663939216832265 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "original_inst_id": "clap-rs__clap-3420", "base_commit": "135b15467eda2394b017f2a7d25cda1417c0feec", "status": "ok", "trajectory": { "recall": 0.5645645645645646, "precision": 0.1551155115511551, "f1": 0.24336569579288025 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "original_inst_id": "tokio-rs__bytes-721", "base_commit": "9965a04b5684079bb614addd750340ffc165a9f5", "status": "ok", "trajectory": { "recall": 0.6521739130434783, "precision": 0.02697841726618705, "f1": 0.05181347150259066 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "original_inst_id": "serverless__serverless-3457", "base_commit": "fea269947104889165e3ebb0f64638921caf2052", "status": "ok", "trajectory": { "recall": 0.8773234200743495, "precision": 0.36932707355242567, "f1": 0.5198237885462554 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "original_inst_id": "nushell__nushell-13357", "base_commit": "5417c89387b67c3192ae9043473b556cd669ee15", "status": "ok", "trajectory": { "recall": 0.7248677248677249, "precision": 0.1282371294851794, "f1": 0.21792152704135737 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "original_inst_id": "google__gson-1391", "base_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "status": "ok", "trajectory": { "recall": 0.44876325088339225, "precision": 0.8639455782312925, "f1": 0.5906976744186048 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "original_inst_id": "cli__cli-4253", "base_commit": "e6ff77ce73c201b0ee36d2b802ea45e9e1ad1822", "status": "ok", "trajectory": { "recall": 1.0, "precision": 0.036458333333333336, "f1": 0.07035175879396986 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "original_inst_id": "alibaba__fastjson2-2775", "base_commit": "12b40c7ba3e7c30e35977195770c80beb34715c5", "status": "ok", "trajectory": { "recall": 0.375, "precision": 0.05647058823529412, "f1": 0.098159509202454 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "original_inst_id": "instance_flipt-io__flipt-c154dd1a3590954dfd3b901555fc6267f646a289", "base_commit": "e432032cf2d11e3bb6f558748a5ee41a90640daa", "status": "ok", "trajectory": { "recall": 0.22832369942196531, "precision": 0.11318051575931232, "f1": 0.15134099616858238 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "original_inst_id": "axios__axios-5661", "base_commit": "0abc70564746496eb211bbd951041b4655aec268", "status": "ok", "trajectory": { "recall": 0.2270220588235294, "precision": 0.5678160919540229, "f1": 0.3243598161523309 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "original_inst_id": "pydata__xarray-2905", "base_commit": "7c4e2ac83f7b4306296ff9b7b51aaf016e5ad614", "status": "ok", "trajectory": { "recall": 0.5846994535519126, "precision": 0.27648578811369506, "f1": 0.375438596491228 } } ] } }, "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" }, "comparisons": { "primary": { "contrast": "functions_and_directed_graph_over_native", "metrics": { "recall": { "baseline": 0.5335256710568391, "candidate": 0.5401686329564104, "difference": 0.006642961899571264, "paired_bootstrap_95_interval": [ -0.041123996794128176, 0.05591541359811768 ] }, "precision": { "baseline": 0.31624050372742957, "candidate": 0.2860920178829965, "difference": -0.0301484858444331, "paired_bootstrap_95_interval": [ -0.07305829744775158, 0.0037484294601860123 ] }, "f1": { "baseline": 0.31176405157915105, "candidate": 0.3014815297089224, "difference": -0.01028252187022866, "paired_bootstrap_95_interval": [ -0.037074972214231956, 0.015619874227425445 ] } }, "complete": true, "interpretation": "Complete selected primary pair; development only." }, "functions": { "contrast": "online_function_completion", "metrics": { "recall": { "baseline": 0.5335256710568391, "candidate": 0.12064738088464838, "difference": -0.41287829017219074, "paired_bootstrap_95_interval": [ -0.5417121350676875, -0.2842740156987571 ] }, "precision": { "baseline": 0.31624050372742957, "candidate": 0.042582924380788985, "difference": -0.2736575793466406, "paired_bootstrap_95_interval": [ -0.37993615038723794, -0.1779350318663182 ] }, "f1": { "baseline": 0.31176405157915105, "candidate": 0.054296528980621366, "difference": -0.25746752259852973, "paired_bootstrap_95_interval": [ -0.3415253883735656, -0.1766083451891632 ] } }, "complete": false, "interpretation": "Incomplete function-only arm; not interpretable as a policy or marginal graph quality effect." }, "graph": { "contrast": "directed_graph_over_function_completion", "metrics": { "recall": { "baseline": 0.12064738088464838, "candidate": 0.5401686329564104, "difference": 0.41952125207176205, "paired_bootstrap_95_interval": [ 0.3017910482408834, 0.5373840928628781 ] }, "precision": { "baseline": 0.042582924380788985, "candidate": 0.2860920178829965, "difference": 0.24350909350220748, "paired_bootstrap_95_interval": [ 0.16170675403291523, 0.3345622637759854 ] }, "f1": { "baseline": 0.054296528980621366, "candidate": 0.3014815297089224, "difference": 0.24718500072830105, "paired_bootstrap_95_interval": [ 0.17845776748032188, 0.31914253399299386 ] } }, "complete": false, "interpretation": "Incomplete function-only arm; not interpretable as a policy or marginal graph quality effect." } }, "development_gate": { "protocol_sha256": "3def16e46468b5a98c9e4901fe8f68a768b87857cc21c4aef28f237b441edc96", "analysis_sha256": "ced6aed2de953016861328ec3be240d91cb69694328ed0c7166185675d13f950", "gates": { "all_selected_tasks_finish_and_score": false, "every_available_graph_called": true, "at_least_two_languages_receive_previously_unseen_annotated_graph_source": true, "primary_recall_point_increased": true, "primary_f1_point_not_lower": false }, "passed": false, "graph_adoption": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_unseen_neighbors": 1 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "graph_available": true, "new_physical_lines": 52, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 52, "source_returned": 5 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "graph_available": true, "new_physical_lines": 132, "new_annotated_lines": 132, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 132, "source_returned": 1 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "graph_available": true, "new_physical_lines": 15, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 15, "source_returned": 3 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_definition_at_line": 1 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "graph_available": true, "new_physical_lines": 16, "new_annotated_lines": 2, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 16, "source_returned": 5 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "graph_available": true, "new_physical_lines": 12, "new_annotated_lines": 5, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 12, "source_returned": 2 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_returned_neighbors": 1 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_unseen_neighbors": 1 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "graph_available": true, "new_physical_lines": 131, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 131, "source_returned": 1 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_definition_at_line": 1 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 } } ], "primary_metrics": { "recall": { "baseline": 0.5335256710568391, "candidate": 0.5401686329564104, "difference": 0.006642961899571264, "paired_bootstrap_95_interval": [ -0.041123996794128176, 0.05591541359811768 ] }, "precision": { "baseline": 0.31624050372742957, "candidate": 0.2860920178829965, "difference": -0.0301484858444331, "paired_bootstrap_95_interval": [ -0.07305829744775158, 0.0037484294601860123 ] }, "f1": { "baseline": 0.31176405157915105, "candidate": 0.3014815297089224, "difference": -0.01028252187022866, "paired_bootstrap_95_interval": [ -0.037074972214231956, 0.015619874227425445 ] } }, "scope": "Development selection gate only; no independent quality or leaderboard claim." }, "graph_delivery_totals": { "calls": 24, "available_tasks": 11, "source_tasks": 6, "confirmed_new_physical_lines": 358, "confirmed_new_annotated_physical_lines": 139, "annotated_languages": [ "go", "rust" ] }, "interrupted_function_arm": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "reason": "The model attempted an unregistered graph tool. The normalizer aborted before the SDK terminal result, losing usage and stopping later tasks.", "official_valid": 5, "interpretation": "Four completed traces and the partial failed trace have valid official observations; 19 unstarted cases remain zero. All 24 positions remain in the denominator. No retry replaces an original result.", "metering_recovered": false }, "receipt_fix": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "scope": "One post-fix reliability check on the development case that triggered an unavailable graph-tool attempt. Excluded from quality comparisons; original failed arm is unchanged.", "implementation_commit": "25e5e31e4ed53f1bcb64d40ac60e1acc93d5b787", "protocol_sha256": "591a527766d678384ea65de9b2019823aafd3781fd10dae753d987bebe8dc294", "gates": { "terminal_result_and_meter": true, "isolated_inventory": true, "no_unavailable_graph_instructions": true, "frozen_runtime": true }, "passed": true, "status": "ok", "sdk_reported_cost_usd": 0.42661920000000003, "unavailable_tool_attempts": [], "actual_models": [ "claude-opus-5-5" ], "original_stream_replay": { "unavailable_attempts_retained": [ "mcp__source__graph_neighbors" ], "terminal_result_available": false, "metering_recovered": false, "new_model_calls": 0 }, "actual_bill_usd": null, "quality_score": null }, "known_retrieval_sdk_estimate_usd": 12.8310857, "conclusion": "Graph use and novel annotated delivery now meet their gates, but F1 decreases and one ablation is interrupted. The candidate fails the declared development gate; do not advance it to new independent validation or official 500.", "offline_graph_preparation_probe": { "protocol": { "scope": "Offline development graph preparation diagnosis. No model calls or quality score; no replacement of frozen 24-task graph artifacts.", "selection": "First C++ task in the previously disclosed 24-task selection, chosen from metadata only.", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "original_inst_id": "fmtlib__fmt-2394", "base_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "implementation_commit": "25e5e31e4ed53f1bcb64d40ac60e1acc93d5b787", "driver_sha256": "3ad8c400b84fc213af393878de0a65921849480d9e0efd3df1e8c7d3b9ed1bc9", "timeout_per_arm_seconds": 300, "arms": { "readonly": false, "prepared": true }, "invariant": "Each arm indexes its own shared clone. Reuse bounded_graph owner cleanup and SIGTERM unwinding; reap the worker before deleting its workspace. Original checkout remains source-audited and unmodified.", "gate": "Enabling project preparation creates a compiler graph where the matched read-only control does not; source snapshot remains clean. This is availability evidence only.", "model_calls": 0 }, "analysis": { "protocol_sha256": "31e404c790fda42b2e64f54242b3cadb3b1615009442823e5621ec4c42765c10", "results": { "readonly": { "available": false, "language": "cpp", "returncode": 0, "failed_languages": { "cpp": "indexer returned no graph" }, "index_generation_reports": {}, "build_wall_s": 4.163067555055022, "sha256": "a738a254184e9c06e3f1aabc584ac3b4e2672331f2dc3042b0517a24dc75d6a2" }, "prepared": { "available": true, "language": "cpp", "returncode": 0, "failed_languages": {}, "index_generation_reports": {}, "nodes": 8247, "edges": 39953, "build_wall_s": 18.178339838050306, "sha256": "4241fb24d892c2368f2e9315ef4683ec1feab439346099a3fdd9ecb492cab4b9" } }, "source_snapshot_unchanged": true, "passed": true, "model_calls": 0 }, "interpretation": "One metadata-selected development C++ availability diagnostic. No model calls, quality score or changes to previous frozen artifacts." }, "final_source_set_diagnostic": { "scope": "Post-hoc physical-line diagnostic on all 24 disclosed development tasks. Model calls zero. No new quality score, independent validation or causal attribution to graph/function completion; the function-only arm was interrupted.", "source_manifest_sha256": "3574d68ad8832273efc58cae36db0b2361ec80a764e1714a16e5a89f5daaa43f", "driver_sha256": "1b7b0060577c12fad35e3a01553b69b5919ab2b98a117144ad975e79ea57f81b", "totals": { "native_final_physical": 16307, "candidate_final_physical": 17254, "added_physical": 5151, "lost_physical": 4204, "added_annotated": 472, "lost_annotated": 464, "graph_new_at_delivery": 358, "graph_new_annotated_at_delivery": 139, "graph_new_annotated_also_read_by_native": 90, "graph_new_annotated_not_read_by_native": 49 }, "by_language": { "c": { "native_final_physical": 3470, "candidate_final_physical": 3591, "added_physical": 991, "lost_physical": 870, "added_annotated": 170, "lost_annotated": 211, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, "cpp": { "native_final_physical": 950, "candidate_final_physical": 904, "added_physical": 149, "lost_physical": 195, "added_annotated": 20, "lost_annotated": 51, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, "go": { "native_final_physical": 1307, "candidate_final_physical": 1593, "added_physical": 385, "lost_physical": 99, "added_annotated": 50, "lost_annotated": 36, "graph_new_at_delivery": 263, "graph_new_annotated_at_delivery": 132, "graph_new_annotated_also_read_by_native": 83, "graph_new_annotated_not_read_by_native": 49 }, "java": { "native_final_physical": 1196, "candidate_final_physical": 1321, "added_physical": 450, "lost_physical": 325, "added_annotated": 121, "lost_annotated": 63, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, "javascript": { "native_final_physical": 1990, "candidate_final_physical": 2204, "added_physical": 561, "lost_physical": 347, "added_annotated": 8, "lost_annotated": 18, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, "python": { "native_final_physical": 616, "candidate_final_physical": 806, "added_physical": 428, "lost_physical": 238, "added_annotated": 65, "lost_annotated": 17, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, "rust": { "native_final_physical": 4617, "candidate_final_physical": 4973, "added_physical": 1510, "lost_physical": 1154, "added_annotated": 38, "lost_annotated": 38, "graph_new_at_delivery": 28, "graph_new_annotated_at_delivery": 7, "graph_new_annotated_also_read_by_native": 7, "graph_new_annotated_not_read_by_native": 0 }, "typescript": { "native_final_physical": 2161, "candidate_final_physical": 1862, "added_physical": 677, "lost_physical": 976, "added_annotated": 0, "lost_annotated": 30, "graph_new_at_delivery": 67, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "native_final_physical": 125, "candidate_final_physical": 224, "added_physical": 143, "lost_physical": 44, "added_annotated": 57, "lost_annotated": 8, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "native_final_physical": 737, "candidate_final_physical": 794, "added_physical": 350, "lost_physical": 293, "added_annotated": 0, "lost_annotated": 30, "graph_new_at_delivery": 52, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "native_final_physical": 914, "candidate_final_physical": 648, "added_physical": 211, "lost_physical": 477, "added_annotated": 0, "lost_annotated": 0, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "native_final_physical": 224, "candidate_final_physical": 195, "added_physical": 49, "lost_physical": 78, "added_annotated": 0, "lost_annotated": 0, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "native_final_physical": 1621, "candidate_final_physical": 1748, "added_physical": 586, "lost_physical": 459, "added_annotated": 159, "lost_annotated": 24, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "native_final_physical": 756, "candidate_final_physical": 606, "added_physical": 27, "lost_physical": 177, "added_annotated": 11, "lost_annotated": 159, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "native_final_physical": 411, "candidate_final_physical": 375, "added_physical": 7, "lost_physical": 43, "added_annotated": 0, "lost_annotated": 0, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "native_final_physical": 245, "candidate_final_physical": 511, "added_physical": 297, "lost_physical": 31, "added_annotated": 50, "lost_annotated": 20, "graph_new_at_delivery": 132, "graph_new_annotated_at_delivery": 132, "graph_new_annotated_also_read_by_native": 83, "graph_new_annotated_not_read_by_native": 49 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "native_final_physical": 625, "candidate_final_physical": 602, "added_physical": 167, "lost_physical": 190, "added_annotated": 73, "lost_annotated": 28, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "native_final_physical": 510, "candidate_final_physical": 420, "added_physical": 116, "lost_physical": 206, "added_annotated": 0, "lost_annotated": 0, "graph_new_at_delivery": 15, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "native_final_physical": 323, "candidate_final_physical": 260, "added_physical": 34, "lost_physical": 97, "added_annotated": 20, "lost_annotated": 45, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "native_final_physical": 216, "candidate_final_physical": 269, "added_physical": 108, "lost_physical": 55, "added_annotated": 0, "lost_annotated": 6, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "native_final_physical": 1093, "candidate_final_physical": 1237, "added_physical": 378, "lost_physical": 234, "added_annotated": 0, "lost_annotated": 28, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "native_final_physical": 988, "candidate_final_physical": 1130, "added_physical": 304, "lost_physical": 162, "added_annotated": 7, "lost_annotated": 1, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "native_final_physical": 1174, "candidate_final_physical": 1212, "added_physical": 252, "lost_physical": 214, "added_annotated": 5, "lost_annotated": 10, "graph_new_at_delivery": 16, "graph_new_annotated_at_delivery": 2, "graph_new_annotated_also_read_by_native": 2, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "native_final_physical": 417, "candidate_final_physical": 556, "added_physical": 243, "lost_physical": 104, "added_annotated": 2, "lost_annotated": 0, "graph_new_at_delivery": 12, "graph_new_annotated_at_delivery": 5, "graph_new_annotated_also_read_by_native": 5, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "native_final_physical": 656, "candidate_final_physical": 639, "added_physical": 147, "lost_physical": 164, "added_annotated": 1, "lost_annotated": 7, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "native_final_physical": 3026, "candidate_final_physical": 3205, "added_physical": 1015, "lost_physical": 836, "added_annotated": 31, "lost_annotated": 28, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "native_final_physical": 320, "candidate_final_physical": 294, "added_physical": 25, "lost_physical": 51, "added_annotated": 24, "lost_annotated": 35, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "native_final_physical": 341, "candidate_final_physical": 384, "added_physical": 44, "lost_physical": 1, "added_annotated": 0, "lost_annotated": 0, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "native_final_physical": 251, "candidate_final_physical": 425, "added_physical": 258, "lost_physical": 84, "added_annotated": 24, "lost_annotated": 0, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "native_final_physical": 721, "candidate_final_physical": 698, "added_physical": 44, "lost_physical": 67, "added_annotated": 0, "lost_annotated": 16, "graph_new_at_delivery": 131, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "native_final_physical": 346, "candidate_final_physical": 435, "added_physical": 110, "lost_physical": 21, "added_annotated": 0, "lost_annotated": 10, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "native_final_physical": 267, "candidate_final_physical": 387, "added_physical": 236, "lost_physical": 116, "added_annotated": 8, "lost_annotated": 9, "graph_new_at_delivery": 0, "graph_new_annotated_at_delivery": 0, "graph_new_annotated_also_read_by_native": 0, "graph_new_annotated_not_read_by_native": 0 } ], "model_calls": 0, "annotation_correction_protocol_sha256": "12abf234264f904a4b54d4df6615fae1d2b9b6210eed671fdb926644ec96bf43" } }, "fresh_validation_pool": { "scope": "Historical metadata-only inventory before the current independent32 selection; no annotation inspection or inference at inventory time. See post_neighbors_validation_untouched_pool for current remaining counts.", "dataset_revision": "c2855792b006af41c67202d33883fb9d46362853", "dataset_sha256": "e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791", "columns_read": [ "instance_id", "language" ], "total": 500, "excluded_development": 105, "excluded_completed_validation": 32, "remaining": 363, "remaining_per_language": { "c": 11, "cpp": 0, "go": 26, "java": 4, "javascript": 44, "python": 220, "rust": 9, "typescript": 49 }, "consequence": "A second independent four-per-language 32-task selection is impossible on this pinned release: all ten C++ tasks are already development or completed validation. A revised candidate needs an explicitly changed sampling protocol or new untouched tasks; do not recycle the prior 32." }, "multi_swe_native_reference_readiness": { "protocol": { "scope": "One development C++ native-harness reference/negative-control check. No solver, model calls, retrieval quality or agent Pass@1 claim.", "instance_id": "fmtlib__fmt-2394", "contextbench_instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "harness_revision": "24f493f8a103e72312ded4f6b9c89f081d69cb09", "harness_version": "1.1.0", "dataset_receipt_sha256": "f5b14f5321113536331c778d7b145c877de9c2f0de31dd32cabaa0783c0f661d", "native_case_sha256": "e85e7d612570c50723c691753abfef9455eec60c20d81c3f321db430c5d232f4", "image_id": "sha256:801d11f190c6669e3b2a4a1273c5f5e237f26acbecfa379300c37c2b2acaf8e8", "image_repo_digests": [ "mswebench/fmtlib_m_fmt@sha256:a277e173e1671c25bd1fb412e45665be5e8b6c2f96f30887fa3dbc12e1dac949" ], "source_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "source_tree": "c793e859f77f2ef96bcc8f1b1804f06ebce087aa", "driver_sha256": "41290e41373f09c3875dc1e8112a7f888f5f6c17a92547b00646cdb15d9ce544", "stages": { "test_only": "/home/test-run.sh", "reference": "/home/fix-run.sh" }, "bounds": { "native_seconds": 600, "grading_seconds": 60, "memory": "3g", "cpus": 2, "pids": 256, "network": "none", "host_mounts": [] }, "gate": "Original native grader rejects the test-only control and accepts the reference patch, with exact clean source and patch identity. Both owned containers are removed.", "solver_image_requirement": "This official grading image contains reference/test patches. Any future solver needs a separate source-only image with those artifacts removed before inference.", "model_calls": 0 }, "analysis": { "protocol_sha256": "a8cec334c7e02bac2d31476c7e2e7cf924f6e069d3c222d512afc6e7c1d4acbc", "passed": true, "results": { "test_only": { "stage": "test_only", "native_script": "/home/test-run.sh", "model_calls": 0, "source": { "commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "tree": "c793e859f77f2ef96bcc8f1b1804f06ebce087aa", "tracked_clean": true }, "native_patch_sha256": { "fix_patch": "f9a464860e99bab76108c5cd5bced442645c82800bde08d89020761dbaeefd8e", "test_patch": "76862918b5520d200adec1db9a1751b6ee40d9ccf041eaa358d9b4ee6d1ab19b" }, "native_exit_code": 2, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "fmtlib/fmt:pr-2394" ], "empty_patch_ids": [], "error_ids": [] }, "native_report_sha256": "913b3112d4d851ebbe6050a958c432cf6b2957ebec820c0038c2d678cc0e5c7d", "native_log_sha256": "ed4fb115e538bba8525e5967cd62e7ae483663e2fdeab3de881acd82a0d2d1c8", "status": "complete", "container_removed": true, "elapsed_seconds": 50.719128431985155 }, "reference": { "stage": "reference", "native_script": "/home/fix-run.sh", "model_calls": 0, "source": { "commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "tree": "c793e859f77f2ef96bcc8f1b1804f06ebce087aa", "tracked_clean": true }, "native_patch_sha256": { "fix_patch": "f9a464860e99bab76108c5cd5bced442645c82800bde08d89020761dbaeefd8e", "test_patch": "76862918b5520d200adec1db9a1751b6ee40d9ccf041eaa358d9b4ee6d1ab19b" }, "native_exit_code": 0, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [ "fmtlib/fmt:pr-2394" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "native_report_sha256": "8e32d5c7ce399ea86801c9bda0e40b833aa6191a37ad34997bd752b21b5e464c", "native_log_sha256": "4b3221a05456b4a88a5ba34575fa80f82d3cbfe42e7b4872889994d1744c46a5", "status": "complete", "container_removed": true, "elapsed_seconds": 213.48750283499248 } }, "model_calls": 0, "agent_quality_score": null }, "pinned_harness_code": { "harness_module_sha256": { "harness/gen_report.py": "02ebc8a5414898d12f4f5a9ba0c11a8f57c9f34a0bdc02c2311afac9f654847d", "harness/report.py": "5a025fd496d42c4b7377fc0702d64c6d0e356b117eaf2face47e73a52c29902f", "harness/test_result.py": "5411af794920cf4b170fe9dbe8c21c12cc63e2bbe2280d6d82acb850f4808be3", "harness/dataset.py": "dd49f55baf63b60fff309b6a5b2a1826697e2b85ad1a9bccff18321dcdc200fc", "harness/repos/cpp/fmtlib/fmt.py": "d0a5146eb77b4416cccb86216c3df08f200d58c33bb158d953aaac385f922918" }, "code_matches_pinned_checkout": true }, "solver_image": { "base_image_id": "sha256:801d11f190c6669e3b2a4a1273c5f5e237f26acbecfa379300c37c2b2acaf8e8", "solver_image_id": "sha256:0c36ed672e54cd9a7f09b03d60a29d3731fd34e5250227f9154306aaf81013bf", "size_bytes": 1509842023, "dockerfile_sha256": "1958604c95f669695cd49c4499786fa3e75b0b28c5748b6ff6138f7c9a524e89", "checks": { "source_tree": "c793e859f77f2ef96bcc8f1b1804f06ebce087aa", "single_commit_count": 1, "source_commit": "91454c5532df6c481eb9cc96c108ce56d26fe86a", "status": "", "reference_artifacts_absent": true, "owned_container_removed": true }, "model_calls": 0, "status": "ready", "scope": "Source-only C++ image preparation. Solver adapter and agent inference are still pending. Native grading uses the separate original pinned image." }, "native_dataset": { "repo": "ByteDance-Seed/Multi-SWE-bench", "revision": "56ff018c04a38e27ada1e9d0a6d5839a51f88f0d", "files": [ "cpp/fmtlib__fmt_dataset.jsonl" ] }, "native_dataset_record_sha256": "e85e7d612570c50723c691753abfef9455eec60c20d81c3f321db430c5d232f4", "reference_passed_tests": 18, "reusable_multi_solver_adapter_implemented": false, "agent_inference_completed": false, "agent_pass_at_1": null, "conclusion": "Original native grader rejects the test-only control and accepts the reference patch. The exact clean source-only image is ready. This is one development harness-readiness check with no model calls; it establishes no CodeNib solving result or comparative quality gain.", "harness_repository": "https://github.com/multi-swe-bench/multi-swe-bench", "native_dataset_source_url": "https://huggingface.co/datasets/ByteDance-Seed/Multi-SWE-bench/resolve/56ff018c04a38e27ada1e9d0a6d5839a51f88f0d/cpp/fmtlib__fmt_dataset.jsonl" }, "offline_graph_preparation_24": { "scope": "All 24 previously disclosed development tasks; offline graph-availability experiment only. No model inference, reference patches or quality scores. Original retrieval artifacts remain unchanged.", "protocol": { "scope": "All 24 previously disclosed development tasks; offline graph-availability experiment only. No model inference, reference patches or quality scores. Original retrieval artifacts remain unchanged.", "selection": "Retain the complete disclosed 24-task cohort without score-based filtering, including all prior unavailable graphs.", "selection_sha256": "122a70d4c61abcea5c27f87781fc204db768d933dd0d3bb7516e20117866f513", "source_manifest_sha256": "3574d68ad8832273efc58cae36db0b2361ec80a764e1714a16e5a89f5daaa43f", "prior_graph_manifest_sha256": "1d8de298ffb44da0aa511ce439b4dab60ac85d0d3517cbcefdede2dffcfdec11", "implementation_commit": "64f0746217ebb621e75b32d6e615036c434930d4", "driver_sha256": "290976395a8512be053cb36c18d311d698df5f0b5c240fe046a5ddaee1900ba0", "timeout_per_case_seconds": 300, "allow_project_preparation": true, "n": 24, "workers": 1, "model_calls": 0, "interpretation": "Availability diagnosis only: both project-preparation policy and the previous 120-second timeout change. Only the separate matched fmt probe isolates the preparation switch. Do not attribute all coverage changes to preparation or claim Recall/F1 improvements." }, "analysis": { "protocol_sha256": "a38b3ed776c509cb3fc63b4cc04c9f33f330d9f3035b5ca4026ad37c2cdf1445", "graph_manifest_sha256": "f701d2cac13757112ed0017c2312eccc3aba12bda1a352a642913b4c1502e7f7", "prior_available": 11, "available": 14, "newly_available": 8, "lost_available": 5, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "previous_available": true, "available": true, "build_wall_s": 45.573048056918196, "failed_languages": {}, "error": null, "snapshot": { "commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "clean": true }, "graph_sha256": "fec8971a7ba35eac4d4876a324da8a91e74903b5c85263d526666aecf23856ec", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "previous_available": true, "available": false, "build_wall_s": 301.3752404219704, "failed_languages": {}, "error": "graph build exceeded 300.0s", "snapshot": { "commit": "d276a4f3e914aaccc291f7b2513e5d978919d0f9", "clean": true }, "graph_sha256": "74ff08c6b3e2095d5e905530add5fee6c27ba8454d9865c898037782530732d5", "failure_class": "timeout" }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "previous_available": false, "available": false, "build_wall_s": 302.6058837809833, "failed_languages": {}, "error": "graph build exceeded 300.0s", "snapshot": { "commit": "a993ada9300a531c09dfa6a1be772b1abc1492ad", "clean": true }, "graph_sha256": "f09ffdbf69fe7cb84635ab1a9ede74017c266572da7e397038e3a9098cb829dd", "failure_class": "timeout" }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "previous_available": false, "available": true, "build_wall_s": 234.85654022498056, "failed_languages": {}, "error": null, "snapshot": { "commit": "37c5b8c07be104fd5288cd87f101e48cb7a40298", "clean": true }, "graph_sha256": "9623277777325e827a679e3a11a97ee53721025a3ec13ebdb931ca58faf964f6", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "previous_available": false, "available": true, "build_wall_s": 60.943939688033424, "failed_languages": {}, "error": null, "snapshot": { "commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "clean": true }, "graph_sha256": "a4b21dd7c1e6a930a65d470959ffcfca8cfe0f8a7b6d5c454328c08d38efa1d9", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "previous_available": false, "available": false, "build_wall_s": 55.759866998996586, "failed_languages": {}, "error": "graph worker failed; see build.log", "snapshot": { "commit": "7f547827e47b5ade563a293329deb4226496d72f", "clean": true }, "graph_sha256": "ffa9767c73f97bd0b9ce0ed582c6b0bf7f68521d2a6c05d6991651b216156e04", "failure_class": "tracked_source_changed_rejected" }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "previous_available": false, "available": true, "build_wall_s": 20.677948256023228, "failed_languages": {}, "error": null, "snapshot": { "commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "clean": true }, "graph_sha256": "05c348c39fd3c3ddbf612c2ccaa5835d9f41a1d42577f9ac138e98d58a249733", "failure_class": null }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "previous_available": true, "available": false, "build_wall_s": 4.4483089469140396, "failed_languages": { "go": "indexer returned no graph" }, "error": null, "snapshot": { "commit": "43b46cb324f64076e4d9e807c0b60c4b9ce11a82", "clean": true }, "graph_sha256": "d7df0e940204e13b3ebc2aab39a5c2660690be6f1518cb85ca6591381589a898", "failure_class": "managed_indexer_missing_from_path" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "previous_available": false, "available": false, "build_wall_s": 4.394553430960514, "failed_languages": { "java": "indexer returned no graph" }, "error": null, "snapshot": { "commit": "7c9e7c1e2acd9c55926b9d2592fc65234d9c3ff7", "clean": true }, "graph_sha256": "4001aa22ffb139eb492ad2e223e9ec8ce8defc8a1274cb8c2524c1694a6196d4", "failure_class": "managed_indexer_missing_from_path" }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "previous_available": true, "available": true, "build_wall_s": 29.822332373005338, "failed_languages": {}, "error": null, "snapshot": { "commit": "991883df4d5910851130e3dc0e21fcbce604ea7d", "clean": true }, "graph_sha256": "bd33ff635dfacf9d5e7e8edb5aa42b6585b02a9bb84672e036ddf70ed0be3fac", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "previous_available": false, "available": true, "build_wall_s": 29.903728391043842, "failed_languages": {}, "error": null, "snapshot": { "commit": "2d48a4d9c5e8f0a5ce914922eb2a45dc0ec93ee3", "clean": true }, "graph_sha256": "2bdb420b0e707c3739794ae43116cc6f2c8e4d2cddb7cd142d13c45ee58256b2", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "previous_available": false, "available": true, "build_wall_s": 33.226809118059464, "failed_languages": {}, "error": null, "snapshot": { "commit": "979bbf03bb00bc55ca09783791b5091a2247df68", "clean": true }, "graph_sha256": "957489e881e094d24ee7c6498edf0bfc6b33a5b577f6804aed14e7d42d1d2667", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "previous_available": false, "available": true, "build_wall_s": 35.61954666196834, "failed_languages": {}, "error": null, "snapshot": { "commit": "a505463710aa34bccafd268c44064c129cdfb3e2", "clean": true }, "graph_sha256": "7833a61bef3b9bb3f448925addc8c6d8206d9364f8f45fd657d959ccd7afa759", "failure_class": null }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "previous_available": true, "available": true, "build_wall_s": 39.521483383956365, "failed_languages": {}, "error": null, "snapshot": { "commit": "1e137b07052bc3ea0da44ed201702c94055b8ad2", "clean": true }, "graph_sha256": "39af7e0357ed6b47df97e62dc83a332a1aedbff27ebba7fb7b35fa2c104f07c5", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "previous_available": true, "available": true, "build_wall_s": 44.31255821697414, "failed_languages": {}, "error": null, "snapshot": { "commit": "135b15467eda2394b017f2a7d25cda1417c0feec", "clean": true }, "graph_sha256": "e53b4286d5c1fb757ae44227ce28cc327f050514ba814c8b44684819049ee8d2", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "previous_available": true, "available": true, "build_wall_s": 10.441258113016374, "failed_languages": {}, "error": null, "snapshot": { "commit": "9965a04b5684079bb614addd750340ffc165a9f5", "clean": true }, "graph_sha256": "3fcb1b7a23cff7a377aaa15325507f08cf325e5d774ff606ff7683c7d1098dcf", "failure_class": null }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "previous_available": true, "available": false, "build_wall_s": 16.865822591935284, "failed_languages": {}, "error": "graph worker failed; see build.log", "snapshot": { "commit": "fea269947104889165e3ebb0f64638921caf2052", "clean": true }, "graph_sha256": "bc6a946ca177613e7fd5e160a50aece4a001d32d94df0d079d51e87a4b73eb89", "failure_class": "tracked_source_changed_rejected" }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "previous_available": false, "available": true, "build_wall_s": 127.07240782503504, "failed_languages": {}, "error": null, "snapshot": { "commit": "5417c89387b67c3192ae9043473b556cd669ee15", "clean": true }, "graph_sha256": "0531fe9fef890c163b5feaf49858dbad68f64fd890baa79b988eb86c5e5708d6", "failure_class": null }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "previous_available": false, "available": false, "build_wall_s": 4.3256130299996585, "failed_languages": { "java": "indexer returned no graph" }, "error": null, "snapshot": { "commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "clean": true }, "graph_sha256": "195df6871e84b065705bf3fab337736efff11b2a557b2201a7b04fb8d8bbf03c", "failure_class": "managed_indexer_missing_from_path" }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "previous_available": true, "available": false, "build_wall_s": 3.6200944010633975, "failed_languages": { "go": "indexer returned no graph" }, "error": null, "snapshot": { "commit": "e6ff77ce73c201b0ee36d2b802ea45e9e1ad1822", "clean": true }, "graph_sha256": "7e12343fe35f46243d1d2600cdbd7934c802125a6f0e3180506254e049200745", "failure_class": "managed_indexer_missing_from_path" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "previous_available": false, "available": false, "build_wall_s": 5.659347126958892, "failed_languages": { "java": "indexer returned no graph" }, "error": null, "snapshot": { "commit": "12b40c7ba3e7c30e35977195770c80beb34715c5", "clean": true }, "graph_sha256": "37167376b1c4f65c7924677db91789d0f3c49821a5fdcc0734e5d0d739e9e324", "failure_class": "managed_indexer_missing_from_path" }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "previous_available": true, "available": false, "build_wall_s": 3.647785610985011, "failed_languages": { "go": "indexer returned no graph" }, "error": null, "snapshot": { "commit": "e432032cf2d11e3bb6f558748a5ee41a90640daa", "clean": true }, "graph_sha256": "a865e56b0d0ec41ebc1e6fe610e3c58e6a25c10de00a6fa7ba8e0310fb97b1e5", "failure_class": "managed_indexer_missing_from_path" }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "previous_available": true, "available": true, "build_wall_s": 28.608394160983153, "failed_languages": {}, "error": null, "snapshot": { "commit": "0abc70564746496eb211bbd951041b4655aec268", "clean": true }, "graph_sha256": "1c8a14177d4dde192bb8fb227f2e074ec5fa30f69f35f209a5b0ed8b5634abc2", "failure_class": null }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "previous_available": false, "available": true, "build_wall_s": 166.47758592502214, "failed_languages": {}, "error": null, "snapshot": { "commit": "7c4e2ac83f7b4306296ff9b7b51aaf016e5ad614", "clean": true }, "graph_sha256": "9ac61e4e09330aace5e11690d50ecb20582bac5ba8fbfe30238aa0e8cdf39004", "failure_class": null } ], "all_frozen_runtime_hashes_match": true, "model_calls": 0, "quality_score": null }, "limitations": "Preparation policy, timeout and available PATH differ from prior builds. The six Go/Java failures reflect missing managed commands on this invocation PATH; the tools are installed. Two changed-source builds are rejected, and two TypeScript builds time out. These are retained preparation outcomes, not retrieval scores or a policy-only comparison.", "source_replay": { "protocol": { "scope": "Offline source-delivery diagnostic on the frozen native 24-task development traces. No model calls, autonomous adoption, counterfactual quality score or independent validation.", "anchor": "Use the prepared graph if available, otherwise the prior graph. Choose the first actual read response intersecting a compiler callable, then the callable with most returned physical lines, smallest span and lexical name; query its first returned line. Use the same anchor and prefix for both graph artifacts and both directions.", "selection_uses_annotations": false, "annotation_check": "Measure overlap only after selecting the anchor and retrieving every response for that task.", "limits": { "results": 5, "tokens": 2048, "raw_edges": 1000 }, "native_config_sha256": "9a71dfb5dc293cfc96f4be92a67cea47c5589bfdf335afcce9e432d13df09ea6", "graph_manifest_sha256": { "prior": "1d8de298ffb44da0aa511ce439b4dab60ac85d0d3517cbcefdede2dffcfdec11", "prepared": "f701d2cac13757112ed0017c2312eccc3aba12bda1a352a642913b4c1502e7f7" }, "driver_sha256": "b811ee484d1bf99fb9bf7c2fa887ffdd73477973e011b1ffc22a5959e3f30ab4", "model_calls": 0 }, "protocol_sha256": "2e9dd19d52891c1d410960dd0d8e66836cec8d0f0416b68b051a0023b05b8da5", "totals": { "prior": { "new_annotated_physical_lines": 177, "new_physical_lines": 687 }, "prepared": { "new_annotated_physical_lines": 44, "new_physical_lines": 890 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "available": { "prior": true, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 31, "statuses": { "source_returned": 2 } }, "outgoing": { "new_physical_lines": 1, "statuses": { "source_returned": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 31, "statuses": { "source_returned": 2 } }, "outgoing": { "new_physical_lines": 1, "statuses": { "source_returned": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 32, "new_annotated_physical_lines": 21 }, "prepared": { "new_physical_lines": 32, "new_annotated_physical_lines": 21 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "available": { "prior": true, "prepared": false }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } }, "outgoing": { "new_physical_lines": 14, "statuses": { "source_returned": 5 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 14, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "available": { "prior": false, "prepared": false }, "directions": {}, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 90, "statuses": { "source_returned": 4, "source_not_returned": 1 } }, "outgoing": { "new_physical_lines": 37, "statuses": { "source_returned": 2 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 127, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "no_unseen_neighbors": 1 } }, "outgoing": { "new_physical_lines": 26, "statuses": { "source_returned": 5 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 26, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "available": { "prior": false, "prepared": false }, "directions": {}, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "source_not_returned": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "source_not_returned": 3 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "available": { "prior": true, "prepared": false }, "directions": { "prior": { "incoming": { "new_physical_lines": 132, "statuses": { "source_returned": 1, "source_not_returned": 1 } }, "outgoing": { "new_physical_lines": 16, "statuses": { "source_returned": 5 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 148, "new_annotated_physical_lines": 145 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "available": { "prior": false, "prepared": false }, "directions": {}, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "available": { "prior": true, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 36, "statuses": { "source_returned": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 36, "statuses": { "source_returned": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 36, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 36, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 1, "statuses": { "source_returned": 1 } }, "outgoing": { "new_physical_lines": 14, "statuses": { "source_returned": 3 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 15, "new_annotated_physical_lines": 1 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } }, "outgoing": { "new_physical_lines": 3, "statuses": { "source_returned": 3 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 3, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 99, "statuses": { "source_returned": 2, "source_not_returned": 2 } }, "outgoing": { "new_physical_lines": 48, "statuses": { "source_returned": 5 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 147, "new_annotated_physical_lines": 0 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "available": { "prior": true, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } }, "outgoing": { "new_physical_lines": 38, "statuses": { "source_returned": 3 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } }, "outgoing": { "new_physical_lines": 38, "statuses": { "source_returned": 3 } } } }, "union_directions": { "prior": { "new_physical_lines": 38, "new_annotated_physical_lines": 9 }, "prepared": { "new_physical_lines": 38, "new_annotated_physical_lines": 9 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "available": { "prior": true, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "no_unseen_neighbors": 1 } }, "outgoing": { "new_physical_lines": 46, "statuses": { "source_returned": 5 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "no_unseen_neighbors": 1 } }, "outgoing": { "new_physical_lines": 46, "statuses": { "source_returned": 5 } } } }, "union_directions": { "prior": { "new_physical_lines": 46, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 46, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "available": { "prior": true, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 90, "statuses": { "source_not_returned": 1, "source_returned": 4 } }, "outgoing": { "new_physical_lines": 14, "statuses": { "source_returned": 2 } } }, "prepared": { "incoming": { "new_physical_lines": 90, "statuses": { "source_not_returned": 1, "source_returned": 4 } }, "outgoing": { "new_physical_lines": 14, "statuses": { "source_returned": 2 } } } }, "union_directions": { "prior": { "new_physical_lines": 104, "new_annotated_physical_lines": 2 }, "prepared": { "new_physical_lines": 104, "new_annotated_physical_lines": 2 } } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "available": { "prior": true, "prepared": false }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "no_unseen_neighbors": 1 } }, "outgoing": { "new_physical_lines": 102, "statuses": { "source_returned": 5 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 102, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "available": { "prior": false, "prepared": false }, "directions": {}, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "available": { "prior": true, "prepared": false }, "directions": { "prior": { "incoming": { "new_physical_lines": 83, "statuses": { "source_returned": 1 } }, "outgoing": { "new_physical_lines": 55, "statuses": { "source_returned": 5 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 138, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "available": { "prior": false, "prepared": false }, "directions": {}, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "available": { "prior": true, "prepared": false }, "directions": { "prior": { "incoming": { "new_physical_lines": 131, "statuses": { "source_returned": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "no_unseen_neighbors": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 131, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "available": { "prior": true, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "no_definition_at_line": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "no_definition_at_line": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 0, "statuses": { "no_returned_neighbors": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "no_unseen_neighbors": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "available": { "prior": false, "prepared": true }, "directions": { "prior": { "incoming": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } }, "outgoing": { "new_physical_lines": 0, "statuses": { "graph_unavailable": 1 } } }, "prepared": { "incoming": { "new_physical_lines": 158, "statuses": { "source_returned": 2, "source_not_returned": 1 } }, "outgoing": { "new_physical_lines": 56, "statuses": { "source_returned": 4, "source_not_returned": 1 } } } }, "union_directions": { "prior": { "new_physical_lines": 0, "new_annotated_physical_lines": 0 }, "prepared": { "new_physical_lines": 214, "new_annotated_physical_lines": 11 } } } ], "interpretation": "Canonical annotation overlap changes 177 to 44, including lost Go coverage. This fixed-prefix, zero-model probe does not measure agent adoption or quality.", "annotation_correction_protocol_sha256": "12abf234264f904a4b54d4df6615fae1d2b9b6210eed671fdb926644ec96bf43" }, "prospective_composition": { "protocol": { "scope": "New graph input for future development only. No change to earlier retrieval or graph-build results; no model calls or new compiler builds.", "selection": "Retain all 24 tasks. Prefer a newly prepared available graph; otherwise reuse the prior authenticated graph for the identical selected source. If neither is available, retain the fresh failure. No annotations, scores or model outputs select artifacts.", "input_manifest_sha256": { "fresh": "f701d2cac13757112ed0017c2312eccc3aba12bda1a352a642913b4c1502e7f7", "fallback": "1d8de298ffb44da0aa511ce439b4dab60ac85d0d3517cbcefdede2dffcfdec11" }, "source_manifest_sha256": "3574d68ad8832273efc58cae36db0b2361ec80a764e1714a16e5a89f5daaa43f", "builder_sha256": "d20238cdaca8febdf7707c351b4a9c026078db9b17c33ce9ac2809d7b3c232b4", "model_calls": 0, "new_compiler_builds": 0 }, "manifest_sha256": "21310839449890abf2bf4e43a4623b58fd8bb42c93a158096a0db2f1d9ccac82", "selected": 24, "available": 19, "fresh_available": 14, "fallback_available": 5, "unavailable": 5, "all_source_snapshots_audited": true, "all_copied_bytes_authenticated": true, "model_calls": 0, "new_compiler_builds": 0, "interpretation": "A separate future-development input retains all 24 tasks, using 14 fresh graphs and five authenticated same-source prior graphs. Earlier 11/24 and 14/24 build records and all retrieval outcomes remain unchanged. It has not been used in an online quality run." } }, "multi_swe_native_adapter": { "scope": "Source-checkout fmt C++ solver/native adapter. Native reference/control checks are separate from agent quality; no model calls in these checks. Other Multi repositories, Poly/Pro and complete solver source-line attribution remain open.", "implementation_commit": "fe4cbb7243dc4776131c52192b2a4064b6c79e37", "implementation": { "scripts/benchmark_context_solver.py": "1d8f9173546102c8aa2d8239a2f6b1502dd3605da5306f07a85f38c64389b34c", "scripts/contextbench_multiswe.py": "16effa29d3eb7ed3dd3b3b4ae01ab536a16c9997dde2ade80c8f0fdceb5c9ea6" }, "supported_repo": "fmtlib/fmt", "source_only_preflight": { "one_source_commit": true, "reference_artifacts_absent": true, "exact_clean_source_tree": true, "container_removed": true }, "attempts": [ { "attempt": "initial", "protocol_sha256": "827c815ca56c7072c102946ff03565f5108bc9bddff684674002bc3c5bfbd93e", "passed": false, "container_start_timeout_seconds": 60, "results": { "unchanged_issue": { "status": "graded", "resolved": false, "container_removed": true, "wall_s": 129.246481811977, "native_exit_code": 2, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "fmtlib/fmt:pr-2394" ], "empty_patch_ids": [], "error_ids": [] }, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "f9a464860e99bab76108c5cd5bced442645c82800bde08d89020761dbaeefd8e", "test_patch": "76862918b5520d200adec1db9a1751b6ee40d9ccf041eaa358d9b4ee6d1ab19b" }, "applied_solver_patch_sha256": "90445ffd8a33d0547a4b23a2a968d4378252718828a57959256a02ea60e49c87", "native_log_sha256": "ed4fb115e538bba8525e5967cd62e7ae483663e2fdeab3de881acd82a0d2d1c8" }, "reference": { "status": "error", "resolved": false, "container_removed": true, "wall_s": 60.089845404960215 } }, "failures": { "reference": { "kind": "container_start_timeout", "before_native_test": true } } }, { "attempt": "reference_retry", "protocol_sha256": "92245bf76680f65f50a9d4b4a333d2faa88ad7c68ed78d7eab0bf039f390727f", "passed": false, "container_start_timeout_seconds": 60, "results": { "reference": { "status": "error", "resolved": false, "container_removed": true, "wall_s": 60.084325133939274 } }, "failures": { "reference": { "kind": "container_start_timeout", "before_native_test": true } } }, { "attempt": "startup_180", "protocol_sha256": "eb5b60b5f5572294e67aa529ad24d31ac034936dfd0744111ed4af56f2edd32d", "passed": true, "container_start_timeout_seconds": 180, "results": { "unchanged_issue": { "status": "graded", "resolved": false, "container_removed": true, "wall_s": 115.94812042196281, "native_exit_code": 2, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "fmtlib/fmt:pr-2394" ], "empty_patch_ids": [], "error_ids": [] }, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "f9a464860e99bab76108c5cd5bced442645c82800bde08d89020761dbaeefd8e", "test_patch": "76862918b5520d200adec1db9a1751b6ee40d9ccf041eaa358d9b4ee6d1ab19b" }, "applied_solver_patch_sha256": "90445ffd8a33d0547a4b23a2a968d4378252718828a57959256a02ea60e49c87", "native_log_sha256": "ed4fb115e538bba8525e5967cd62e7ae483663e2fdeab3de881acd82a0d2d1c8" }, "reference": { "status": "graded", "resolved": true, "container_removed": true, "wall_s": 330.4129523159936, "native_exit_code": 0, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [ "fmtlib/fmt:pr-2394" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "f9a464860e99bab76108c5cd5bced442645c82800bde08d89020761dbaeefd8e", "test_patch": "76862918b5520d200adec1db9a1751b6ee40d9ccf041eaa358d9b4ee6d1ab19b" }, "applied_solver_patch_sha256": "f9a464860e99bab76108c5cd5bced442645c82800bde08d89020761dbaeefd8e", "native_log_sha256": "13d2058e88543cade7703b36d55a345ec69663b7cd927cef00b95b18548a99cc" } }, "failures": {} } ], "latest_native_gate_passed": true, "same_runtime_through_native_checks": true, "solver_inference_started": true, "agent_quality_score": null, "model_calls": 0, "validation": { "related_unit_tests_passed": 150, "pinned_integration_tests_passed": 4, "public_docs_boundary_tests_passed": 16 } }, "multi_fmt_development_solver_pair": { "scope": "One disclosed C++ development task, two fresh downstream solver arms. No independent validation, new retrieval, graph-policy quality gate or official submission claim.", "protocol_sha256": "f01ff70f0fe4e3057fc7ec09622e15bfad65ca4105cbe4f3162e12d29309d4de", "implementation_commit": "fe4cbb7243dc4776131c52192b2a4064b6c79e37", "all_frozen_runtime_and_input_hashes_match": true, "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "original_inst_id": "fmtlib__fmt-2394", "unique_tasks": 1, "language": "cpp", "source": "Multi", "arms": { "native": { "status": "Submitted", "model_calls": 5, "actual_models": [ "claude-opus-5-5" ], "known_sdk_reported_estimate_usd": 0.7349067, "missing_metering_calls": 0, "all_model_inputs_confirmed": true, "confirmed_model_inputs": 5, "sdk_api_key_sources": [ "none" ], "verified_subscription_route": true, "solver_container_removed": true, "native_status": "graded", "resolved": true, "native_container_removed": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [ "fmtlib/fmt:pr-2394" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "model_patch_sha256": "4834e3b6e1f54eda978188083742f3d2eefedd4619ad529026ee306573b423e1", "solver_receipt_sha256": "5b7c37b0c9342c0173fbced35e95f093ecb7b94137b7e596039ae3f53854e239", "native_receipt_sha256": "bbe54b3d84f317284d49c2c691f48e0283b73bd959003293f64b0f58287eb3bd" }, "functions_graph": { "status": "Submitted", "model_calls": 5, "actual_models": [ "claude-opus-5-5" ], "known_sdk_reported_estimate_usd": 0.5868949, "missing_metering_calls": 0, "all_model_inputs_confirmed": true, "confirmed_model_inputs": 5, "sdk_api_key_sources": [ "none" ], "verified_subscription_route": true, "solver_container_removed": true, "native_status": "graded", "resolved": true, "native_container_removed": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "fmtlib/fmt:pr-2394" ], "completed_ids": [ "fmtlib/fmt:pr-2394" ], "incomplete_ids": [], "resolved_ids": [ "fmtlib/fmt:pr-2394" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "model_patch_sha256": "b0785bd4536e2d3551bf858f62724f270f3ff99fdf8618093e343b7d3fd5643c", "solver_receipt_sha256": "da039ed8683526d511e17df561000e94345da4231030841b5457f3eb5b950f40", "native_receipt_sha256": "e7d06816bb819295231b96bb812bb82252bb764b59a595f70d0762802dd08e01" } }, "context_budget": 16384, "max_turns": 20, "max_sdk_meter_usd_per_arm": 2, "source_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "source_tree": "c793e859f77f2ef96bcc8f1b1804f06ebce087aa", "source_only_image_id": "sha256:0c36ed672e54cd9a7f09b03d60a29d3731fd34e5250227f9154306aaf81013bf", "native_image_id": "sha256:801d11f190c6669e3b2a4a1273c5f5e237f26acbecfa379300c37c2b2acaf8e8", "native_adapter_check_sha256": "5e98aea466ff4bd877cde323b7bb8b558fb0c3c4fc147b24c2fc83d03c167e97", "retrieval_receipt_sha256": { "native": "4f0e3b55bdd7aef3f3765618bb7fd86293a63bc782ee089a7fb202c0d068ca49", "functions_graph": "24e64f1f0d843c16510479542ddcc12c0be88609dfe010f9603c4197d0e85974" }, "graph_available_during_original_retrieval": false, "retrieval_recomputed": false, "independent_validation": false, "known_sdk_reported_estimate_usd": 1.3218016000000001, "same_actual_model": true, "conclusion": "One unique development task. Retain both arms and any infrastructure or budget failure. It is not independent Pass@1 evidence, a retrieval-quality gate or evidence of graph-source gains. The first-source 24 and independent 32 quality failures remain unchanged." }, "completion_source_followup": { "scope": "Source-checkout completion efficiency and fixed-query graph diagnostics on all 24 disclosed development tasks. Not new model results, independent validation, an official quality score or a product promotion.", "implementation_commit": "c2e11085734f9601eef0140813b60f0e96940378", "completion_source_policy": "Explicit unseen option, preserving historical callable selection and budget reservations before removing duplicate lines. Historical definitions remains the default.", "existing_source_attribution": { "scope": "Post-hoc source attribution on all 24 disclosed development tasks. Zero model calls. Fixed existing tool calls cannot establish online policy quality or causal gains.", "protocol_sha256": "5d95b7cf50a64ecb1bae060b2b70a670e1d08b6eb02bfecf9664b42552daff15", "all_frozen_runtime_and_input_hashes_match": true, "n": 24, "categories": { "completion_grep": { "first_exposure_annotated": 894, "first_exposure_physical": 4502, "repeated_line_deliveries": 310, "replies": 73, "unique_annotated_absent_from_native": 148, "unique_physical": 4713, "unique_physical_absent_from_native": 2353 }, "completion_read": { "first_exposure_annotated": 219, "first_exposure_physical": 1348, "repeated_line_deliveries": 416, "replies": 44, "unique_annotated_absent_from_native": 169, "unique_physical": 1639, "unique_physical_absent_from_native": 1172 }, "graph": { "first_exposure_annotated": 139, "first_exposure_physical": 358, "repeated_line_deliveries": 0, "replies": 24, "unique_annotated_absent_from_native": 49, "unique_physical": 358, "unique_physical_absent_from_native": 115 }, "grep": { "first_exposure_annotated": 204, "first_exposure_physical": 1096, "repeated_line_deliveries": 41, "replies": 115, "unique_annotated_absent_from_native": 17, "unique_physical": 1130, "unique_physical_absent_from_native": 408 }, "read": { "first_exposure_annotated": 2531, "first_exposure_physical": 9950, "repeated_line_deliveries": 829, "replies": 109, "unique_annotated_absent_from_native": 141, "unique_physical": 10779, "unique_physical_absent_from_native": 1170 } }, "fixed_trace_removal": { "without_grep_completion": { "lost_annotated": 635, "lost_physical": 3858 }, "without_any_completion": { "lost_annotated": 957, "lost_physical": 5464 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "categories": { "grep": { "replies": 2, "unique_physical": 26, "first_exposure_physical": 26, "first_exposure_annotated": 25, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 1, "unique_annotated_absent_from_native": 1 }, "completion_grep": { "replies": 1, "unique_physical": 56, "first_exposure_physical": 56, "first_exposure_annotated": 43, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 40, "unique_annotated_absent_from_native": 27 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 3, "unique_physical": 143, "first_exposure_physical": 138, "first_exposure_annotated": 65, "repeated_line_deliveries": 5, "unique_physical_absent_from_native": 103, "unique_annotated_absent_from_native": 30 }, "completion_read": { "replies": 1, "unique_physical": 4, "first_exposure_physical": 4, "first_exposure_annotated": 4, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 4248, "read": 226 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 56, "lost_annotated": 43 }, "without_any_completion": { "lost_physical": 60, "lost_annotated": 47 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "categories": { "grep": { "replies": 8, "unique_physical": 109, "first_exposure_physical": 109, "first_exposure_annotated": 0, "repeated_line_deliveries": 1, "unique_physical_absent_from_native": 57, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 5, "unique_physical": 234, "first_exposure_physical": 230, "first_exposure_annotated": 0, "repeated_line_deliveries": 87, "unique_physical_absent_from_native": 97, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 52, "first_exposure_physical": 52, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 12, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 6, "unique_physical": 385, "first_exposure_physical": 302, "first_exposure_annotated": 50, "repeated_line_deliveries": 83, "unique_physical_absent_from_native": 90, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 1, "unique_physical": 101, "first_exposure_physical": 101, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 96, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 25555, "read": 3317 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 201, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 302, "lost_annotated": 0 } } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "categories": { "grep": { "replies": 5, "unique_physical": 59, "first_exposure_physical": 52, "first_exposure_annotated": 1, "repeated_line_deliveries": 7, "unique_physical_absent_from_native": 26, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 2, "unique_physical": 161, "first_exposure_physical": 152, "first_exposure_annotated": 2, "repeated_line_deliveries": 9, "unique_physical_absent_from_native": 105, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 4, "unique_physical": 425, "first_exposure_physical": 380, "first_exposure_annotated": 56, "repeated_line_deliveries": 45, "unique_physical_absent_from_native": 30, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 2, "unique_physical": 64, "first_exposure_physical": 64, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 57, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 15819, "read": 3273 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 117, "lost_annotated": 2 }, "without_any_completion": { "lost_physical": 180, "lost_annotated": 2 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "categories": { "grep": { "replies": 2, "unique_physical": 5, "first_exposure_physical": 5, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 2, "unique_physical": 160, "first_exposure_physical": 160, "first_exposure_annotated": 71, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 49, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 1, "unique_physical": 30, "first_exposure_physical": 30, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 13287 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 160, "lost_annotated": 71 }, "without_any_completion": { "lost_physical": 160, "lost_annotated": 71 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "categories": { "grep": { "replies": 11, "unique_physical": 111, "first_exposure_physical": 108, "first_exposure_annotated": 14, "repeated_line_deliveries": 4, "unique_physical_absent_from_native": 64, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 8, "unique_physical": 598, "first_exposure_physical": 450, "first_exposure_annotated": 40, "repeated_line_deliveries": 148, "unique_physical_absent_from_native": 217, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 7, "unique_physical": 1000, "first_exposure_physical": 972, "first_exposure_annotated": 404, "repeated_line_deliveries": 28, "unique_physical_absent_from_native": 150, "unique_annotated_absent_from_native": 40 }, "completion_read": { "replies": 3, "unique_physical": 220, "first_exposure_physical": 218, "first_exposure_annotated": 174, "repeated_line_deliveries": 2, "unique_physical_absent_from_native": 155, "unique_annotated_absent_from_native": 119 } }, "completion_characters": { "grep": 35791, "read": 6410 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 438, "lost_annotated": 40 }, "without_any_completion": { "lost_physical": 656, "lost_annotated": 214 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "categories": { "grep": { "replies": 4, "unique_physical": 17, "first_exposure_physical": 16, "first_exposure_annotated": 9, "repeated_line_deliveries": 1, "unique_physical_absent_from_native": 1, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 4, "unique_physical": 511, "first_exposure_physical": 502, "first_exposure_annotated": 152, "repeated_line_deliveries": 9, "unique_physical_absent_from_native": 15, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 2, "unique_physical": 88, "first_exposure_physical": 88, "first_exposure_annotated": 88, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 11, "unique_annotated_absent_from_native": 11 } }, "completion_characters": { "grep": 5813 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 88, "lost_annotated": 88 }, "without_any_completion": { "lost_physical": 88, "lost_annotated": 88 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "categories": { "grep": { "replies": 3, "unique_physical": 40, "first_exposure_physical": 40, "first_exposure_annotated": 11, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 1, "unique_physical": 320, "first_exposure_physical": 300, "first_exposure_annotated": 196, "repeated_line_deliveries": 20, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 1, "unique_physical": 35, "first_exposure_physical": 35, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 7, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 2522 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 35, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 35, "lost_annotated": 0 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "categories": { "grep": { "replies": 4, "unique_physical": 26, "first_exposure_physical": 26, "first_exposure_annotated": 9, "repeated_line_deliveries": 1, "unique_physical_absent_from_native": 14, "unique_annotated_absent_from_native": 1 }, "completion_grep": { "replies": 3, "unique_physical": 309, "first_exposure_physical": 309, "first_exposure_annotated": 65, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 227, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 132, "first_exposure_physical": 132, "first_exposure_annotated": 132, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 49, "unique_annotated_absent_from_native": 49 }, "read": { "replies": 2, "unique_physical": 153, "first_exposure_physical": 38, "first_exposure_annotated": 27, "repeated_line_deliveries": 115, "unique_physical_absent_from_native": 35, "unique_annotated_absent_from_native": 30 }, "completion_read": { "replies": 2, "unique_physical": 25, "first_exposure_physical": 6, "first_exposure_annotated": 1, "repeated_line_deliveries": 19, "unique_physical_absent_from_native": 21, "unique_annotated_absent_from_native": 19 } }, "completion_characters": { "grep": 15404, "read": 910 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 309, "lost_annotated": 65 }, "without_any_completion": { "lost_physical": 315, "lost_annotated": 66 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "categories": { "grep": { "replies": 6, "unique_physical": 38, "first_exposure_physical": 38, "first_exposure_annotated": 16, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 4, "unique_physical": 231, "first_exposure_physical": 231, "first_exposure_annotated": 95, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 117, "unique_annotated_absent_from_native": 50 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 5, "unique_physical": 296, "first_exposure_physical": 281, "first_exposure_annotated": 50, "repeated_line_deliveries": 15, "unique_physical_absent_from_native": 10, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 3, "unique_physical": 65, "first_exposure_physical": 52, "first_exposure_annotated": 25, "repeated_line_deliveries": 13, "unique_physical_absent_from_native": 40, "unique_annotated_absent_from_native": 23 } }, "completion_characters": { "grep": 28801, "read": 3365 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 211, "lost_annotated": 75 }, "without_any_completion": { "lost_physical": 273, "lost_annotated": 110 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "categories": { "grep": { "replies": 3, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 2, "unique_physical": 400, "first_exposure_physical": 400, "first_exposure_annotated": 20, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 99, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 1, "unique_physical": 5, "first_exposure_physical": 5, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 5, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 15, "first_exposure_physical": 15, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 12, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "read": 143 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 0, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 5, "lost_annotated": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "categories": { "grep": { "replies": 4, "unique_physical": 26, "first_exposure_physical": 26, "first_exposure_annotated": 7, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 16, "unique_annotated_absent_from_native": 2 }, "completion_grep": { "replies": 1, "unique_physical": 8, "first_exposure_physical": 8, "first_exposure_annotated": 8, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 4, "unique_physical": 241, "first_exposure_physical": 226, "first_exposure_annotated": 117, "repeated_line_deliveries": 15, "unique_physical_absent_from_native": 20, "unique_annotated_absent_from_native": 20 } }, "completion_characters": { "grep": 682 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 0, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 0, "lost_annotated": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "categories": { "grep": { "replies": 4, "unique_physical": 76, "first_exposure_physical": 73, "first_exposure_annotated": 1, "repeated_line_deliveries": 4, "unique_physical_absent_from_native": 52, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 2, "unique_physical": 27, "first_exposure_physical": 27, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 27, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 4, "unique_physical": 177, "first_exposure_physical": 169, "first_exposure_annotated": 43, "repeated_line_deliveries": 8, "unique_physical_absent_from_native": 30, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 2401 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 27, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 27, "lost_annotated": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "categories": { "grep": { "replies": 6, "unique_physical": 122, "first_exposure_physical": 117, "first_exposure_annotated": 9, "repeated_line_deliveries": 6, "unique_physical_absent_from_native": 11, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 5, "unique_physical": 305, "first_exposure_physical": 290, "first_exposure_annotated": 1, "repeated_line_deliveries": 15, "unique_physical_absent_from_native": 169, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 15, "unique_physical": 707, "first_exposure_physical": 655, "first_exposure_annotated": 170, "repeated_line_deliveries": 52, "unique_physical_absent_from_native": 30, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 4, "unique_physical": 175, "first_exposure_physical": 175, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 170, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 23673, "read": 7847 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 289, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 464, "lost_annotated": 0 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "categories": { "grep": { "replies": 7, "unique_physical": 72, "first_exposure_physical": 72, "first_exposure_annotated": 17, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 20, "unique_annotated_absent_from_native": 1 }, "completion_grep": { "replies": 3, "unique_physical": 263, "first_exposure_physical": 263, "first_exposure_annotated": 63, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 151, "unique_annotated_absent_from_native": 6 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 7, "unique_physical": 830, "first_exposure_physical": 757, "first_exposure_annotated": 127, "repeated_line_deliveries": 73, "unique_physical_absent_from_native": 101, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 3, "unique_physical": 60, "first_exposure_physical": 38, "first_exposure_annotated": 4, "repeated_line_deliveries": 22, "unique_physical_absent_from_native": 33, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 15403, "read": 2379 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 206, "lost_annotated": 6 }, "without_any_completion": { "lost_physical": 265, "lost_annotated": 31 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "categories": { "grep": { "replies": 9, "unique_physical": 30, "first_exposure_physical": 25, "first_exposure_annotated": 6, "repeated_line_deliveries": 5, "unique_physical_absent_from_native": 2, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 8, "unique_physical": 442, "first_exposure_physical": 407, "first_exposure_annotated": 85, "repeated_line_deliveries": 51, "unique_physical_absent_from_native": 111, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 16, "first_exposure_physical": 16, "first_exposure_annotated": 2, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 10, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 6, "unique_physical": 789, "first_exposure_physical": 693, "first_exposure_annotated": 90, "repeated_line_deliveries": 96, "unique_physical_absent_from_native": 70, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 4, "unique_physical": 95, "first_exposure_physical": 71, "first_exposure_annotated": 5, "repeated_line_deliveries": 24, "unique_physical_absent_from_native": 59, "unique_annotated_absent_from_native": 5 } }, "completion_characters": { "grep": 29178, "read": 4663 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 304, "lost_annotated": 68 }, "without_any_completion": { "lost_physical": 398, "lost_annotated": 73 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "categories": { "grep": { "replies": 4, "unique_physical": 50, "first_exposure_physical": 50, "first_exposure_annotated": 3, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 6, "unique_annotated_absent_from_native": 2 }, "completion_grep": { "replies": 3, "unique_physical": 314, "first_exposure_physical": 314, "first_exposure_annotated": 2, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 221, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 12, "first_exposure_physical": 12, "first_exposure_annotated": 5, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 5, "unique_physical": 255, "first_exposure_physical": 166, "first_exposure_annotated": 5, "repeated_line_deliveries": 89, "unique_physical_absent_from_native": 11, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 3, "unique_physical": 14, "first_exposure_physical": 14, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 5, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 17172, "read": 782 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 251, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 265, "lost_annotated": 0 } } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "categories": { "grep": { "replies": 4, "unique_physical": 84, "first_exposure_physical": 84, "first_exposure_annotated": 14, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 51, "unique_annotated_absent_from_native": 1 }, "completion_grep": { "replies": 1, "unique_physical": 35, "first_exposure_physical": 35, "first_exposure_annotated": 35, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 2, "unique_physical": 527, "first_exposure_physical": 514, "first_exposure_annotated": 184, "repeated_line_deliveries": 13, "unique_physical_absent_from_native": 96, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 1, "unique_physical": 6, "first_exposure_physical": 6, "first_exposure_annotated": 3, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 3279, "read": 178 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 35, "lost_annotated": 35 }, "without_any_completion": { "lost_physical": 41, "lost_annotated": 38 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "categories": { "grep": { "replies": 7, "unique_physical": 106, "first_exposure_physical": 101, "first_exposure_annotated": 22, "repeated_line_deliveries": 5, "unique_physical_absent_from_native": 42, "unique_annotated_absent_from_native": 8 }, "completion_grep": { "replies": 6, "unique_physical": 509, "first_exposure_physical": 509, "first_exposure_annotated": 53, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 357, "unique_annotated_absent_from_native": 23 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 13, "unique_physical": 2188, "first_exposure_physical": 2150, "first_exposure_annotated": 336, "repeated_line_deliveries": 38, "unique_physical_absent_from_native": 189, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 6, "unique_physical": 493, "first_exposure_physical": 445, "first_exposure_annotated": 0, "repeated_line_deliveries": 48, "unique_physical_absent_from_native": 428, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 37954, "read": 21374 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 446, "lost_annotated": 23 }, "without_any_completion": { "lost_physical": 938, "lost_annotated": 41 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "categories": { "grep": { "replies": 3, "unique_physical": 29, "first_exposure_physical": 28, "first_exposure_annotated": 26, "repeated_line_deliveries": 1, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 3, "unique_physical": 129, "first_exposure_physical": 129, "first_exposure_annotated": 91, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 1, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 2, "unique_physical": 204, "first_exposure_physical": 134, "first_exposure_annotated": 134, "repeated_line_deliveries": 70, "unique_physical_absent_from_native": 21, "unique_annotated_absent_from_native": 21 }, "completion_read": { "replies": 1, "unique_physical": 11, "first_exposure_physical": 3, "first_exposure_annotated": 3, "repeated_line_deliveries": 8, "unique_physical_absent_from_native": 3, "unique_annotated_absent_from_native": 3 } }, "completion_characters": { "grep": 14162, "read": 428 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 60, "lost_annotated": 22 }, "without_any_completion": { "lost_physical": 71, "lost_annotated": 33 } } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "categories": { "grep": { "replies": 2, "unique_physical": 13, "first_exposure_physical": 13, "first_exposure_annotated": 1, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 2, "unique_physical": 100, "first_exposure_physical": 100, "first_exposure_annotated": 12, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 3, "unique_physical": 214, "first_exposure_physical": 207, "first_exposure_annotated": 1, "repeated_line_deliveries": 7, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 2, "unique_physical": 76, "first_exposure_physical": 64, "first_exposure_annotated": 0, "repeated_line_deliveries": 12, "unique_physical_absent_from_native": 44, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 5630, "read": 2929 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 88, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 163, "lost_annotated": 11 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "categories": { "grep": { "replies": 6, "unique_physical": 21, "first_exposure_physical": 21, "first_exposure_annotated": 1, "repeated_line_deliveries": 1, "unique_physical_absent_from_native": 15, "unique_annotated_absent_from_native": 1 }, "completion_grep": { "replies": 5, "unique_physical": 226, "first_exposure_physical": 226, "first_exposure_annotated": 23, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 185, "unique_annotated_absent_from_native": 23 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 2, "unique_physical": 180, "first_exposure_physical": 152, "first_exposure_annotated": 0, "repeated_line_deliveries": 28, "unique_physical_absent_from_native": 40, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 2, "unique_physical": 40, "first_exposure_physical": 26, "first_exposure_annotated": 0, "repeated_line_deliveries": 14, "unique_physical_absent_from_native": 19, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 24144, "read": 2131 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 188, "lost_annotated": 23 }, "without_any_completion": { "lost_physical": 228, "lost_annotated": 23 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "categories": { "grep": { "replies": 5, "unique_physical": 15, "first_exposure_physical": 11, "first_exposure_annotated": 4, "repeated_line_deliveries": 4, "unique_physical_absent_from_native": 2, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 1, "unique_physical": 127, "first_exposure_physical": 127, "first_exposure_annotated": 36, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 131, "first_exposure_physical": 131, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 32, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 6, "unique_physical": 414, "first_exposure_physical": 410, "first_exposure_annotated": 39, "repeated_line_deliveries": 4, "unique_physical_absent_from_native": 10, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 4, "unique_physical": 148, "first_exposure_physical": 19, "first_exposure_annotated": 0, "repeated_line_deliveries": 254, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 5587, "read": 8457 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 0, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 144, "lost_annotated": 35 } } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "categories": { "grep": { "replies": 2, "unique_physical": 33, "first_exposure_physical": 33, "first_exposure_annotated": 3, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 22, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 1, "unique_physical": 95, "first_exposure_physical": 95, "first_exposure_annotated": 7, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 88, "unique_annotated_absent_from_native": 0 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 3, "unique_physical": 320, "first_exposure_physical": 307, "first_exposure_annotated": 237, "repeated_line_deliveries": 13, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 5782 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 88, "lost_annotated": 0 }, "without_any_completion": { "lost_physical": 88, "lost_annotated": 0 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "categories": { "grep": { "replies": 4, "unique_physical": 22, "first_exposure_physical": 22, "first_exposure_annotated": 5, "repeated_line_deliveries": 1, "unique_physical_absent_from_native": 6, "unique_annotated_absent_from_native": 0 }, "completion_grep": { "replies": 4, "unique_physical": 261, "first_exposure_physical": 261, "first_exposure_annotated": 74, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 173, "unique_annotated_absent_from_native": 8 }, "graph": { "replies": 1, "unique_physical": 0, "first_exposure_physical": 0, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 0, "unique_annotated_absent_from_native": 0 }, "read": { "replies": 2, "unique_physical": 70, "first_exposure_physical": 67, "first_exposure_annotated": 28, "repeated_line_deliveries": 3, "unique_physical_absent_from_native": 20, "unique_annotated_absent_from_native": 0 }, "completion_read": { "replies": 1, "unique_physical": 37, "first_exposure_physical": 37, "first_exposure_annotated": 0, "repeated_line_deliveries": 0, "unique_physical_absent_from_native": 37, "unique_annotated_absent_from_native": 0 } }, "completion_characters": { "grep": 18747, "read": 2166 }, "fixed_trace_removal": { "without_grep_completion": { "lost_physical": 261, "lost_annotated": 74 }, "without_any_completion": { "lost_physical": 298, "lost_annotated": 74 } } } ], "model_calls": 0, "quality_score": null, "annotation_correction_protocol_sha256": "12abf234264f904a4b54d4df6615fae1d2b9b6210eed671fdb926644ec96bf43" }, "prepared_graph_fixed_query_replay": { "scope": "Fixed-query graph replay only; annotations checked after retrieving source. Zero model calls, no quality score.", "protocol_sha256": "06fd04a7697dcee51828a58e672a8b26e05e603f59656c408859805a17f67710", "n": 24, "all_frozen_runtime_and_input_hashes_match": true, "historical_graph_responses_reproduced_exactly": true, "totals": { "prior": { "available": 11, "queries": 24, "new_physical_at_delivery": 358, "new_annotated_at_delivery": 139, "new_physical_absent_from_native": 115, "new_annotated_absent_from_native": 49 }, "composed": { "available": 19, "queries": 24, "new_physical_at_delivery": 932, "new_annotated_at_delivery": 356, "new_physical_absent_from_native": 556, "new_annotated_absent_from_native": 194 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "no_unseen_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "no_unseen_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "source_returned": 5 }, "new_physical_at_delivery": 52, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 12, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 5 }, "new_physical_at_delivery": 52, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 12, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 4, "source_not_returned": 1 }, "new_physical_at_delivery": 90, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 88, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 202, "new_annotated_at_delivery": 202, "new_physical_absent_from_native": 145, "new_annotated_absent_from_native": 145 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_not_returned": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 132, "new_annotated_at_delivery": 132, "new_physical_absent_from_native": 49, "new_annotated_absent_from_native": 49 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 132, "new_annotated_at_delivery": 132, "new_physical_absent_from_native": 49, "new_annotated_absent_from_native": 49 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "source_returned": 3 }, "new_physical_at_delivery": 15, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 12, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 3 }, "new_physical_at_delivery": 15, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 12, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 1, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 1, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 5 }, "new_physical_at_delivery": 5, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 3, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 2, "source_not_returned": 2 }, "new_physical_at_delivery": 99, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 48, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "no_definition_at_line": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "no_definition_at_line": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "source_returned": 5 }, "new_physical_at_delivery": 16, "new_annotated_at_delivery": 2, "new_physical_absent_from_native": 10, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 5 }, "new_physical_at_delivery": 16, "new_annotated_at_delivery": 2, "new_physical_absent_from_native": 10, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "source_returned": 2 }, "new_physical_at_delivery": 12, "new_annotated_at_delivery": 5, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 2 }, "new_physical_at_delivery": 12, "new_annotated_at_delivery": 5, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "no_returned_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "no_returned_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 19, "new_annotated_at_delivery": 15, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "no_unseen_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "no_unseen_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 131, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 32, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 1 }, "new_physical_at_delivery": 131, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 32, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "artifacts": { "prior": { "available": true, "queries": 1, "statuses": { "no_definition_at_line": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "no_returned_neighbors": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "artifacts": { "prior": { "available": false, "queries": 1, "statuses": { "graph_unavailable": 1 }, "new_physical_at_delivery": 0, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0 }, "composed": { "available": true, "queries": 1, "statuses": { "source_returned": 2, "source_not_returned": 1 }, "new_physical_at_delivery": 158, "new_annotated_at_delivery": 0, "new_physical_absent_from_native": 156, "new_annotated_absent_from_native": 0 } } } ], "model_calls": 0, "quality_score": null, "annotation_correction_protocol_sha256": "12abf234264f904a4b54d4df6615fae1d2b9b6210eed671fdb926644ec96bf43" }, "validation": { "related_unit_passed": 158, "pinned_integration_passed": 4, "pre_commit": "passed", "model_calls": 0, "budget_refill_regression_reproduced": true }, "next_gate": "Late relationship pilot completes all three arms but graph-source novelty beyond native reaches only C, so its two-language development gate fails. Diagnose oversized definitions and missing graph definitions before a revised controlled candidate; no official 500.", "initial_budget_refill_replay": { "scope": "Fixed-trajectory source replay only; no model calls or new quality score.", "n": 24, "protocol_sha256": "29b8b923ab149ce06301bcdeabb232b9bcca0d46895e7af606c81e081c6aab06", "all_frozen_runtime_and_input_hashes_match": true, "historical_responses_reproduced_exactly": true, "totals": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 63, "added_annotated": 0 }, "policies": { "definitions": { "source_responses": 234, "changed_responses": 0, "characters": 1008050, "repeated_completion_line_deliveries": 726 }, "unseen": { "source_responses": 234, "changed_responses": 29, "characters": 974302, "repeated_completion_line_deliveries": 0 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 14190, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 14190, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "policies": { "definitions": { "source_responses": 14, "changed_responses": 0, "characters": 55825, "repeated_completion_line_deliveries": 87 }, "unseen": { "source_responses": 14, "changed_responses": 1, "characters": 48982, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 47565, "repeated_completion_line_deliveries": 9 }, "unseen": { "source_responses": 9, "changed_responses": 1, "characters": 46516, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "policies": { "definitions": { "source_responses": 3, "changed_responses": 0, "characters": 15343, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 3, "changed_responses": 0, "characters": 15343, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "policies": { "definitions": { "source_responses": 20, "changed_responses": 0, "characters": 84893, "repeated_completion_line_deliveries": 150 }, "unseen": { "source_responses": 20, "changed_responses": 3, "characters": 75875, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "policies": { "definitions": { "source_responses": 8, "changed_responses": 0, "characters": 30123, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 8, "changed_responses": 2, "characters": 31809, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 50, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "policies": { "definitions": { "source_responses": 4, "changed_responses": 0, "characters": 19854, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 4, "changed_responses": 0, "characters": 19854, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 24650, "repeated_completion_line_deliveries": 19 }, "unseen": { "source_responses": 6, "changed_responses": 2, "characters": 24062, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "policies": { "definitions": { "source_responses": 12, "changed_responses": 0, "characters": 51328, "repeated_completion_line_deliveries": 13 }, "unseen": { "source_responses": 12, "changed_responses": 1, "characters": 50718, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 18837, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 18837, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 13937, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 9, "changed_responses": 0, "characters": 13937, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "policies": { "definitions": { "source_responses": 8, "changed_responses": 0, "characters": 17930, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 8, "changed_responses": 0, "characters": 17930, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "policies": { "definitions": { "source_responses": 21, "changed_responses": 0, "characters": 81125, "repeated_completion_line_deliveries": 15 }, "unseen": { "source_responses": 21, "changed_responses": 2, "characters": 80706, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 13, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "policies": { "definitions": { "source_responses": 14, "changed_responses": 0, "characters": 57654, "repeated_completion_line_deliveries": 22 }, "unseen": { "source_responses": 14, "changed_responses": 1, "characters": 56724, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "policies": { "definitions": { "source_responses": 15, "changed_responses": 0, "characters": 70809, "repeated_completion_line_deliveries": 75 }, "unseen": { "source_responses": 15, "changed_responses": 3, "characters": 66256, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 31904, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 9, "changed_responses": 0, "characters": 31904, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 39220, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 39220, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "policies": { "definitions": { "source_responses": 20, "changed_responses": 0, "characters": 174068, "repeated_completion_line_deliveries": 48 }, "unseen": { "source_responses": 20, "changed_responses": 3, "characters": 172228, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 27979, "repeated_completion_line_deliveries": 8 }, "unseen": { "source_responses": 6, "changed_responses": 2, "characters": 27714, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "policies": { "definitions": { "source_responses": 5, "changed_responses": 0, "characters": 17372, "repeated_completion_line_deliveries": 12 }, "unseen": { "source_responses": 5, "changed_responses": 1, "characters": 16788, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 38535, "repeated_completion_line_deliveries": 14 }, "unseen": { "source_responses": 9, "changed_responses": 2, "characters": 37761, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "policies": { "definitions": { "source_responses": 13, "changed_responses": 0, "characters": 28545, "repeated_completion_line_deliveries": 254 }, "unseen": { "source_responses": 13, "changed_responses": 4, "characters": 20584, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "policies": { "definitions": { "source_responses": 5, "changed_responses": 0, "characters": 20709, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 5, "changed_responses": 0, "characters": 20709, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 25655, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 1, "characters": 25655, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 } ], "model_calls": 0, "quality_score": null }, "initial_downstream_packing": { "scope": "Fixed-source downstream-packing diagnostic, not a solver result or quality score.", "protocol_sha256": "6c9119f0047aff949151ac1985e2313aa2485947257f09b8bfc3459341a71508", "n": 24, "historical_16k_32k_payloads_reproduced_exactly": true, "all_frozen_runtime_and_input_hashes_match": true, "totals": { "16384": { "lost_physical": 14, "lost_annotated": 0, "added_physical": 63, "added_annotated": 0, "before_tokens": 195734, "after_tokens": 196262 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 63, "added_annotated": 0, "before_tokens": 218853, "after_tokens": 219612 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2797, "after_tokens": 2797 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2797, "after_tokens": 2797 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10651, "after_tokens": 10651 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10651, "after_tokens": 10651 } } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10785, "after_tokens": 10785 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10785, "after_tokens": 10785 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2724, "after_tokens": 2724 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2724, "after_tokens": 2724 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 16373, "after_tokens": 16373 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 19892, "after_tokens": 19892 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 50, "added_annotated": 0, "before_tokens": 8530, "after_tokens": 9058 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 50, "added_annotated": 0, "before_tokens": 8530, "after_tokens": 9058 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5368, "after_tokens": 5368 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5368, "after_tokens": 5368 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7188, "after_tokens": 7188 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7188, "after_tokens": 7188 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7535, "after_tokens": 7535 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7535, "after_tokens": 7535 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4750, "after_tokens": 4750 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4750, "after_tokens": 4750 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3372, "after_tokens": 3372 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3372, "after_tokens": 3372 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4686, "after_tokens": 4686 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4686, "after_tokens": 4686 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "budgets": { "16384": { "lost_physical": 14, "lost_annotated": 0, "added_physical": 13, "added_annotated": 0, "before_tokens": 16380, "after_tokens": 16380 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 13, "added_annotated": 0, "before_tokens": 19586, "after_tokens": 19817 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 15684, "after_tokens": 15684 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 15684, "after_tokens": 15684 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 13720, "after_tokens": 13720 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 13720, "after_tokens": 13720 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7296, "after_tokens": 7296 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7296, "after_tokens": 7296 } } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 8813, "after_tokens": 8813 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 8813, "after_tokens": 8813 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 16374, "after_tokens": 16374 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 32768, "after_tokens": 32768 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3563, "after_tokens": 3563 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3563, "after_tokens": 3563 } } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5062, "after_tokens": 5062 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5062, "after_tokens": 5062 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 6023, "after_tokens": 6023 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 6023, "after_tokens": 6023 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7850, "after_tokens": 7850 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7850, "after_tokens": 7850 } } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5140, "after_tokens": 5140 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5140, "after_tokens": 5140 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5070, "after_tokens": 5070 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5070, "after_tokens": 5070 } } } ], "model_calls": 0, "quality_score": null }, "initial_implementation_commit": "1d47363e7babc492b4c8ece42363c84bae630115", "unseen_function_replay": { "scope": "Fixed-trajectory source replay only; no model calls or new quality score.", "n": 24, "protocol_sha256": "8788f358fbe2f8e2e2aa98aca32280b590e691b8266943320d316f5970bb1bd4", "all_frozen_runtime_and_input_hashes_match": true, "historical_responses_reproduced_exactly": true, "totals": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, "policies": { "definitions": { "source_responses": 234, "changed_responses": 0, "characters": 1008050, "repeated_completion_line_deliveries": 726 }, "unseen": { "source_responses": 234, "changed_responses": 20, "characters": 971508, "repeated_completion_line_deliveries": 0 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 14190, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 14190, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "policies": { "definitions": { "source_responses": 14, "changed_responses": 0, "characters": 55825, "repeated_completion_line_deliveries": 87 }, "unseen": { "source_responses": 14, "changed_responses": 1, "characters": 48982, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 47565, "repeated_completion_line_deliveries": 9 }, "unseen": { "source_responses": 9, "changed_responses": 1, "characters": 46516, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "policies": { "definitions": { "source_responses": 3, "changed_responses": 0, "characters": 15343, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 3, "changed_responses": 0, "characters": 15343, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "policies": { "definitions": { "source_responses": 20, "changed_responses": 0, "characters": 84893, "repeated_completion_line_deliveries": 150 }, "unseen": { "source_responses": 20, "changed_responses": 3, "characters": 75875, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "policies": { "definitions": { "source_responses": 8, "changed_responses": 0, "characters": 30123, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 8, "changed_responses": 0, "characters": 30123, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "policies": { "definitions": { "source_responses": 4, "changed_responses": 0, "characters": 19854, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 4, "changed_responses": 0, "characters": 19854, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 24650, "repeated_completion_line_deliveries": 19 }, "unseen": { "source_responses": 6, "changed_responses": 1, "characters": 24062, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "policies": { "definitions": { "source_responses": 12, "changed_responses": 0, "characters": 51328, "repeated_completion_line_deliveries": 13 }, "unseen": { "source_responses": 12, "changed_responses": 1, "characters": 50718, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 18837, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 18837, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 13937, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 9, "changed_responses": 0, "characters": 13937, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "policies": { "definitions": { "source_responses": 8, "changed_responses": 0, "characters": 17930, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 8, "changed_responses": 0, "characters": 17930, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "policies": { "definitions": { "source_responses": 21, "changed_responses": 0, "characters": 81125, "repeated_completion_line_deliveries": 15 }, "unseen": { "source_responses": 21, "changed_responses": 2, "characters": 79530, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "policies": { "definitions": { "source_responses": 14, "changed_responses": 0, "characters": 57654, "repeated_completion_line_deliveries": 22 }, "unseen": { "source_responses": 14, "changed_responses": 1, "characters": 56724, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "policies": { "definitions": { "source_responses": 15, "changed_responses": 0, "characters": 70809, "repeated_completion_line_deliveries": 75 }, "unseen": { "source_responses": 15, "changed_responses": 3, "characters": 66256, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 31904, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 9, "changed_responses": 0, "characters": 31904, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 39220, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 39220, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "policies": { "definitions": { "source_responses": 20, "changed_responses": 0, "characters": 174068, "repeated_completion_line_deliveries": 48 }, "unseen": { "source_responses": 20, "changed_responses": 1, "characters": 172228, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 27979, "repeated_completion_line_deliveries": 8 }, "unseen": { "source_responses": 6, "changed_responses": 1, "characters": 27782, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "policies": { "definitions": { "source_responses": 5, "changed_responses": 0, "characters": 17372, "repeated_completion_line_deliveries": 12 }, "unseen": { "source_responses": 5, "changed_responses": 1, "characters": 16788, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "policies": { "definitions": { "source_responses": 9, "changed_responses": 0, "characters": 38535, "repeated_completion_line_deliveries": 14 }, "unseen": { "source_responses": 9, "changed_responses": 1, "characters": 37761, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "policies": { "definitions": { "source_responses": 13, "changed_responses": 0, "characters": 28545, "repeated_completion_line_deliveries": 254 }, "unseen": { "source_responses": 13, "changed_responses": 3, "characters": 20584, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "policies": { "definitions": { "source_responses": 5, "changed_responses": 0, "characters": 20709, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 5, "changed_responses": 0, "characters": 20709, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "policies": { "definitions": { "source_responses": 6, "changed_responses": 0, "characters": 25655, "repeated_completion_line_deliveries": 0 }, "unseen": { "source_responses": 6, "changed_responses": 0, "characters": 25655, "repeated_completion_line_deliveries": 0 } }, "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0 } ], "model_calls": 0, "quality_score": null }, "downstream_packing": { "scope": "Fixed-source downstream-packing diagnostic, not a solver result or quality score.", "protocol_sha256": "f56dde37b5cd2735e412306760f2bfaa1aa8aaf8b1305a4099aef9233ce0a74f", "n": 24, "historical_16k_32k_payloads_reproduced_exactly": true, "candidate_16k_32k_payloads_identical_to_historical": true, "all_frozen_runtime_and_input_hashes_match": true, "totals": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 195734, "after_tokens": 195734 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 218853, "after_tokens": 218853 } }, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2797, "after_tokens": 2797 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2797, "after_tokens": 2797 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10651, "after_tokens": 10651 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10651, "after_tokens": 10651 } } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10785, "after_tokens": 10785 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 10785, "after_tokens": 10785 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2724, "after_tokens": 2724 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 2724, "after_tokens": 2724 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 16373, "after_tokens": 16373 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 19892, "after_tokens": 19892 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 8530, "after_tokens": 8530 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 8530, "after_tokens": 8530 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5368, "after_tokens": 5368 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5368, "after_tokens": 5368 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7188, "after_tokens": 7188 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7188, "after_tokens": 7188 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7535, "after_tokens": 7535 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7535, "after_tokens": 7535 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4750, "after_tokens": 4750 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4750, "after_tokens": 4750 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3372, "after_tokens": 3372 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3372, "after_tokens": 3372 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4686, "after_tokens": 4686 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 4686, "after_tokens": 4686 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 16380, "after_tokens": 16380 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 19586, "after_tokens": 19586 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 15684, "after_tokens": 15684 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 15684, "after_tokens": 15684 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 13720, "after_tokens": 13720 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 13720, "after_tokens": 13720 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7296, "after_tokens": 7296 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7296, "after_tokens": 7296 } } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 8813, "after_tokens": 8813 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 8813, "after_tokens": 8813 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 16374, "after_tokens": 16374 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 32768, "after_tokens": 32768 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3563, "after_tokens": 3563 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 3563, "after_tokens": 3563 } } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5062, "after_tokens": 5062 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5062, "after_tokens": 5062 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 6023, "after_tokens": 6023 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 6023, "after_tokens": 6023 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7850, "after_tokens": 7850 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 7850, "after_tokens": 7850 } } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5140, "after_tokens": 5140 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5140, "after_tokens": 5140 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "budgets": { "16384": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5070, "after_tokens": 5070 }, "32768": { "lost_physical": 0, "lost_annotated": 0, "added_physical": 0, "added_annotated": 0, "before_tokens": 5070, "after_tokens": 5070 } } } ], "model_calls": 0, "quality_score": null } }, "annotation_path_correction": { "scope": "Retrospective auxiliary-overlap correction using the pinned official Gold.line_spans_init() path normalization. Original frozen experiment files remain unchanged. The listed auxiliary count blocks are amended here; official scores, confidence intervals and model outputs are unchanged.", "artifact": "contextbench_annotation_correction.json", "protocol_sha256": "12abf234264f904a4b54d4df6615fae1d2b9b6210eed671fdb926644ec96bf43", "amended_blocks": [ "development_source_attribution", "first_callable_source_24_development.final_source_set_diagnostic", "offline_graph_preparation_24.source_replay", "completion_source_followup.existing_source_attribution", "completion_source_followup.prepared_graph_fixed_query_replay" ] }, "multi_gson_native_readiness": { "scope": "One disclosed Java development task. Native reference/control, source-only preflight and synthetic adapter fixtures only; zero model calls and no agent repair score.", "contextbench_instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "dataset": { "url": "https://huggingface.co/datasets/ByteDance-Seed/Multi-SWE-bench/resolve/56ff018c04a38e27ada1e9d0a6d5839a51f88f0d/java/google__gson_dataset.jsonl", "source_file_sha256": "fc4e6547d8ccd761719c68dc09053b9e10ba9aca8db97b4b565dbf88c83473f6", "source_bytes": 122857, "source_rows": 5, "selected_record_sha256": "a04ab6cc537076d0b6c7a6136d71a59cc91f66f79933488de03338c350b83658", "instance_id": "google__gson-1391", "base_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "org": "google", "repo": "gson", "number": 1391, "test_counts": { "f2p_tests": 1, "p2p_tests": 95, "s2p_tests": 0, "n2p_tests": 0 }, "model_calls": 0, "scope": "Native grader reference/control only; reference data never enters solver inputs or image." }, "source_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "source_tree": "abba7deda8852191b18a879e838fbbea4a8b19ff", "native_image": "mswebench/google_m_gson@sha256:135ea364c947f1959102c1efe6f085e879f756cc8eecb3ded672b3c3c0f93d7e", "native_image_id": "sha256:3b2567e7897733e4b04fa5087d0db089cc56d247f33ca62d0f80360594efe16e", "solver_image_id": "sha256:a258cc0df8fa2815689f4f2680bd4d062c0f758b6848d98a924c973cf8d521ea", "solver_dockerfile_sha256": "ddf1507ae401fb15f5c2b62073921bd67293f7b235e5855d681ebe2c4ebbb223", "solver_dockerfile": "FROM mswebench/google_m_gson@sha256:135ea364c947f1959102c1efe6f085e879f756cc8eecb3ded672b3c3c0f93d7e\n\nWORKDIR /home/gson\nRUN git ls-files -z > /tmp/codenib-selected-files \\\n && rm -rf .git \\\n && rm -f /home/fix.patch /home/test.patch /home/fix-run.sh \\\n /home/test-run.sh /home/prepare.sh /home/check_git_changes.sh \\\n && git init -q \\\n && git config user.name Benchmark \\\n && git config user.email benchmark@example.invalid \\\n && git add --force --pathspec-from-file=/tmp/codenib-selected-files --pathspec-file-nul \\\n && git commit -qm 'chore(benchmark): initialize selected source snapshot' \\\n && test \"$(git rev-parse HEAD^{tree})\" = abba7deda8852191b18a879e838fbbea4a8b19ff \\\n && test \"$(git rev-list --all --count)\" = 1 \\\n && test -z \"$(git status --porcelain --untracked-files=all)\" \\\n && python3 --version \\\n && rm /tmp/codenib-selected-files\n", "harness_revision": "24f493f8a103e72312ded4f6b9c89f081d69cb09", "harness_version": "1.1.0", "harness_module_sha256": { "__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "harness/gen_report.py": "02ebc8a5414898d12f4f5a9ba0c11a8f57c9f34a0bdc02c2311afac9f654847d", "harness/report.py": "5a025fd496d42c4b7377fc0702d64c6d0e356b117eaf2face47e73a52c29902f", "harness/test_result.py": "5411af794920cf4b170fe9dbe8c21c12cc63e2bbe2280d6d82acb850f4808be3", "harness/dataset.py": "dd49f55baf63b60fff309b6a5b2a1826697e2b85ad1a9bccff18321dcdc200fc", "harness/image.py": "86074812495b97026efb42c57acbf7738864b1f0167f99e3b9f9309458972ae9", "harness/instance.py": "93da4f6db69e006949bba856e22e4a464b5e93140590258eda38833ec5f19a9e", "harness/pull_request.py": "32b49f48b39124f67727f408898bd96cce91c0a362faa716ac858dcb0b0b47c7", "harness/repos/java/google/gson.py": "faec1d6ba360ab902a663718b9a8668d1a46a860bc613980b310107153adf890" }, "native_script_sha256": { "run.sh": "928b9d846144bb38ff4e5314d8b24b06225c335f32a44115350a735d0f9a4bc3", "test-run.sh": "def4691d818e711ed1e42b0b1c812216a9530f606f0bc50a97f4c9b03a110762", "fix-run.sh": "9d60f6e78b1903556a79f60f37b6bf007a3ac2cbbc4aed7a91bad7a6e9c62bd3" }, "original_native_protocol_sha256": "0ddc4234dc8f136cdb76d9ed1c9a816086ca29d849c18c75a04c68f5330e9cc9", "adapter_protocol_sha256": "c4bb87f172c152ed8896ee14944c2aed1dc87c597d58c5c7dfafec157c9bc3be", "adapter_implementation": { "scripts/benchmark_context_solver.py": "0ec005afed874d11b9acda178b1e8fb338e37e34f00d4b37fa1561209e88255e", "scripts/contextbench_multiswe.py": "8f96c14f4707172b3009529322cd90dd86cad0be0985d43d4785c8be035f64e4", "scripts/contextbench_solver_sources.py": "a9d81954376b28752351a2f81494aa5722227e9cf157d51fdc439c60879b0142", "scripts/_contextbench_solver_read.py": "68e2e2e417150ce45e9aa692ee2c6b852b8500df404ee705949fc21b4c6f131c" }, "source_preflight": { "model_calls": 0, "isolation": { "network": "none", "host_mounts": [] }, "image_snapshot": { "commit": "369ade7f23b1588a285bb4d89ecb7b6674eeebbc", "tree": "abba7deda8852191b18a879e838fbbea4a8b19ff", "matches_selected_tree": true }, "source_only": { "one_source_commit": true, "reference_artifacts_absent": true }, "controlled_source_read_passed": true, "model_delivery_confirmed": false, "status": "passed", "container_removed": true }, "original_native_controls": { "test_only": { "stage": "test_only", "status": "complete", "model_calls": 0, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "9277dc6f50b4841a90cec3379a0bd804eacc3a754bd435e1ca373238b0796bcc", "test_patch": "f1354f494072033f276e78a565745cb717b7fc28f1f32d86dc12c7ffa2df5a54" }, "native_script_sha256": { "run.sh": "928b9d846144bb38ff4e5314d8b24b06225c335f32a44115350a735d0f9a4bc3", "test-run.sh": "def4691d818e711ed1e42b0b1c812216a9530f606f0bc50a97f4c9b03a110762", "fix-run.sh": "9d60f6e78b1903556a79f60f37b6bf007a3ac2cbbc4aed7a91bad7a6e9c62bd3" }, "native_exit_code": 1, "native_parser_counts": { "passed": 95, "failed": 2, "skipped": 0, "granularity": "test_class" }, "regression_passed": 0, "regression_failed": 1, "preservation_passed": 95, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "google/gson:pr-1391" ], "completed_ids": [ "google/gson:pr-1391" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "google/gson:pr-1391" ], "empty_patch_ids": [], "error_ids": [] }, "native_report_sha256": "96ae0cde6bba9b83aafde363b9956e7ecbc912d33462ee1221fc386bd0fda1ee", "native_log_sha256": "b6d0f4098f871a64e7f3cc6a65f63a491d195e4a3ccc8518ffce3c25484e6a88", "container_removed": true, "wall_s": 14.667143717058934 }, "reference": { "stage": "reference", "status": "complete", "model_calls": 0, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "9277dc6f50b4841a90cec3379a0bd804eacc3a754bd435e1ca373238b0796bcc", "test_patch": "f1354f494072033f276e78a565745cb717b7fc28f1f32d86dc12c7ffa2df5a54" }, "native_script_sha256": { "run.sh": "928b9d846144bb38ff4e5314d8b24b06225c335f32a44115350a735d0f9a4bc3", "test-run.sh": "def4691d818e711ed1e42b0b1c812216a9530f606f0bc50a97f4c9b03a110762", "fix-run.sh": "9d60f6e78b1903556a79f60f37b6bf007a3ac2cbbc4aed7a91bad7a6e9c62bd3" }, "native_exit_code": 1, "native_parser_counts": { "passed": 96, "failed": 1, "skipped": 0, "granularity": "test_class" }, "regression_passed": 1, "regression_failed": 0, "preservation_passed": 95, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "google/gson:pr-1391" ], "completed_ids": [ "google/gson:pr-1391" ], "incomplete_ids": [], "resolved_ids": [ "google/gson:pr-1391" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "native_report_sha256": "b09f3d4d9e672c3211b77c7b18705fc3bc89ecc662afb4ae177f8d190d74a1d0", "native_log_sha256": "57165b571a97bacd4ff6b18ba394a02421a22dd2a5297a226e145a21270edc3e", "container_removed": true, "wall_s": 13.41349797998555 } }, "adapter_controls": { "neutral": { "status": "graded", "resolved": false, "model_calls": 0, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "9277dc6f50b4841a90cec3379a0bd804eacc3a754bd435e1ca373238b0796bcc", "test_patch": "f1354f494072033f276e78a565745cb717b7fc28f1f32d86dc12c7ffa2df5a54" }, "native_script_sha256": { "run.sh": "928b9d846144bb38ff4e5314d8b24b06225c335f32a44115350a735d0f9a4bc3", "test-run.sh": "def4691d818e711ed1e42b0b1c812216a9530f606f0bc50a97f4c9b03a110762", "fix-run.sh": "9d60f6e78b1903556a79f60f37b6bf007a3ac2cbbc4aed7a91bad7a6e9c62bd3" }, "applied_solver_patch_sha256": "438aa919ff02bbcb90d8e974e1ee68df7fc13148f37973e40e94f9ef75129f8b", "native_exit_code": 1, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "google/gson:pr-1391" ], "completed_ids": [ "google/gson:pr-1391" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "google/gson:pr-1391" ], "empty_patch_ids": [], "error_ids": [] }, "native_log_sha256": "b347c62fac45c6b3f0743d04fc2c4002cd5ddf1e4534732d0bc0fa2eb0d1ec32", "container_removed": true, "wall_s": 14.357268824940547, "process_exit_code": 0 }, "reference": { "status": "graded", "resolved": true, "model_calls": 0, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "9277dc6f50b4841a90cec3379a0bd804eacc3a754bd435e1ca373238b0796bcc", "test_patch": "f1354f494072033f276e78a565745cb717b7fc28f1f32d86dc12c7ffa2df5a54" }, "native_script_sha256": { "run.sh": "928b9d846144bb38ff4e5314d8b24b06225c335f32a44115350a735d0f9a4bc3", "test-run.sh": "def4691d818e711ed1e42b0b1c812216a9530f606f0bc50a97f4c9b03a110762", "fix-run.sh": "9d60f6e78b1903556a79f60f37b6bf007a3ac2cbbc4aed7a91bad7a6e9c62bd3" }, "applied_solver_patch_sha256": "9277dc6f50b4841a90cec3379a0bd804eacc3a754bd435e1ca373238b0796bcc", "native_exit_code": 1, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "google/gson:pr-1391" ], "completed_ids": [ "google/gson:pr-1391" ], "incomplete_ids": [], "resolved_ids": [ "google/gson:pr-1391" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "native_log_sha256": "92f739cd0a60a7f3ac916acc324135b74e3e06d64a82af86f56bf5210071015d", "container_removed": true, "wall_s": 24.335912427981384, "process_exit_code": 0 } }, "parser_parity": { "scope": "Separate post-run full native-parser comparison; original frozen protocol/results retained.", "all_test_label_sets_match_pinned_dataset": true, "always_failing_class": "com.google.gson.DefaultDateTypeAdapterTest", "always_failure_present_in_pinned_run_test_and_fix": true, "both_native_build_exit_codes": { "test_only": 1, "reference": 1 }, "native_quality_score": null, "model_calls": 0 }, "adapter_parser_parity": { "scope": "Post-run exact parser comparison; no new model calls or agent quality evidence.", "stages": { "neutral": { "passed_classes": 95, "failed_classes": 2, "skipped_classes": 0, "required_regression_passed": 0, "required_preservation_passed": 95, "matches_original_native_control": true }, "reference": { "passed_classes": 96, "failed_classes": 1, "skipped_classes": 0, "required_regression_passed": 1, "required_preservation_passed": 95, "matches_original_native_control": true } } }, "cpp_entrypoint_compatibility": { "scope": "C++ image compatibility for newly authenticated native entrypoints, no native test rerun or model calls.", "native_harness_hashes": { "harness/gen_report.py": "02ebc8a5414898d12f4f5a9ba0c11a8f57c9f34a0bdc02c2311afac9f654847d", "harness/report.py": "5a025fd496d42c4b7377fc0702d64c6d0e356b117eaf2face47e73a52c29902f", "harness/test_result.py": "5411af794920cf4b170fe9dbe8c21c12cc63e2bbe2280d6d82acb850f4808be3", "harness/dataset.py": "dd49f55baf63b60fff309b6a5b2a1826697e2b85ad1a9bccff18321dcdc200fc", "harness/image.py": "86074812495b97026efb42c57acbf7738864b1f0167f99e3b9f9309458972ae9", "harness/instance.py": "93da4f6db69e006949bba856e22e4a464b5e93140590258eda38833ec5f19a9e", "harness/pull_request.py": "32b49f48b39124f67727f408898bd96cce91c0a362faa716ac858dcb0b0b47c7", "harness/repos/cpp/fmtlib/fmt.py": "d0a5146eb77b4416cccb86216c3df08f200d58c33bb158d953aaac385f922918", "harness/repos/java/google/gson.py": "faec1d6ba360ab902a663718b9a8668d1a46a860bc613980b310107153adf890" }, "expected_script_sha256": { "run.sh": "8d556d2484f4e8847a40c4cc39e7eb3fdd916f46eaf094c4913fdf733ce54922", "test-run.sh": "fe480a99fb0e475058e6cdb91947477612bc9636603fbccf343cff5b2def1a5e", "fix-run.sh": "dff349820a4f0eb01d047ca1f33b778f6450d49270c688fbb5710ebf3b0d48e6" }, "model_calls": 0, "passed": true, "actual_script_sha256": { "run.sh": "8d556d2484f4e8847a40c4cc39e7eb3fdd916f46eaf094c4913fdf733ce54922", "test-run.sh": "fe480a99fb0e475058e6cdb91947477612bc9636603fbccf343cff5b2def1a5e", "fix-run.sh": "dff349820a4f0eb01d047ca1f33b778f6450d49270c688fbb5710ebf3b0d48e6" }, "container_removed": true }, "all_frozen_runtime_and_input_hashes_match": true, "model_calls": 0, "agent_quality_score": null, "new_api_charges_usd": 0, "other_repository_gate": "Jackson-databind preparation rewrites its tracked POM in the pinned harness, conflicting with the current exact-clean-source gate. This is source inspection, not a failed image probe; no Jackson image or model run. No wider Multi, Poly or Pro support is claimed." }, "multi_gson_development_solver_pair": { "protocol": { "scope": "One disclosed Java development task, two fresh downstream solver arms. No independent validation, new retrieval, graph-policy quality gate or official submission claim.", "selection": "The previously disclosed Gson Java task selected for native adapter validation after Jackson preparation was deferred. Selection uses environment compatibility, not repair scores; this is a one-task development pair, not an independent or representative sample.", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "original_inst_id": "google__gson-1391", "retrieval_scope": "Frozen first-source 24-task native and functions_graph delivered contexts. This task had no graph in that retrieval run, so any difference cannot establish a graph-source contribution. The newer graph artifacts are not used.", "common_reader": "Corrected standalone-reader usage feedback at the existing environment boundary; identical instructions in both arms. Source coverage remains partial.", "acceptance": "Report both single-attempt native repair outcomes and confirmed original-source receipts, including any zero or failure. No quality promotion, independent score or population Pass@1 claim.", "inputs": { "native": { "retrieval_sha256": "2a9cea40a531a17c0baa59903ba4e12ea9ca4fe273dc4a11c6934f6ace32c9c0", "source_tree": "abba7deda8852191b18a879e838fbbea4a8b19ff", "retrieval_config_sha256": "9a71dfb5dc293cfc96f4be92a67cea47c5589bfdf335afcce9e432d13df09ea6" }, "functions_graph": { "retrieval_sha256": "d88f01c34f03907fd9e6db75943bd5c091d4cf1e9f17421528fb79126f4ae818", "source_tree": "abba7deda8852191b18a879e838fbbea4a8b19ff", "retrieval_config_sha256": "a17ead620e469c99a8183e8f55a04b94cf3a08cd04938b285ba6b75ea4b0ba0f" } }, "arms": [ "native", "functions_graph" ], "source_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "image_id": "sha256:a258cc0df8fa2815689f4f2680bd4d062c0f758b6848d98a924c973cf8d521ea", "native_adapter_check_sha256": "26230296d5a603c17d3ff4bbec01e4cebf320a3e2b16bfa8437fa0fc427eed3b", "native_image_id": "sha256:3b2567e7897733e4b04fa5087d0db089cc56d247f33ca62d0f80360594efe16e", "native_dataset_sha256": "a04ab6cc537076d0b6c7a6136d71a59cc91f66f79933488de03338c350b83658", "native_test_timeout_seconds": 600, "native_start_timeout_seconds": 180, "implementation_commit": "0039b2a28d6fc07000156518e60456bc713469a9", "implementation": { "scripts/benchmark_context_agent.py": "5c4f9c451af797558174d19829f7f59f7ee8188b270295d9789ad4bbaee47721", "scripts/benchmark_context_claude.py": "ca2feb3556e5ec46f656f70ba1175334a71a1600eb25c8b8ff59bce7c84c9b6d", "scripts/contextbench_graph_navigation.py": "bf27c49e1688d1f2a9fb8e42240c4e96556baa4f9bc7dec03cfb69a50f287775", "scripts/benchmark_context_serving.py": "6fa40f7ad145938efcb1edceddcad864b2febe0c1c61dda315d9cd5122048fe2", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "1d6d1718d24abaccb46e4f6743ac8033e145628fbc55ceec85d07c0390b39609", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/eval/benchmarks/_contextbench_evaluate.py": "26b82cc9e800759469c52d972ac2f4b5cfda69f052601841f026f05e28cf222c", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99", "scripts/benchmark_context_solver.py": "0ec005afed874d11b9acda178b1e8fb338e37e34f00d4b37fa1561209e88255e", "scripts/contextbench_multiswe.py": "8f96c14f4707172b3009529322cd90dd86cad0be0985d43d4785c8be035f64e4", "scripts/contextbench_solver_sources.py": "a9d81954376b28752351a2f81494aa5722227e9cf157d51fdc439c60879b0142", "scripts/_contextbench_solver_read.py": "68e2e2e417150ce45e9aa692ee2c6b852b8500df404ee705949fc21b4c6f131c", "mini/__init__.py": "79695f6e5b2f6f90093cd15f08dd9bba2ab099e67487f4ac93bb739e22e19fff", "mini/agents/default.py": "faf4442506b1cdaad886f146182dff98ed740d45536729617efbdcb6d5fb5524", "mini/environments/docker.py": "e3c42a76d0aee03edb0e14e693c963d4fc2e28b678ca36a72f774c90dc89dbd4", "mini/config/extra/swebench.yaml": "81bb47b91311e19b73ff0f8bc670898dc5b807f774ff687e11556038e5878162" }, "driver_sha256": "dfcc219ad563458fdfdda14e4643eb953cb60a249507d43dc127b9be9a736da4", "grading_driver_sha256": "1be0af88cf48816fcbbcf8303c1602f0c130daad567ec2d5113a9f4018238c3d", "model": "claude-opus-5-5", "effort": "medium", "context_budget": 16384, "max_turns": 20, "max_meter_usd_per_arm": 2, "sdk_meter_is_actual_billing": false, "preferred_route": "existing_claude_max", "new_api_charges_authorized_ceiling_usd": 100, "standalone_api_key_fallback": false, "stop_rule": "No retry. Stop further inference on process/solver errors or missing terminal metering. Preserve not-run arms and missing usage. Grade only after both solver attempts end, without feedback to either model.", "grading": "Original Multi Gson script in a separate pinned native image and isolated pinned native harness. One unique task; reference checks are excluded from agent Pass@1." }, "analysis": { "protocol_sha256": "3e42a546571c16b1ebc33b27eff275156807b3b28348b66d21be6a9c59aa35f2", "scope": "One disclosed Java development task, two fresh downstream solver arms. No independent validation, new retrieval, graph-policy quality gate or official submission claim.", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "original_inst_id": "google__gson-1391", "implementation_commit": "0039b2a28d6fc07000156518e60456bc713469a9", "arms": { "native": { "status": "Submitted", "resolved": true, "model_calls": 5, "sdk_estimate_usd": 0.5006415000000001, "known_sdk_estimate_usd": 0.5006415000000001, "missing_metering_calls": 0, "actual_models": [ "claude-opus-5-5" ], "sdk_init_api_key_sources": [ "none" ], "all_model_inputs_confirmed": true, "initial_base_source_lines": 318, "base_source_line_count": 318, "new_original_source_lines": 0, "source_read_events": 2, "confirmed_original_source_reads": 1, "partial_source_coverage": true, "missing_execution_receipt_indexes": [], "unattributed_delivered_shell_events": 2, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": true, "native_exit_code": 1, "container_removed": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "google/gson:pr-1391" ], "completed_ids": [ "google/gson:pr-1391" ], "incomplete_ids": [], "resolved_ids": [ "google/gson:pr-1391" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "native_log_sha256": "5a7a27e2b83e81d91b03d8d14d282b5fc3b4f1409c57fb3118be97ec434e2e01" }, "model_patch_sha256": "756bba27f6b66d57570ba930890974917af950005a9f4a3f90df98a333d092e0" }, "functions_graph": { "status": "Submitted", "resolved": true, "model_calls": 6, "sdk_estimate_usd": 0.587861, "known_sdk_estimate_usd": 0.587861, "missing_metering_calls": 0, "actual_models": [ "claude-opus-5-5" ], "sdk_init_api_key_sources": [ "none" ], "all_model_inputs_confirmed": true, "initial_base_source_lines": 293, "base_source_line_count": 294, "new_original_source_lines": 1, "source_read_events": 2, "confirmed_original_source_reads": 1, "partial_source_coverage": true, "missing_execution_receipt_indexes": [], "unattributed_delivered_shell_events": 3, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": true, "native_exit_code": 1, "container_removed": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "google/gson:pr-1391" ], "completed_ids": [ "google/gson:pr-1391" ], "incomplete_ids": [], "resolved_ids": [ "google/gson:pr-1391" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "native_log_sha256": "de4640fd05f00286b65b786d4d06edfd97cc5743b5985f77c0aad2cfbcc0682d" }, "model_patch_sha256": "12b0205398cd7328d3e66ffcd413ab319c29213ed5106bbaf61757a98cd6786d" } }, "graph_available_in_original_retrieval": false, "independent_quality_score": null, "known_program_sdk_estimate_before": 73.58967420000002, "matched_configs_and_common_prompt": true, "known_sdk_estimate_usd": 1.0885025000000002, "sdk_estimate_usd": 1.0885025000000002, "known_program_sdk_estimate_usd": 74.67817670000002, "new_api_key_charges_usd": 0, "all_frozen_runtime_and_input_hashes_match_after_grading": true, "sdk_meter_is_actual_billing": false }, "native_test_counts": { "scope": "Original native parser applied after both real solver arms terminate.", "arms": { "native": { "required_regression_passed": 1, "required_preservation_passed": 95, "passed_classes": 96, "failed_classes": 1, "skipped_classes": 0, "granularity": "test_class", "known_failure_retained": true }, "functions_graph": { "required_regression_passed": 1, "required_preservation_passed": 95, "passed_classes": 96, "failed_classes": 1, "skipped_classes": 0, "granularity": "test_class", "known_failure_retained": true } } }, "conclusion": "Both arms resolve the same one disclosed Java task, 1/1 each. No comparative gain, no graph in original retrieval and no independent validation. Controlled-reader coverage stays explicitly partial." }, "multi_ponyc_native_readiness": { "scope": "One outcome-informed disclosed C development task. Original native reference/control and source-only baseline/reader checks; no model calls or agent quality score.", "selection": { "scope": "Native environment readiness for one disclosed C development task; no model calls or independent selection.", "contextbench_instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "original_inst_id": "ponylang__ponyc-2205", "base_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "reason": "The completed late-relationship development trace supplies 197 annotated graph lines absent from native. Test whether such source can support a future matched repair after validating the native harness. This is an outcome-informed development diagnostic, not an unbiased quality cohort.", "api_ceiling_usd": 100, "new_api_calls": 0, "model_calls": 0 }, "dataset": { "url": "https://huggingface.co/datasets/ByteDance-Seed/Multi-SWE-bench/resolve/56ff018c04a38e27ada1e9d0a6d5839a51f88f0d/c/ponylang__ponyc_dataset.jsonl", "sha256": "cddd6e471bd8a1ad9e9971d8a16ca5a0e9f002789cefc5a40ee025f8875122ee", "bytes": 26444998, "rows": 82, "selected_record_sha256": "2967a9f7095b07c8a586150c6c426951e39353b5982eb0a7ca566c78c424c222", "instance_id": "ponylang__ponyc-2205", "base_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "test_counts": { "f2p_tests": 7, "p2p_tests": 1010, "s2p_tests": 0, "n2p_tests": 40 }, "model_calls": 0, "scope": "Native grader data only; no reference material enters solver inputs." }, "native_protocol_sha256": "a7a21664742a4d07ca15b24076d2195be7b1affdf445ed2d2d8f05bb0da75ecd", "native_image_ref": "mswebench/ponylang_m_ponyc@sha256:9b0d71d45b6f6e1a1ce0f4e142cbedc95363c0b9b439cd22b16c0c65d88b41cc", "native_image_id": "sha256:3b1afa3e1afd231bd1c93a003f1fbf34b24888a5014cce278f026b13361b5216", "source_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "source_tree": "012befc5225065c77600560ceacc63453852c152", "harness_revision": "24f493f8a103e72312ded4f6b9c89f081d69cb09", "harness_version": "1.1.0", "harness_module_sha256": { "__init__.py": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "harness/gen_report.py": "02ebc8a5414898d12f4f5a9ba0c11a8f57c9f34a0bdc02c2311afac9f654847d", "harness/report.py": "5a025fd496d42c4b7377fc0702d64c6d0e356b117eaf2face47e73a52c29902f", "harness/test_result.py": "5411af794920cf4b170fe9dbe8c21c12cc63e2bbe2280d6d82acb850f4808be3", "harness/dataset.py": "dd49f55baf63b60fff309b6a5b2a1826697e2b85ad1a9bccff18321dcdc200fc", "harness/image.py": "86074812495b97026efb42c57acbf7738864b1f0167f99e3b9f9309458972ae9", "harness/instance.py": "93da4f6db69e006949bba856e22e4a464b5e93140590258eda38833ec5f19a9e", "harness/pull_request.py": "32b49f48b39124f67727f408898bd96cce91c0a362faa716ac858dcb0b0b47c7", "harness/repos/c/ponylang/ponyc.py": "be551b5b2b3bb0367e12b1f46a5bc5a5d7e5ab161cced3921cd05e18f90ae744" }, "native_script_sha256": { "run.sh": "86a3c6ab8ee3f2e8acd00b14ad8c47c1d1eba7e62eeae39a36912d430fbf90dd", "test-run.sh": "caa353f087b8b9df6bd27d6c50089ea24ddda738cf3bace8250a0c359eca65e5", "fix-run.sh": "746c4cfc9c5fea71fb32a399a78bfdd17e3d04fe65049f161005c08b6a575676" }, "native_controls": { "test_only": { "stage": "test_only", "status": "complete", "model_calls": 0, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "6f50cbec2c943b0c6e3b83804ec24718e1a3769e2000f078e6000e94a21c281b", "test_patch": "c5a5e0495ec35ae2552f2e51c0dd0f31feada0530010446bdbdd946e960d54c1" }, "native_script_sha256": { "run.sh": "86a3c6ab8ee3f2e8acd00b14ad8c47c1d1eba7e62eeae39a36912d430fbf90dd", "test-run.sh": "caa353f087b8b9df6bd27d6c50089ea24ddda738cf3bace8250a0c359eca65e5", "fix-run.sh": "746c4cfc9c5fea71fb32a399a78bfdd17e3d04fe65049f161005c08b6a575676" }, "source_reader_python": { "returncode": 127, "output": "" }, "native_exit_code": 2, "native_parser_counts": { "passed": 1010, "failed": 7, "skipped": 0, "granularity": "test_case" }, "regression_passed": 0, "regression_failed": 7, "preservation_passed": 1010, "new_cases_passed": 0, "parser_sets_match_pinned_dataset": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "ponylang/ponyc:pr-2205" ], "completed_ids": [ "ponylang/ponyc:pr-2205" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "ponylang/ponyc:pr-2205" ], "empty_patch_ids": [], "error_ids": [] }, "native_report_sha256": "77f8d2a3e579e4b3dac774cd667c98b3d1080952b5b110329121eb061ec52833", "native_log_sha256": "2ffcd81beff5648e55f4e3bc74f5d745582d6efb51047e432492ccd0e6b67d3d", "container_removed": true, "wall_s": 122.97219668806065 }, "reference": { "stage": "reference", "status": "complete", "model_calls": 0, "source_snapshot_verified": true, "native_patch_sha256": { "fix_patch": "6f50cbec2c943b0c6e3b83804ec24718e1a3769e2000f078e6000e94a21c281b", "test_patch": "c5a5e0495ec35ae2552f2e51c0dd0f31feada0530010446bdbdd946e960d54c1" }, "native_script_sha256": { "run.sh": "86a3c6ab8ee3f2e8acd00b14ad8c47c1d1eba7e62eeae39a36912d430fbf90dd", "test-run.sh": "caa353f087b8b9df6bd27d6c50089ea24ddda738cf3bace8250a0c359eca65e5", "fix-run.sh": "746c4cfc9c5fea71fb32a399a78bfdd17e3d04fe65049f161005c08b6a575676" }, "source_reader_python": { "returncode": 127, "output": "" }, "native_exit_code": 2, "native_parser_counts": { "passed": 1057, "failed": 0, "skipped": 0, "granularity": "test_case" }, "regression_passed": 7, "regression_failed": 0, "preservation_passed": 1010, "new_cases_passed": 40, "parser_sets_match_pinned_dataset": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 1, "unresolved_instances": 0, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "ponylang/ponyc:pr-2205" ], "completed_ids": [ "ponylang/ponyc:pr-2205" ], "incomplete_ids": [], "resolved_ids": [ "ponylang/ponyc:pr-2205" ], "unresolved_ids": [], "empty_patch_ids": [], "error_ids": [] }, "native_report_sha256": "8eddfc39a0df4c3b1d46f9d020f3ddd5faadb09b6e12d32b7a55107e08aebd11", "native_log_sha256": "646143b9cc16ca6be535d7048f0eb8e5493a826f5eaceaac70bd93d1b873ce20", "container_removed": true, "wall_s": 145.97421881591436 } }, "cold_source_image_id": "sha256:370bfd936a11b76f8e4b2e4b433dd509c8871ecac6ffa45fa6b2b162c736663f", "warm_source_image_id": "sha256:8e792f86261b3fc70dc9cc5f1c914f001d650b869c9a70754718c28384c4f20f", "python_runtime": { "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20260211/cpython-3.11.14%2B20260211-x86_64-unknown-linux-gnu-install_only_stripped.tar.gz", "archive_sha256": "1fb6a89daa750f7eff00eb65d901c3c236713812eb9c8c7e07ecdfcd78203c8e", "archive_bytes": 31215121, "version": "3.11.14", "release": "20260211", "target": "x86_64-unknown-linux-gnu", "provenance": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260211", "compatibility_reference": "https://gregoryszorc.com/docs/python-build-standalone/stable/running.html", "scope": "Private solver-image runtime only. No host-global installation, package dependency, native-grader change or model calls." }, "cold_dockerfile": "FROM mswebench/ponylang_m_ponyc@sha256:9b0d71d45b6f6e1a1ce0f4e142cbedc95363c0b9b439cd22b16c0c65d88b41cc\n\nCOPY python /opt/codenib-python\nENV PATH=\"/opt/codenib-python/bin:${PATH}\"\nWORKDIR /home/ponyc\nRUN git ls-files -z > /tmp/codenib-selected-files \\\n && rm -rf .git \\\n && rm -f /home/fix.patch /home/test.patch /home/fix-run.sh \\\n /home/test-run.sh /home/prepare.sh /home/check_git_changes.sh \\\n && git init -q \\\n && git config user.name Benchmark \\\n && git config user.email benchmark@example.invalid \\\n && git config gc.auto 0 \\\n && xargs -0 -r git add --force -- < /tmp/codenib-selected-files \\\n && git commit -qm 'chore(benchmark): initialize selected source snapshot' \\\n && test \"$(git rev-parse HEAD^{tree})\" = 012befc5225065c77600560ceacc63453852c152 \\\n && test \"$(git rev-list --all --count)\" = 1 \\\n && test -z \"$(git status --porcelain --untracked-files=all)\" \\\n && python3 --version \\\n && python3 -c 'import hashlib, json, pathlib, subprocess' \\\n && rm /tmp/codenib-selected-files\n", "cold_dockerfile_sha256": "a0c9f2387e9f73cafcc9665b2f79ac6ab6437f1605fc683ed725efc4debb4dad", "warm_dockerfile": "FROM mswebench/ponylang_m_ponyc@sha256:9b0d71d45b6f6e1a1ce0f4e142cbedc95363c0b9b439cd22b16c0c65d88b41cc\n\nCOPY python /opt/codenib-python\nENV PATH=\"/opt/codenib-python/bin:${PATH}\"\nWORKDIR /home/ponyc\nRUN git ls-files -z > /tmp/codenib-selected-files \\\n && rm -rf .git \\\n && rm -f /home/fix.patch /home/test.patch /home/fix-run.sh \\\n /home/test-run.sh /home/prepare.sh /home/check_git_changes.sh \\\n && git init -q \\\n && git config user.name Benchmark \\\n && git config user.email benchmark@example.invalid \\\n && git config gc.auto 0 \\\n && xargs -0 -r git add --force -- < /tmp/codenib-selected-files \\\n && git commit -qm 'chore(benchmark): initialize selected source snapshot' \\\n && test \"$(git rev-parse HEAD^{tree})\" = 012befc5225065c77600560ceacc63453852c152 \\\n && test \"$(git rev-list --all --count)\" = 1 \\\n && test -z \"$(git status --porcelain --untracked-files=all)\" \\\n && python3 --version \\\n && python3 -c 'import hashlib, json, pathlib, subprocess' \\\n && rm /tmp/codenib-selected-files\n\nRUN make \\\n && test \"$(git rev-parse HEAD^{tree})\" = 012befc5225065c77600560ceacc63453852c152 \\\n && test -z \"$(git status --porcelain --untracked-files=all)\"\n", "warm_dockerfile_sha256": "94ebcc0dd7b6100942124b02efbba0fa366cf04e2959000ef6d7ccc752562a6c", "preflights": { "cold": { "protocol_sha256": "a683a579e424ded10b41548927a7c2a074b69835e5b22a477feba1a973f724f5", "passed": true, "all_frozen_runtime_and_input_hashes_match": true, "source_preflight": { "model_calls": 0, "isolation": { "network": "none", "host_mounts": [] }, "image_snapshot": { "commit": "878d69c5de08a180f12cec5b0a242adf6e18a6d8", "tree": "012befc5225065c77600560ceacc63453852c152", "matches_selected_tree": true }, "source_only": { "one_source_commit": true, "reference_artifacts_absent": true }, "controlled_source_read_passed": true, "model_delivery_confirmed": false, "status": "passed", "container_removed": true } }, "warm": { "protocol_sha256": "ea72f3722cf7f170abcf75664a93487512171c8f19765ce21c44c3d8c06599af", "passed": true, "all_frozen_runtime_and_input_hashes_match": true, "source_preflight": { "model_calls": 0, "isolation": { "network": "none", "host_mounts": [] }, "image_snapshot": { "commit": "878d69c5de08a180f12cec5b0a242adf6e18a6d8", "tree": "012befc5225065c77600560ceacc63453852c152", "matches_selected_tree": true }, "source_only": { "one_source_commit": true, "reference_artifacts_absent": true }, "controlled_source_read_passed": true, "model_delivery_confirmed": false, "base_test_exit_code": 2, "base_test_log_sha256": "10170aee115a8c02a3efef94e7df6ef51e0cd2cb66b0ca01e919d4c7f85e5fdf", "missing_link_libraries": [ "crypto", "pcre2-8", "ssl" ], "source_snapshot_unchanged_after_base_test": true, "status": "passed", "container_removed": true } } }, "base_test_analysis": { "scope": "Unmodified source-only image baseline, no model or repair score.", "passed_tests": 1045, "failed_tests": 0, "skipped_tests": 0, "full_parser_sets_match_pinned_baseline": true, "command_exit_code": 2, "missing_link_libraries": [ "crypto", "pcre2-8", "ssl" ], "full_stdlib_suite_verified": false, "native_required_label_gate_unchanged": true, "model_calls": 0 }, "original_native_parser_parity": true, "original_native_gate_passed": true, "model_calls": 0, "new_api_key_charges_usd": 0, "agent_quality_score": null }, "multi_ponyc_development_solver_three_arm": { "protocol": { "scope": "One disclosed C development task, three fresh downstream solver arms. No independent validation, new retrieval, graph-policy quality gate or official submission claim.", "selection": "Selected after observing the late-graph development source contribution: all 200 returned lines remain in the candidate payload and 197 are absent from native. This outcome-informed choice is a one-task development diagnostic, not an unbiased, independent or representative sample; no reference patch or solver score selects a new task.", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "original_inst_id": "ponylang__ponyc-2205", "retrieval_scope": "Frozen late-graph eight-task native/functions/functions_graph 16K delivered contexts. All three retrieval attempts completed. Preserve the failed first-source function attempt separately. Any downstream differences are observational context-arm evidence; they do not isolate graph causality from the other retrieved-source differences.", "common_reader": "Corrected standalone-reader usage feedback at the existing environment boundary; identical instructions in all arms. Source coverage remains partial.", "acceptance": "Report all single-attempt native repair outcomes and confirmed original-source receipts, including any zero or failure. No quality promotion, independent score or population Pass@1 claim.", "inputs": { "native": { "retrieval_sha256": "136fa604d826922131ad32cd886d49a86827424ff3aff8976a26ecca449112f9", "source_tree": "012befc5225065c77600560ceacc63453852c152", "retrieval_config_sha256": "9cbab4f5c5f9ecd44f7791c4e6a3492dea651b95ebe49bd0265b08d4adafe624" }, "functions": { "retrieval_sha256": "22234ffb6e0598a5d9efc58d8b77484ab876f8aaccf8dcef628ad9b76b7454f7", "source_tree": "012befc5225065c77600560ceacc63453852c152", "retrieval_config_sha256": "894462d40876627d407003abb6383e220a1c4cf70652aff4679808caa4048580" }, "functions_graph": { "retrieval_sha256": "4e470a82bec565cbeed30b29416195baad0885c68b47d2df0da7fc2486e5f901", "source_tree": "012befc5225065c77600560ceacc63453852c152", "retrieval_config_sha256": "c457f839495d1b28e14feedab19d5bbc09ea4181a17d5ecb59804f2a5b00ce0e" } }, "arms": [ "native", "functions", "functions_graph" ], "source_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "image_id": "sha256:8e792f86261b3fc70dc9cc5f1c914f001d650b869c9a70754718c28384c4f20f", "native_readiness_sha256": "68345713f27db7372f4b6cdc752997d82a9af6b24a033f6cde2ab6c7fbd9f9d6", "warm_source_preflight_sha256": "5a79e3c4cfa1f9fcf5cf48634310effa22068fcfc00877d0cbf4b1e6853e282b", "readiness_selection_sha256": "24485047023d41e30143de8c143813aca29cb8396cee07ba847ef6222710d81c", "native_image_id": "sha256:3b1afa3e1afd231bd1c93a003f1fbf34b24888a5014cce278f026b13361b5216", "native_dataset_sha256": "2967a9f7095b07c8a586150c6c426951e39353b5982eb0a7ca566c78c424c222", "native_test_timeout_seconds": 600, "native_start_timeout_seconds": 180, "implementation_commit": "6c1585b8d22589462c67a81fecd452071f22883d", "implementation": { "scripts/benchmark_context_agent.py": "5c4f9c451af797558174d19829f7f59f7ee8188b270295d9789ad4bbaee47721", "scripts/benchmark_context_claude.py": "ca2feb3556e5ec46f656f70ba1175334a71a1600eb25c8b8ff59bce7c84c9b6d", "scripts/contextbench_graph_navigation.py": "bf27c49e1688d1f2a9fb8e42240c4e96556baa4f9bc7dec03cfb69a50f287775", "scripts/benchmark_context_serving.py": "6fa40f7ad145938efcb1edceddcad864b2febe0c1c61dda315d9cd5122048fe2", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "1d6d1718d24abaccb46e4f6743ac8033e145628fbc55ceec85d07c0390b39609", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/eval/benchmarks/_contextbench_evaluate.py": "26b82cc9e800759469c52d972ac2f4b5cfda69f052601841f026f05e28cf222c", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99", "scripts/benchmark_context_solver.py": "440a5db2b0ecf749a7b79a6ffd9ebaa2c50c9cf1ba260308bb80edf3dc11676f", "scripts/contextbench_multiswe.py": "eb13f96c8ecb80536e02688dc52ff8ccd9341e0e81c4c20d9979891268f91ea5", "scripts/contextbench_solver_sources.py": "a9d81954376b28752351a2f81494aa5722227e9cf157d51fdc439c60879b0142", "scripts/_contextbench_solver_read.py": "68e2e2e417150ce45e9aa692ee2c6b852b8500df404ee705949fc21b4c6f131c", "mini/__init__.py": "79695f6e5b2f6f90093cd15f08dd9bba2ab099e67487f4ac93bb739e22e19fff", "mini/agents/default.py": "faf4442506b1cdaad886f146182dff98ed740d45536729617efbdcb6d5fb5524", "mini/environments/docker.py": "e3c42a76d0aee03edb0e14e693c963d4fc2e28b678ca36a72f774c90dc89dbd4", "mini/config/extra/swebench.yaml": "81bb47b91311e19b73ff0f8bc670898dc5b807f774ff687e11556038e5878162" }, "driver_sha256": "7fb451c862540fcfd524affe599e491681c3c732f244a7a3a11c64cd0ce8cd7f", "grading_driver_sha256": "257df08c96529ee760f7388cb494deaede86a5108c94678bacc243ebe1edd9e0", "model": "claude-opus-5-5", "effort": "medium", "context_budget": 16384, "max_turns": 20, "max_meter_usd_per_arm": 3, "sdk_meter_is_actual_billing": false, "known_program_sdk_estimate_before": 74.67817670000002, "analysis_driver_sha256": "30a97e80c98c7258094bbe8f131363139a3dd4673628264c0d148a33a445efe9", "preferred_route": "existing_claude_max", "new_api_charges_authorized_ceiling_usd": 100, "standalone_api_key_fallback": false, "stop_rule": "No retry. Stop further inference on process/solver errors or missing terminal metering. Preserve not-run arms and missing usage. Grade only after all solver attempts end, without feedback to any model.", "grading": "Original Multi ponyc script in a separate pinned native image and isolated pinned native harness. One unique task; reference checks are excluded from agent Pass@1. Required-label resolution and known incomplete stdlib linkage are reported separately. No grading or reference contents enter solver inputs." }, "original_analysis": { "protocol_sha256": "ea199d9940f8bc77a2cdaaba917117af1e8349fd2e9bf1cf85e6ec7bf33d26c1", "scope": "One disclosed C development task, three fresh downstream solver arms. No independent validation, new retrieval, graph-policy quality gate or official submission claim.", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "original_inst_id": "ponylang__ponyc-2205", "implementation_commit": "6c1585b8d22589462c67a81fecd452071f22883d", "arms": { "native": { "status": "Submitted", "resolved": false, "model_calls": 9, "sdk_estimate_usd": 2.134897, "known_sdk_estimate_usd": 2.134897, "missing_metering_calls": 0, "actual_models": [ "claude-opus-5-5" ], "sdk_init_api_key_sources": [ "none" ], "all_model_inputs_confirmed": true, "initial_base_source_lines": 1025, "base_source_line_count": 1025, "new_original_source_lines": 0, "source_read_events": 2, "confirmed_original_source_reads": 1, "partial_source_coverage": true, "missing_execution_receipt_indexes": [], "unattributed_delivered_shell_events": 6, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": false, "native_exit_code": 2, "container_removed": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "ponylang/ponyc:pr-2205" ], "completed_ids": [ "ponylang/ponyc:pr-2205" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "ponylang/ponyc:pr-2205" ], "empty_patch_ids": [], "error_ids": [] }, "native_log_sha256": "1183f3f8d9d4abe7cb2186fd8456e158c9b009c271193566264fba3007722462" }, "model_patch_sha256": "4684aafaa64df34c44dcc84b01dea407b6211009ad2728aacbace97786a2d942" }, "functions": { "status": "Submitted", "resolved": false, "model_calls": 9, "sdk_estimate_usd": 2.6849676000000002, "known_sdk_estimate_usd": 2.6849676000000002, "missing_metering_calls": 0, "actual_models": [ "claude-opus-5-5" ], "sdk_init_api_key_sources": [ "none" ], "all_model_inputs_confirmed": true, "initial_base_source_lines": 1486, "base_source_line_count": 1677, "new_original_source_lines": 191, "source_read_events": 3, "confirmed_original_source_reads": 2, "partial_source_coverage": true, "missing_execution_receipt_indexes": [], "unattributed_delivered_shell_events": 5, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": false, "native_exit_code": 2, "container_removed": true, "native_report": { "total_instances": 1, "submitted_instances": 1, "completed_instances": 1, "incomplete_instances": 0, "resolved_instances": 0, "unresolved_instances": 1, "empty_patch_instances": 0, "error_instances": 0, "submitted_ids": [ "ponylang/ponyc:pr-2205" ], "completed_ids": [ "ponylang/ponyc:pr-2205" ], "incomplete_ids": [], "resolved_ids": [], "unresolved_ids": [ "ponylang/ponyc:pr-2205" ], "empty_patch_ids": [], "error_ids": [] }, "native_log_sha256": "9c26026bdbd25f3922af5e4baa3270357c4abc8d3f79e88c8419f09a2e21e181" }, "model_patch_sha256": "dc603c52a4dc2acf5c19d9ae4739653692fee51ff11ba1e1d5571ededca84e17" }, "functions_graph": { "status": "error", "resolved": false, "model_calls": 11, "sdk_estimate_usd": null, "known_sdk_estimate_usd": 2.9738612, "missing_metering_calls": 1, "actual_models": [ "claude-opus-5-5" ], "sdk_init_api_key_sources": [ "none" ], "all_model_inputs_confirmed": true, "initial_base_source_lines": 1393, "base_source_line_count": 1393, "new_original_source_lines": 0, "source_read_events": 2, "confirmed_original_source_reads": 1, "partial_source_coverage": true, "missing_execution_receipt_indexes": [], "unattributed_delivered_shell_events": 8, "solver_container_removed": true, "native_grading": { "status": "no_prediction", "resolved": false, "native_exit_code": null, "container_removed": null, "native_report": null, "native_log_sha256": null }, "model_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } }, "graph_available_in_original_retrieval": true, "independent_quality_score": null, "known_program_sdk_estimate_before": 74.67817670000002, "matched_configs_and_common_prompt": true, "known_sdk_estimate_usd": 7.793725800000001, "sdk_estimate_usd": null, "known_program_sdk_estimate_usd": 82.47190250000003, "new_api_key_charges_usd": 0, "all_frozen_runtime_and_input_hashes_match_after_grading": true, "sdk_meter_is_actual_billing": false }, "metering_reconciliation": { "protocol_sha256": "955db06b47399b76fec281ab182e7c17d77cb48cb2c67e519487b7941ad9694e", "original_solver_protocol_sha256": "ea199d9940f8bc77a2cdaaba917117af1e8349fd2e9bf1cf85e6ec7bf33d26c1", "scope": "Offline metering correction and native-label analysis; original solver failure, protocol, inputs, predictions and reports remain byte-identical. No retry or new model call.", "arms": { "native": { "original_status": "Submitted", "original_sdk_estimate_usd": 2.134897, "original_known_sdk_estimate_usd": 2.134897, "original_missing_metering_calls": 0, "reconciled_known_sdk_estimate_usd": 2.134897, "reconciled_sdk_estimate_usd": 2.134897, "missing_terminal_usage_turns": [], "attempted_model_calls": 9, "sdk_guard_usd": 3, "prediction_exists": true, "patch_files": [ "src/libponyc/codegen/codegen.c", "src/libponyc/codegen/codegen.h", "src/libponyc/codegen/gencall.c" ] }, "functions": { "original_status": "Submitted", "original_sdk_estimate_usd": 2.6849676000000002, "original_known_sdk_estimate_usd": 2.6849676000000002, "original_missing_metering_calls": 0, "reconciled_known_sdk_estimate_usd": 2.6849676000000002, "reconciled_sdk_estimate_usd": 2.6849676000000002, "missing_terminal_usage_turns": [], "attempted_model_calls": 9, "sdk_guard_usd": 3, "prediction_exists": true, "patch_files": [ "src/libponyc/codegen/codegen.c", "src/libponyc/codegen/codegen.h", "src/libponyc/codegen/gencall.c" ] }, "functions_graph": { "original_status": "error", "original_sdk_estimate_usd": null, "original_known_sdk_estimate_usd": 2.9738612, "original_missing_metering_calls": 1, "reconciled_known_sdk_estimate_usd": 3.295746, "reconciled_sdk_estimate_usd": 3.295746, "missing_terminal_usage_turns": [], "attempted_model_calls": 11, "sdk_guard_usd": 3, "prediction_exists": false, "patch_files": [] } }, "recovered_terminal_events": [ { "arm": "functions_graph", "turn": 11, "event_index": 6, "result_event_sha256": "ad6bf427aa4359a13f35974b1e1d45701f3debb3501ba5969ea716727b314d77", "subtype": "error_max_budget_usd", "terminal_reason": "budget_exhausted", "sdk_estimate_usd": 0.3218848, "original_request_sha256": "0ad714518fec98f2a4f030a62537c56646e54d6aa196dc34affa9cdec32cbf83" } ], "sdk_meter_is_actual_billing": false, "new_api_key_charges_usd": 0, "model_calls": 0, "independent_quality_score": null, "reconciled_known_sdk_estimate_usd": 8.1156106, "reconciled_sdk_estimate_usd": 8.1156106, "reconciled_known_program_sdk_estimate_usd": 82.79378730000002, "all_original_inputs_unchanged": true }, "native_test_counts": { "scope": "Pinned native parser on submitted production patches; no native report is invented for an unsubmitted arm.", "arms": { "native": { "status": "graded", "resolved": false, "passed_tests": 1005, "failed_tests": 12, "skipped_tests": 0, "required_regression_passed": 0, "required_regression_failed": 7, "required_preservation_passed": 1005, "required_preservation_failed": 5, "later_stage_runtime_passed": 0, "native_exit_code": 2, "granularity": "test_case", "full_stdlib_suite_verified": false }, "functions": { "status": "graded", "resolved": false, "passed_tests": 1006, "failed_tests": 11, "skipped_tests": 0, "required_regression_passed": 1, "required_regression_failed": 6, "required_preservation_passed": 1005, "required_preservation_failed": 5, "later_stage_runtime_passed": 0, "native_exit_code": 2, "granularity": "test_case", "full_stdlib_suite_verified": false }, "functions_graph": { "status": "no_prediction", "resolved": false, "parsed_test_counts": null } } }, "initial_context_comparison": { "scope": "Source presence in the three frozen initial payloads only; no repair or annotation score.", "arms": { "native": { "unique_base_source_lines": 1025, "files": 22, "tokens": 11992 }, "functions": { "unique_base_source_lines": 1486, "files": 20, "tokens": 16381 }, "functions_graph": { "unique_base_source_lines": 1393, "files": 24, "tokens": 16377 } }, "pairwise": { "functions vs native": { "added": 673, "removed": 212, "graph_genexpr_15_216_present": 202 }, "functions_graph vs native": { "added": 676, "removed": 308, "graph_genexpr_15_216_present": 202 }, "functions_graph vs functions": { "added": 190, "removed": 283, "graph_genexpr_15_216_present": 202 } }, "interpretation": "Function completion already contains the full genexpr.c 15-216 range covering the prior graph response. The graph arm does not uniquely introduce that response relative to the function arm. Contexts also add and remove other source; downstream outcomes cannot isolate a graph-only causal effect.", "retrieval_sha256": { "native": "136fa604d826922131ad32cd886d49a86827424ff3aff8976a26ecca449112f9", "functions": "22234ffb6e0598a5d9efc58d8b77484ab876f8aaccf8dcef628ad9b76b7454f7", "functions_graph": "4e470a82bec565cbeed30b29416195baad0885c68b47d2df0da7fc2486e5f901" }, "source_tree": "012befc5225065c77600560ceacc63453852c152" }, "cleanup": { "owned_containers": 5, "all_owned_containers_removed_and_absent": true, "readiness_input_hashes_match": true }, "conclusion": "One outcome-informed C development task: native/functions both fail required tests and introduce preservation failures; functions_graph has a retained budget error and no prediction. Function completion already covers the earlier graph-returned source range. No comparative repair gain, graph-only causal effect or independent quality claim; no retry or official-500 launch.", "failure_source_diagnostic": { "protocol": { "scope": "Offline diagnosis of the completed disclosed C comparison. No new model, context selection, grading run or independent result.", "inputs": { "verification/contextbench-graph-opportunity-audit-20261017/ponyc_failure.py": "6062f1957d888f724fbcb3037e5c89af835ed486092a2e149e37819b0cdd38e0", "verification/context-serving-100/repos/Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8/src/libponyc/codegen/gencall.c": "b2fbd32dabce1adbbfb0cdbac65533a9f597f1a093f8fdc1d95c5b22a8cfd039", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/protocol.json": "ea199d9940f8bc77a2cdaaba917117af1e8349fd2e9bf1cf85e6ec7bf33d26c1", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/native-test-counts.json": "db37774f6df2efdefbe7ce22d977b8461374e4db29edf21867060fb351b2e91d", "verification/contextbench-multiswe-pony-readiness-20261017/ponylang__ponyc-2205.jsonl": "2967a9f7095b07c8a586150c6c426951e39353b5982eb0a7ca566c78c424c222", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/native/solver.json": "03f9a4500f970e1437e7cc148250105588d9deff760b176ad21fef0ab73660ff", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/native/model-observations.json": "1f7e3ebefacada1da2a06de31d7d0f1116c8a34f728d5c83e8e98b37fa34ccda", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/grade-native/parsed-tests.json": "cceb051224022fabd6d754205ce3857a54482d5338c5d2b7d21de7c2aafb1fe0", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/functions/solver.json": "9b31bbbb0f3d0d8bd5814af72c5e2d5510ebc44e0ca8e304ef3603bcf20953d2", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/functions/model-observations.json": "14d6fab66b5e3766eeb9d841653d61290316d369b7316f7fb907c1b9d9b79c1d", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/grade-functions/parsed-tests.json": "450212ff1b5325d0a1b7effddbe8be5717449f3ba89b461fa14b265797a192a3", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/functions_graph/solver.json": "06224fe5ba00d427832211da1ca285ad8e1e6117eb51f6a506dae8a14bd82ab0", "verification/contextbench-multiswe-pony-solver-three-arm-20261017/functions_graph/model-observations.json": "115de5966d6b0266c13ee89fb2ded3fc76b6fd190ce5bba2d37e7e19f96a3068" }, "annotation_boundary": "Native failed-test identities and test-patch declaration metadata diagnose provenance only. Never include grading-only tests or reference patches in solver/retriever inputs. No reference patch is interpreted by this script.", "source_evidence": "Use only original base source and already-confirmed model inputs to establish delivered compatibility code. Named helper ranges are diagnostic observations, not a new retrieval rule.", "model_calls": 0, "new_api_charges_usd": 0 }, "analysis": { "protocol_sha256": "495511f28b27f83f922e11de2d7b6f50c3e1a039b36cbfe8b877a6838ee80b4f", "scope": "No new repair or quality score; separates source availability, local validation and native grading labels.", "source_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "source_tree_clean": true, "source_sha256": "b2fbd32dabce1adbbfb0cdbac65533a9f597f1a093f8fdc1d95c5b22a8cfd039", "required_preservation_total": 1010, "preservation_labels_present_in_pinned_baseline": 1004, "preservation_labels_not_in_pinned_baseline": 6, "new_codegen_ffi_file_absent_from_base": true, "arms": { "native": { "original_status": "Submitted", "all_model_inputs_confirmed": true, "regions": { "cast_ffi_arg": { "range": [ 1040, 1064 ], "lines_present": 25, "complete": true }, "gen_ffi": { "range": [ 1066, 1156 ], "lines_present": 91, "complete": true } }, "local_base_test_observations": [ { "command_sha256": "c98f7fdc3d458e73211c2b19e65466be3a25f6dd86ddfffbb7b6652edd769d14", "observation_sha256": "ceda3737739a4c01b216ebef08c77edaa7d8005e45be1a18e346ca312b461c30", "reported_pass_counts": [ 67 ], "reported_failure_counts": [] }, { "command_sha256": "f3c4efde822b8b0af81ab41fec81432af60623263ce5d9ab5bbdc1704ca74b18", "observation_sha256": "52e96bdcccffd7b85f6411da919858df95ff27ae6d366a39e7b0d5631760c519", "reported_pass_counts": [ 1005 ], "reported_failure_counts": [] } ], "failed_preservation_tests": [ "CodegenFFITest.PointerPointerCompatibility", "CodegenFFITest.PointerUSizeCompatibility", "CodegenFFITest.ReferencePointerCompatibility", "CodegenFFITest.ReferenceReferenceCompatibility", "CodegenFFITest.ReferenceUSizeCompatibility" ], "failed_preservation_present_in_baseline": [], "failed_preservation_introduced_by_test_patch": [ "CodegenFFITest.PointerPointerCompatibility", "CodegenFFITest.PointerUSizeCompatibility", "CodegenFFITest.ReferencePointerCompatibility", "CodegenFFITest.ReferenceReferenceCompatibility", "CodegenFFITest.ReferenceUSizeCompatibility" ] }, "functions": { "original_status": "Submitted", "all_model_inputs_confirmed": true, "regions": { "cast_ffi_arg": { "range": [ 1040, 1064 ], "lines_present": 25, "complete": true }, "gen_ffi": { "range": [ 1066, 1156 ], "lines_present": 91, "complete": true } }, "local_base_test_observations": [ { "command_sha256": "bcdc23e59c069ac1a3292bef4652df00a4fdb299c59c75f2a399e778a3e9689a", "observation_sha256": "f2b84abaa501c08bcab33194bb5b5b497e636d9789a8ca9fb7233cef7521933e", "reported_pass_counts": [ 108 ], "reported_failure_counts": [] }, { "command_sha256": "f5c5f4b4dbd1f739be2cab09e720f8b96c30cea108c99a9b3a363ae66df64945", "observation_sha256": "c83a805718e31c4941fe6711978e3c208057f24dc072712fd3f3266894438e4b", "reported_pass_counts": [ 1005 ], "reported_failure_counts": [] } ], "failed_preservation_tests": [ "CodegenFFITest.PointerPointerCompatibility", "CodegenFFITest.PointerUSizeCompatibility", "CodegenFFITest.ReferencePointerCompatibility", "CodegenFFITest.ReferenceReferenceCompatibility", "CodegenFFITest.ReferenceUSizeCompatibility" ], "failed_preservation_present_in_baseline": [], "failed_preservation_introduced_by_test_patch": [ "CodegenFFITest.PointerPointerCompatibility", "CodegenFFITest.PointerUSizeCompatibility", "CodegenFFITest.ReferencePointerCompatibility", "CodegenFFITest.ReferenceReferenceCompatibility", "CodegenFFITest.ReferenceUSizeCompatibility" ] }, "functions_graph": { "original_status": "error", "all_model_inputs_confirmed": true, "regions": { "cast_ffi_arg": { "range": [ 1040, 1064 ], "lines_present": 25, "complete": true }, "gen_ffi": { "range": [ 1066, 1156 ], "lines_present": 91, "complete": true } }, "local_base_test_observations": [ { "command_sha256": "c2131e02110273b8aed442a8de953e85a3e2ccc71113bf5a07382ca3904e9e93", "observation_sha256": "c609373a1fb1ff8503f5788a7ac49de13561b601e40400a933d8b34ae5ab1b1b", "reported_pass_counts": [ 96 ], "reported_failure_counts": [] }, { "command_sha256": "67329f325ca192cd89066da6f501094687b2eea1b15441420cb0a5b5efa21abc", "observation_sha256": "dc6efa4d13bd11ada262a8ad064c81c5f9113cd699a39dca76a93a01bba13937", "reported_pass_counts": [ 1005 ], "reported_failure_counts": [] } ] } }, "model_calls": 0, "new_api_charges_usd": 0, "all_frozen_inputs_unchanged": true, "interpretation": "Both submitted patches pass the 1005 base compiler tests they execute but fail five compatibility tests newly supplied by the grader. All three initial contexts already contain the complete argument-casting helper and FFI call implementation. The failures do not establish missing delivery of those source constraints or justify another automatic expansion of the same functions." } } }, "fixed_query_graph_opportunity_audit": { "protocol": { "scope": "Offline source opportunity audit of every original graph query in the disclosed first-source 24 and late-graph eight traces. Eight repeated tasks are not independent observations. No model calls or new online quality outcome.", "queries": "Freeze original anchor, arguments, preceding normalized messages and original compiler artifacts. Reproduce the actual original response. Enumerate bounded candidates for that direction and its opposite; no alternative anchor selection.", "bounds": { "raw_edges_per_direction": 1000, "definition_max_lines": 1500, "file_max_bytes": 2000000, "individual_reply_tokens": 2048 }, "source_contract": "Source candidates require valid pinned ranges and actual source bytes. Union of independently fitting definitions is a non-deliverable upper bound, not a feasible response or retrieved source credit.", "annotation_use": "Freeze all candidate/source observations before inspecting canonical annotations. Measure additions absent from original candidate and matched-native full trajectories. No held-out data, test patches or reference patches enter this graph audit.", "decision": "Only consider a frozen source-ranking hypothesis if individually fitting omitted candidates have novel canonical annotated source across at least two languages. Even that authorizes no model run without a feasible matched replay gate.", "runtime": { "scripts/benchmark_context_agent.py": "5c4f9c451af797558174d19829f7f59f7ee8188b270295d9789ad4bbaee47721", "scripts/benchmark_context_claude.py": "ca2feb3556e5ec46f656f70ba1175334a71a1600eb25c8b8ff59bce7c84c9b6d", "scripts/contextbench_graph_navigation.py": "bf27c49e1688d1f2a9fb8e42240c4e96556baa4f9bc7dec03cfb69a50f287775", "scripts/benchmark_context_serving.py": "6fa40f7ad145938efcb1edceddcad864b2febe0c1c61dda315d9cd5122048fe2", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "1d6d1718d24abaccb46e4f6743ac8033e145628fbc55ceec85d07c0390b39609", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/eval/benchmarks/_contextbench_evaluate.py": "26b82cc9e800759469c52d972ac2f4b5cfda69f052601841f026f05e28cf222c", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "model_calls": 0, "new_api_charges_usd": 0, "input_receipts_sha256": "7e4db37d4a13462de048d42d6607a40d8f75304e8068015378140f03f533345d" }, "analysis": { "protocol_sha256": "6bd6e9e95bce2ec542321859ddd68c0870a1d05cf220f0574e1462c06cfd3fb9", "frozen_observations_sha256": "bedd1dced723668ec1b6efeb6f1e97ad25aac0a81c915b946d62d3b856be4175", "scope": "Source opportunity upper bounds only; no feasible selection or model outcome.", "queries_reproduced": 33, "unique_tasks": 24, "cohort_records": 32, "positive_languages": { "same_direction_fit": [ "rust" ], "opposite_direction_fit": [ "c", "go", "typescript" ], "same_direction_all": [ "cpp", "rust", "typescript" ], "opposite_direction_all": [ "c", "cpp", "go", "typescript" ] }, "ranking_hypothesis_has_multilanguage_opportunity": false, "cases": [ { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 1, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 1, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 10, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 10, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 11, "novel_annotated": 1, "novel_annotated_absent_native": 1 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 11, "novel_annotated": 1, "novel_annotated_absent_native": 1 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 9, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 9, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 56, "novel_annotated": 12, "novel_annotated_absent_native": 10 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 56, "novel_annotated": 12, "novel_annotated_absent_native": 10 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "late_graph_8", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 1, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 1, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 226, "novel_annotated": 58, "novel_annotated_absent_native": 58 }, "same_direction_all": { "novel_physical": 496, "novel_annotated": 26, "novel_annotated_absent_native": 25 }, "opposite_direction_all": { "novel_physical": 226, "novel_annotated": 58, "novel_annotated_absent_native": 58 } } }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 132, "novel_annotated": 21, "novel_annotated_absent_native": 21 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 132, "novel_annotated": 21, "novel_annotated_absent_native": 21 } } }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 189, "novel_annotated": 157, "novel_annotated_absent_native": 157 }, "opposite_direction_all": { "novel_physical": 189, "novel_annotated": 157, "novel_annotated_absent_native": 157 } } }, { "cohort": "late_graph_8", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 23, "novel_annotated": 13, "novel_annotated_absent_native": 1 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 23, "novel_annotated": 13, "novel_annotated_absent_native": 1 } } }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "graph_available": false, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "late_graph_8", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "graph_available": true, "queries": 2, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "graph_available": true, "queries": 1, "invalid_queries": 0, "opportunities": { "same_direction_fit": { "novel_physical": 3, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_fit": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "same_direction_all": { "novel_physical": 3, "novel_annotated": 0, "novel_annotated_absent_native": 0 }, "opposite_direction_all": { "novel_physical": 0, "novel_annotated": 0, "novel_annotated_absent_native": 0 } } } ], "all_input_runtime_source_hashes_match": true, "model_calls": 0, "new_api_charges_usd": 0, "quality_score": null }, "source_receipts_sha256": "aff5ba201d9b183abf74ff47dfee84a9b69a835e017674ce379b6c5f3d187bb0", "conclusion": "Current-direction source-ranking lacks individually fitting novel annotated candidates in two languages; do not implement or run a ranking policy from this audit. Opposite-direction opportunity is only an upper bound, not delivered source or a quality score." }, "opposite_empty_graph_replay": { "protocol": { "scope": "Offline fixed-trace feasibility only. All 24 development tasks and the overlapping late eight are retained; no independent or online score.", "hypothesis": "At most one fallback per task: if an original graph query returns no numbered source and has an available anchor, try the opposite reference direction on the same anchor. Retain source-bearing original replies unchanged. Keep actual and requested directions explicit.", "selection": "Use the existing opposite-direction ordering and definition semantics, with no issue/annotation ranking. Select only definitions already returned by that opposite call; retain whole missing source and skip records whose metadata would exceed the common reply bound. Later actual messages stay fixed.", "budgets": { "reply_tokens": 2048, "reply_definition_limit": "original query limit (default five)", "raw_edges_per_direction": 1000, "automatic_fallbacks_per_task": 1 }, "gate": "Require higher fixed-trace macro Recall and non-decreasing F1 in both cohorts, no original source loss, and new canonical annotated source absent from native in at least two languages. Passing authorizes only implementation and a newly frozen matched development test.", "controls": "Reproduce every original graph response and original prediction; preserve every original source line. Freeze candidate replies before annotation use and official evaluation.", "runtime": { "scripts/benchmark_context_agent.py": "5c4f9c451af797558174d19829f7f59f7ee8188b270295d9789ad4bbaee47721", "scripts/benchmark_context_claude.py": "ca2feb3556e5ec46f656f70ba1175334a71a1600eb25c8b8ff59bce7c84c9b6d", "scripts/contextbench_graph_navigation.py": "bf27c49e1688d1f2a9fb8e42240c4e96556baa4f9bc7dec03cfb69a50f287775", "scripts/benchmark_context_serving.py": "6fa40f7ad145938efcb1edceddcad864b2febe0c1c61dda315d9cd5122048fe2", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "1d6d1718d24abaccb46e4f6743ac8033e145628fbc55ceec85d07c0390b39609", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/eval/benchmarks/_contextbench_evaluate.py": "26b82cc9e800759469c52d972ac2f4b5cfda69f052601841f026f05e28cf222c", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "fc104ad074548a0ddad9ac7507bea2a3a06877d450edf5afc67be7444ada7c52", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "model_calls": 0, "new_api_charges_usd": 0, "input_receipts_sha256": "54aa46e3c141bc8425f69947b484655e046f6202fdc2fb6c4bd5b73b973675fa" }, "analysis": { "protocol_sha256": "61fc2c4b8fec90aa5495507759e6978d9cf05c3565817aaaa067e9ccdfe73691", "scoring_followup_protocol_sha256": "32073d075b4b41002ef469d2fe06ece9aee96b8bfbdd1298989752b3a53f2f55", "scope": "Fixed-trajectory feasibility only, not an online quality result.", "cohort_metrics": { "first_source_24": { "control": { "recall": 0.5401686329564104, "precision": 0.2860920178829965, "f1": 0.3014815297089224 }, "opposite_empty_replay": { "recall": 0.5401686329564104, "precision": 0.2859787573009859, "f1": 0.30142884307517526 } }, "late_graph_8": { "control": { "recall": 0.4162128649381706, "precision": 0.3456469534940558, "f1": 0.3529206038528339 }, "opposite_empty_replay": { "recall": 0.4565880775232046, "precision": 0.3524017662722136, "f1": 0.3659118936254601 } } }, "positive_languages": [ "go", "typescript" ], "advance_to_fresh_development_hypothesis": false, "original_source_loss": 0, "original_predictions_exact": 32, "graph_replies_reproduced": 33, "rows": [ { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "control": { "recall": 0.5684647302904564, "precision": 0.6116071428571429, "f1": 0.589247311827957 }, "opposite_empty_replay": { "recall": 0.5684647302904564, "precision": 0.6088888888888889, "f1": 0.5879828326180258 }, "added_physical": 1, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 1 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "control": { "recall": 0.25510204081632654, "precision": 0.06297229219143577, "f1": 0.101010101010101 }, "opposite_empty_replay": { "recall": 0.25510204081632654, "precision": 0.06297229219143577, "f1": 0.101010101010101 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "control": { "recall": 1.0, "precision": 0.09104938271604938, "f1": 0.1669024045261669 }, "opposite_empty_replay": { "recall": 1.0, "precision": 0.09104938271604938, "f1": 0.1669024045261669 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "control": { "recall": 1.0, "precision": 0.3641025641025641, "f1": 0.5338345864661653 }, "opposite_empty_replay": { "recall": 1.0, "precision": 0.3641025641025641, "f1": 0.5338345864661653 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "control": { "recall": 0.4616508400292184, "precision": 0.36155606407322655, "f1": 0.4055181264035932 }, "opposite_empty_replay": { "recall": 0.4616508400292184, "precision": 0.36155606407322655, "f1": 0.4055181264035932 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "control": { "recall": 0.5460526315789473, "precision": 0.41089108910891087, "f1": 0.4689265536723164 }, "opposite_empty_replay": { "recall": 0.5460526315789473, "precision": 0.41089108910891087, "f1": 0.4689265536723164 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "control": { "recall": 0.5362694300518135, "precision": 0.552, "f1": 0.5440210249671484 }, "opposite_empty_replay": { "recall": 0.5362694300518135, "precision": 0.552, "f1": 0.5440210249671484 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "control": { "recall": 0.4875, "precision": 0.45792563600782776, "f1": 0.47225025227043393 }, "opposite_empty_replay": { "recall": 0.4875, "precision": 0.45792563600782776, "f1": 0.47225025227043393 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "control": { "recall": 0.19640971488912354, "precision": 0.3089700996677741, "f1": 0.24015493867010976 }, "opposite_empty_replay": { "recall": 0.19640971488912354, "precision": 0.3089700996677741, "f1": 0.24015493867010976 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "control": { "recall": 1.0, "precision": 0.047619047619047616, "f1": 0.0909090909090909 }, "opposite_empty_replay": { "recall": 1.0, "precision": 0.047619047619047616, "f1": 0.0909090909090909 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "control": { "recall": 0.2495274102079395, "precision": 0.5076923076923077, "f1": 0.3346007604562738 }, "opposite_empty_replay": { "recall": 0.2495274102079395, "precision": 0.5076923076923077, "f1": 0.3346007604562738 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "control": { "recall": 0.44, "precision": 0.16356877323420074, "f1": 0.2384823848238482 }, "opposite_empty_replay": { "recall": 0.44, "precision": 0.16356877323420074, "f1": 0.2384823848238482 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "control": { "recall": 0.17716535433070865, "precision": 0.14551333872271624, "f1": 0.15978695073235685 }, "opposite_empty_replay": { "recall": 0.17716535433070865, "precision": 0.14551333872271624, "f1": 0.15978695073235685 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "control": { "recall": 0.36316695352839934, "precision": 0.18672566371681415, "f1": 0.24663939216832265 }, "opposite_empty_replay": { "recall": 0.36316695352839934, "precision": 0.18672566371681415, "f1": 0.24663939216832265 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "control": { "recall": 0.5645645645645646, "precision": 0.1551155115511551, "f1": 0.24336569579288025 }, "opposite_empty_replay": { "recall": 0.5645645645645646, "precision": 0.1551155115511551, "f1": 0.24336569579288025 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "control": { "recall": 0.6521739130434783, "precision": 0.02697841726618705, "f1": 0.05181347150259066 }, "opposite_empty_replay": { "recall": 0.6521739130434783, "precision": 0.02697841726618705, "f1": 0.05181347150259066 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "control": { "recall": 0.8773234200743495, "precision": 0.36932707355242567, "f1": 0.5198237885462554 }, "opposite_empty_replay": { "recall": 0.8773234200743495, "precision": 0.36932707355242567, "f1": 0.5198237885462554 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "control": { "recall": 0.7248677248677249, "precision": 0.1282371294851794, "f1": 0.21792152704135737 }, "opposite_empty_replay": { "recall": 0.7248677248677249, "precision": 0.1282371294851794, "f1": 0.21792152704135737 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "control": { "recall": 0.44876325088339225, "precision": 0.8639455782312925, "f1": 0.5906976744186048 }, "opposite_empty_replay": { "recall": 0.44876325088339225, "precision": 0.8639455782312925, "f1": 0.5906976744186048 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "control": { "recall": 1.0, "precision": 0.036458333333333336, "f1": 0.07035175879396986 }, "opposite_empty_replay": { "recall": 1.0, "precision": 0.036458333333333336, "f1": 0.07035175879396986 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 1 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "control": { "recall": 0.375, "precision": 0.05647058823529412, "f1": 0.098159509202454 }, "opposite_empty_replay": { "recall": 0.375, "precision": 0.05647058823529412, "f1": 0.098159509202454 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "control": { "recall": 0.22832369942196531, "precision": 0.11318051575931232, "f1": 0.15134099616858238 }, "opposite_empty_replay": { "recall": 0.22832369942196531, "precision": 0.11318051575931232, "f1": 0.15134099616858238 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "control": { "recall": 0.2270220588235294, "precision": 0.5678160919540229, "f1": 0.3243598161523309 }, "opposite_empty_replay": { "recall": 0.2270220588235294, "precision": 0.5678160919540229, "f1": 0.3243598161523309 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "control": { "recall": 0.5846994535519126, "precision": 0.27648578811369506, "f1": 0.375438596491228 }, "opposite_empty_replay": { "recall": 0.5846994535519126, "precision": 0.27648578811369506, "f1": 0.375438596491228 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "late_graph_8", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "control": { "recall": 0.5684647302904564, "precision": 0.7653631284916201, "f1": 0.6523809523809525 }, "opposite_empty_replay": { "recall": 0.5684647302904564, "precision": 0.7611111111111111, "f1": 0.6508313539192399 }, "added_physical": 1, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 1 }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "control": { "recall": 0.25510204081632654, "precision": 0.05107252298263534, "f1": 0.0851063829787234 }, "opposite_empty_replay": { "recall": 0.5510204081632653, "precision": 0.10315186246418338, "f1": 0.1737731295253419 }, "added_physical": 68, "added_annotated": 58, "added_annotated_absent_native": 58, "changed_queries": 1 }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "control": { "recall": 0.47552958363769177, "precision": 0.3559322033898305, "f1": 0.4071294559099437 }, "opposite_empty_replay": { "recall": 0.47552958363769177, "precision": 0.3559322033898305, "f1": 0.4071294559099437 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "control": { "recall": 0.5362694300518135, "precision": 0.5376623376623376, "f1": 0.5369649805447471 }, "opposite_empty_replay": { "recall": 0.5362694300518135, "precision": 0.5376623376623376, "f1": 0.5369649805447471 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "late_graph_8", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "control": { "recall": 0.4583333333333333, "precision": 0.47619047619047616, "f1": 0.4670912951167728 }, "opposite_empty_replay": { "recall": 0.48541666666666666, "precision": 0.4824016563146998, "f1": 0.48390446521287644 }, "added_physical": 21, "added_annotated": 13, "added_annotated_absent_native": 1, "changed_queries": 1 }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "control": { "recall": 0.1816261879619852, "precision": 0.2551928783382789, "f1": 0.2122146822948797 }, "opposite_empty_replay": { "recall": 0.1816261879619852, "precision": 0.2551928783382789, "f1": 0.2122146822948797 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "late_graph_8", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "control": { "recall": 0.3648881239242685, "precision": 0.18275862068965518, "f1": 0.24353819643882826 }, "opposite_empty_replay": { "recall": 0.3648881239242685, "precision": 0.18275862068965518, "f1": 0.24353819643882826 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 }, { "cohort": "late_graph_8", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "control": { "recall": 0.4894894894894895, "precision": 0.14100346020761245, "f1": 0.2189388851578241 }, "opposite_empty_replay": { "recall": 0.4894894894894895, "precision": 0.14100346020761245, "f1": 0.2189388851578241 }, "added_physical": 0, "added_annotated": 0, "added_annotated_absent_native": 0, "changed_queries": 0 } ], "all_input_runtime_hashes_match": true, "model_calls": 0, "new_api_charges_usd": 0 }, "scoring_followup_protocol": { "scope": "Score the immutable candidate responses already frozen by the original protocol. Original source-generation phase is not rerun; the missing evaluation directory is the only corrected setup.", "original_protocol_sha256": "61fc2c4b8fec90aa5495507759e6978d9cf05c3565817aaaa067e9ccdfe73691", "frozen_predictions_sha256": "df3b0c5850a13d98606de1bb7fff9c068b64927ddcd5804a081bba98d42a4a54", "failed_scoring_log_sha256": "1ce8228affa14eb8bcdfe88da77548d80e465a5f1f6fc96446ec2e30c16f20c1", "driver_sha256": "a73085bf8de154c57d68a9de6fbd9eb54bf0e58090cf37ed99f4dfd92a9e2c40", "selection_or_gate_changes": false, "model_calls": 0, "new_api_charges_usd": 0 }, "original_scoring_failure": "Candidate generation and freezing completed; the first evaluator call failed because its output directory was absent. Only scoring was rerun after creating a separate output directory, with no changed prediction, gate or selection.", "conclusion": "The general empty-response opposite-hop hypothesis fails its predeclared two-cohort gate and is not implemented or run online. The late-eight improvement is exploratory fixed-trajectory evidence only; a separate late-only hypothesis needs fresh matched development controls." }, "late_opposite_followup_selection": { "scope": "Prospective metadata-only selection for a separate late-stage fallback development hypothesis. These eight tasks are already disclosed development data, not untouched or independent validation. No runtime, model or paid batch is started by this selection.", "selection_rule": "First remaining task per language in the frozen 24-task file order, excluding every task in the completed late-eight cohort. No graph-availability, repair score, annotation or replay gain filters.", "source_selection_sha256": "122a70d4c61abcea5c27f87781fc204db768d933dd0d3bb7516e20117866f513", "excluded_selection_sha256": "3f271fffc1394591c485e4eee5e2501e39e07e0e6146263392ee8ab0e889f568", "selected_sha256": "fa7fb9cd3cc93b39387e936df67d3da0dd1ccc68c577305800508488cec5ec06", "cases": [ { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript" }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python" }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c" }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp" }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust" }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java" }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go" } ], "scope_reason": "The general empty-response fallback failed its two-cohort feasibility gate. Positive late-stage fixed-trace evidence is exploratory; a separate late-only policy must be implemented, frozen and tested with fresh matched controls on this different development selection before any promotion.", "model_calls": 0, "new_api_charges_usd": 0, "current_followup": "late_opposite_followup_development", "current_status": "Implemented and evaluated with frozen fresh three-arm controls; see the separate result for its gates. Original selection receipt remains unchanged.", "selection_time_open_gates": [ "The general fallback remains rejected by its failed replay; this selection establishes no runtime readiness or quality gain.", "A new late-only hypothesis requires explicit runtime and source budgets frozen before fresh native/functions/graph calls.", "Independent 32 remains sealed; broader independent quality and full-500 gates stay unchanged." ], "open_gates": [ "The completed matched development comparison fails its predeclared mechanism gates; no fallback source or graph annotations absent from native are delivered.", "Source-bound compiler coverage and missing-anchor diagnosis precede another online hypothesis.", "Independent 32 stays sealed; new independent quality and full-500 requirements remain open." ] }, "late_opposite_followup_development": { "scope": "Prospective online three-arm development follow-up on eight other already disclosed tasks. Distinct late-only hypothesis; failed general replay and sealed independent-32 results remain unchanged.", "protocol_sha256": "953888b85c63227641b90a92181fb1ece67d3bd246e102d0a6fc8a4b2f217f1a", "implementation_commit": "cf86e0a76f455a6dcf0e4a2ffa76d165329545de", "selection_sha256": "fa7fb9cd3cc93b39387e936df67d3da0dd1ccc68c577305800508488cec5ec06", "selection_rule": "First remaining task per language in the frozen 24-task file order, excluding every task in the completed late-eight cohort. No graph-availability, repair score, annotation or replay gain filters.", "n": 8, "available_graphs": 5, "model_requested": "claude-opus-5-5", "actual_models": [ "claude-opus-5-5" ], "parameters": { "max_turns": 20, "case_timeout_seconds": 600, "completion_tokens": 2048, "completion_source_policy": "unseen", "graph_tokens": 2048, "graph_source_policy": "unseen", "graph_query_policy": "late_issue_reference_hop", "graph_empty_fallback": "Candidate only: when the requested direction returns no source at an available anchor, try the opposite once with the same observed-source snapshot. Actual and requested directions are explicit. No fallback on missing/unavailable anchors; no recursion. Existing source ordering, definition limit and 2048-token/16000-character reply cap remain in force. Each direction scans at most 1000 raw edges. No extra source is appended to a nonempty primary reply.", "graph_query_limit": "Prompt limit of two calls per task, audited as a gate, not a hard executor quota. Each call performs at most two bounded directional scans. This stateless per-call policy differs from the rejected at-most-one-fallback-per-task general replay.", "case_meter_usd": 3, "max_meter_usd_per_arm": 6, "total_sdk_meter_stop_threshold_usd": 18, "meter_guard": "Observed SDK estimates, not invoiced charges. A final in-flight model turn may cross a guard; no retries or API fallback.", "billing_route": "existing_claude_max", "additional_API_spending_ceiling_usd": 100, "automatic_API_fallback": false }, "primary_contrast": "functions_and_late_empty_graph_over_native", "contrasts": { "primary": { "contrast": "functions_and_late_empty_graph_over_native", "metrics": { "recall": { "baseline": 0.7789736557074384, "candidate": 0.8056646767366582, "difference": 0.026691021029219858, "paired_bootstrap_95_interval": [ -0.14864192617649985, 0.20233452801286123 ] }, "precision": { "baseline": 0.3054811929734058, "candidate": 0.31515054386714864, "difference": 0.009669350893742833, "paired_bootstrap_95_interval": [ -0.05282907284345069, 0.06654769708844335 ] }, "f1": { "baseline": 0.33670104886762275, "candidate": 0.37550682313519373, "difference": 0.03880577426757098, "paired_bootstrap_95_interval": [ -0.07605019634298325, 0.1489099739291344 ] } }, "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" } }, "functions": { "contrast": "online_function_completion", "metrics": { "recall": { "baseline": 0.7789736557074384, "candidate": 0.7213663599096929, "difference": -0.05760729579774551, "paired_bootstrap_95_interval": [ -0.189694069916407, 0.06521739130434784 ] }, "precision": { "baseline": 0.3054811929734058, "candidate": 0.31545129194846777, "difference": 0.009970098975061958, "paired_bootstrap_95_interval": [ -0.05621835206308286, 0.07965550865636248 ] }, "f1": { "baseline": 0.33670104886762275, "candidate": 0.3294508707937038, "difference": -0.007250178073919006, "paired_bootstrap_95_interval": [ -0.08329588695500724, 0.07238575013211354 ] } }, "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" } }, "graph": { "contrast": "late_empty_graph_over_function_completion", "metrics": { "recall": { "baseline": 0.7213663599096929, "candidate": 0.8056646767366582, "difference": 0.08429831682696537, "paired_bootstrap_95_interval": [ -0.0929872234935164, 0.26229860661158133 ] }, "precision": { "baseline": 0.31545129194846777, "candidate": 0.31515054386714864, "difference": -0.00030074808131912503, "paired_bootstrap_95_interval": [ -0.07200194779471744, 0.07392217328941865 ] }, "f1": { "baseline": 0.3294508707937038, "candidate": 0.37550682313519373, "difference": 0.046055952341489986, "paired_bootstrap_95_interval": [ -0.061478213982362265, 0.17119309270964173 ] } }, "bootstrap": { "draws": 10000, "seed": 20261016, "unit": "task" } } }, "arms": { "native": { "metrics": { "recall": 0.7789736557074384, "precision": 0.3054811929734058, "f1": 0.33670104886762275, "official_valid": 8 }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.6609501999999998, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 8 }, "cases": 8 }, "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "graph_empty_fallback": false }, "functions": { "metrics": { "recall": 0.7213663599096929, "precision": 0.31545129194846777, "f1": 0.3294508707937038, "official_valid": 8 }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.7132774000000002, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 8 }, "cases": 8 }, "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "graph_empty_fallback": false }, "functions_graph": { "metrics": { "recall": 0.8056646767366582, "precision": 0.31515054386714864, "f1": 0.37550682313519373, "official_valid": 8 }, "usage": { "billing_route": "existing_claude_subscription", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.7130295, "missing_metering_cases": 0, "cost_scope": "SDK metering estimate is not a paid API receipt. Subscription limits and any account-level extra usage remain applicable.", "statuses": { "ok": 8 }, "cases": 8 }, "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "graph_empty_fallback": true } }, "gates": { "opposite_fallback_returned_source": false, "every_graph_reply_respects_budget": true, "subscription_route_confirmed": true, "all_selected_tasks_finish_and_score": true, "all_usage_metered": true, "every_available_graph_called": true, "at_most_two_graph_calls_per_task": true, "graph_anchors_were_previously_observed_after_two_source_replies": true, "at_least_two_languages_receive_annotated_graph_source_absent_from_native": false, "primary_recall_point_increased": true, "primary_f1_point_not_lower": true }, "passed": false, "graph_adoption": [ { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 }, "queries": [ { "source_replies_before": 6, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "graph_unavailable": 1 }, "fallback_source_records": 0, "reply_tokens": 14, "budget_valid": true, "actual_directions": [], "requested_directions": [] } ] }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "graph_available": true, "new_physical_lines": 37, "new_annotated_lines": 0, "new_physical_absent_from_native": 37, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 37, "source_returned": 2 }, "queries": [ { "source_replies_before": 2, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 37, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "source_returned": 2 }, "fallback_source_records": 0, "reply_tokens": 616, "budget_valid": true, "actual_directions": [ "outgoing" ], "requested_directions": [] } ] }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 }, "queries": [ { "source_replies_before": 7, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "graph_unavailable": 1 }, "fallback_source_records": 0, "reply_tokens": 14, "budget_valid": true, "actual_directions": [], "requested_directions": [] } ] }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_definition_at_line": 1 }, "queries": [ { "source_replies_before": 8, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "no_definition_at_line": 1 }, "fallback_source_records": 0, "reply_tokens": 15, "budget_valid": true, "actual_directions": [], "requested_directions": [] } ] }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "graph_available": true, "new_physical_lines": 5, "new_annotated_lines": 5, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 5, "source_returned": 1 }, "queries": [ { "source_replies_before": 7, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 5, "new_annotated_lines": 5, "new_annotated_absent_from_native": 0, "statuses": { "source_returned": 1 }, "fallback_source_records": 0, "reply_tokens": 139, "budget_valid": true, "actual_directions": [ "incoming" ], "requested_directions": [] } ] }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_returned_neighbors": 1 }, "queries": [ { "source_replies_before": 6, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "no_returned_neighbors": 1 }, "fallback_source_records": 0, "reply_tokens": 15, "budget_valid": true, "actual_directions": [], "requested_directions": [] } ] }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "graph_available": false, "new_physical_lines": 0, "new_annotated_lines": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "graph_unavailable": 1 }, "queries": [ { "source_replies_before": 5, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "graph_unavailable": 1 }, "fallback_source_records": 0, "reply_tokens": 14, "budget_valid": true, "actual_directions": [], "requested_directions": [] } ] }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "graph_available": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_physical_absent_from_native": 0, "new_annotated_absent_from_native": 0, "counts": { "calls": 1, "confirmed_responses": 1, "returned_physical_lines": 0, "no_unseen_neighbors": 1 }, "queries": [ { "source_replies_before": 3, "anchor_previously_observed": true, "confirmed": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_from_native": 0, "statuses": { "no_unseen_neighbors": 1 }, "fallback_source_records": 0, "reply_tokens": 15, "budget_valid": true, "actual_directions": [], "requested_directions": [] } ] } ], "sdk_reported_estimate_usd": 5.0872571, "sdk_api_key_sources": { "none": 24 }, "all_frozen_runtime_and_inputs_match": true, "annotation_sha256": "680fae306f038de45ff8af9dd8113f4ca9a3bde4f318703f47fa682d2e88af21", "confounding": "Primary contrast changes function completion, late graph instructions and explicit empty fallback. Graph/functions contrast measures that graph package. Neither isolates fallback alone; earlier cohorts and fixed-trace gains are descriptive only.", "source_delivery": "Only verified numbered replies followed by a later assistant response count as delivered. Canonical annotation paths are applied after inference is terminal. Novelty compares the candidate prefix and full matched native trajectory. No grading content or reference patch enters prompts.", "no_model_retries": true, "official_500_started": false, "retrieval_cost_projection": { "scope": "Mechanical retrieval-only SDK-estimate projection from eight disclosed tasks per arm, not an invoice, an approved run budget or a representative 500-task forecast. Excludes solver/native grading, retries, quota delays and unmeasured account charges.", "per_arm_500_sdk_estimate_usd": { "native": 103.80938749999999, "functions": 107.07983750000001, "functions_graph": 107.06434374999999 }, "native_plus_candidate_500_each_sdk_estimate_usd": 210.87373125, "three_arms_500_each_sdk_estimate_usd": 317.95356875 }, "fixed_query_behavior_replay": { "protocol_sha256": "0296d9710ff98601d48be95e0a57af1513cfa01e513ab249cf0883c0a13d89d7", "queries": 8, "all_actual_replies_reproduced": true, "disabled_fallback_identical_replies": 8, "eligible_opposite_scans": 2, "rows": [ { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "available_graph": false, "primary_statuses": [ "graph_unavailable" ], "eligible_for_opposite_scan": false, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "8e21ece9901721a89a48e66ccda651e4e6961a3e41106df9d0ac5dd2290d4e9c" }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "available_graph": true, "primary_statuses": [ "source_returned" ], "eligible_for_opposite_scan": false, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "fedbd037e78365e38cb59967d9760694c1766115455903377ea1167688150b44" }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "available_graph": false, "primary_statuses": [ "graph_unavailable" ], "eligible_for_opposite_scan": false, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "8e21ece9901721a89a48e66ccda651e4e6961a3e41106df9d0ac5dd2290d4e9c" }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "available_graph": true, "primary_statuses": [ "no_definition_at_line" ], "eligible_for_opposite_scan": false, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "2ee00ea43aa2ea03f409e8075daaf8dc04b5e504c7bf819d5452e9fcdc745919" }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "available_graph": true, "primary_statuses": [ "source_returned" ], "eligible_for_opposite_scan": false, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "4dbb208474b9b8c2ade2799514df3103a07d6d330e2f6aea8cb80c039565ef78" }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "available_graph": true, "primary_statuses": [ "no_returned_neighbors" ], "eligible_for_opposite_scan": true, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "6fb6b9f0faf50714866522f092ed6f1ceed8e0feabd280292ea1d083a7c062ea" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "available_graph": false, "primary_statuses": [ "graph_unavailable" ], "eligible_for_opposite_scan": false, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "8e21ece9901721a89a48e66ccda651e4e6961a3e41106df9d0ac5dd2290d4e9c" }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "available_graph": true, "primary_statuses": [ "no_unseen_neighbors" ], "eligible_for_opposite_scan": true, "actual_response_reproduced": true, "disabled_fallback_reply_byte_identical": true, "reply_sha256": "7559f149f8fd514a3a4df43a8f2b9b937cd61ab376449a3e1f76de8040afa338" } ], "all_runtime_and_inputs_match": true, "model_calls": 0, "sdk_estimate_usd": 0, "inference": "The response/source effect of the optional fallback was not exercised on these saved anchors. Online point-score differences cannot be attributed to extra fallback source; the changed tool guidance and agent trajectory remain part of the contrast." } }, "compiler_coverage_followup": { "scope": "Zero-model compiler/source diagnostics on disclosed development tasks. Fixed queries are not counterfactual agent trajectories or revised official scores.", "runtime_commit": "1edb2a686044209f718b682901f0bd9f63718717", "environment_runtime_commit": "de691eecff0b959fdd78562469eb0d1ce1f02a85", "protocol_sha256": { "initial_index": "58a2d48ad0d429a2d205343b64cd6678e112fe774f2c48a926cd0e3df67b7c91", "environment_followup": "c2d8dda569ca270a00f3a3868112d88c4bf5d0428103b354805aa097c7d22626", "tracked_source_followup": "b2c42effc761ad7cc01bcdbde90df5368f649fd8f80c5dcd2ae93595390fb763", "cpp_repair_replay": "b0f86bba1923a59b5d7dd64f0067299c25e040de8d76b0bd76224d852289e3bc", "combined_fixed_queries": "40bc54186d31d819d67c2e2503a4d1f1555fde27e99324c60f06432a90e5eafe" }, "analysis_sha256": "4a1de3837c9157700ff269717f9b35e54f7c103e22583109ebad2d21679d5044", "implementation": "Follow C/C++ callable declarators and classify compiler ConversionFunction kind 24 as method in both Python and native query paths. Native normalization profile advances to v4; cached materialized graphs require regeneration.", "cpp_rebuilt": [ { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "nodes": 7935, "edges": 29266, "changed_nodes": 1123, "changed_ranges": 1089, "conversion_kind_changes": 81, "compiler_identity_file_selection_exact": true, "edges_exact": true }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "nodes": 2900, "edges": 78577, "changed_nodes": 480, "changed_ranges": 477, "conversion_kind_changes": 7, "compiler_identity_file_selection_exact": true, "edges_exact": true }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "nodes": 3151, "edges": 36891, "changed_nodes": 566, "changed_ranges": 561, "conversion_kind_changes": 13, "compiler_identity_file_selection_exact": true, "edges_exact": true } ], "typescript": { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "available": true, "wall_s": 268.1705442479579, "nodes": 212904, "edges": 467440, "graph_sha256": "50969b963a10c5830ba6951d669402211abe59c090cbbf8c567485e3cf4c8091", "source_unchanged": true, "preparation": "Existing native decoder, read-only dependency policy, original 300-second bound. This is readiness, not a causal speed benchmark." }, "java": { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "available": true, "wall_s": 22.276822463958524, "nodes": 4385, "edges": 31158, "graph_sha256": "d72e723f92a5fc20b185bec42dae4a9f91cb29db2445c07a47bd95f5f10c3142", "tracked_files": 345, "tracked_bytes_exact": true, "tracked_byte_manifest_sha256": "dcae9dcb8b9fd36e9d73851d5e849c9e0d7184d53fc32a2f649369ffce7f8c1d", "preparation": "Existing managed scip-java, explicit Maven selection, java.version=8 command-line override and test-compile. No tracked POM/source edits." }, "c": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "available": false, "wall_s": 57.340631401049905, "original_source_unchanged": true, "root_tracked_bytes_exact": true, "submodules": [ { "path": "modules/oniguruma", "expected_head": "d2f1a14ced5d5d461acac0da0d477ab240a7ab5f", "initialized": true, "actual_head": "d2f1a14ced5d5d461acac0da0d477ab240a7ab5f", "tracked_changes": "M\tINSTALL\nM\tcompile\nM\tconfig.guess\nM\tconfig.sub\nM\tdepcomp\nM\tmissing\nM\ttest-driver\n", "status": " M INSTALL\n M compile\n M config.guess\n M config.sub\n M depcomp\n M missing\n M test-driver\n", "ignored_output_count": 257 } ], "reason": "Private build rewrites seven tracked oniguruma Autotools helper files. Guard rejection is correct; ignored outputs alone do not explain it. No graph accepted and no source guard relaxed." }, "retained_prior_attempts": [ "A follow-up catches three missed abstract-reference conversion declarators. All three regressions fail before the follow-up; the final 198-test unit tier and three integrations pass, followed by a separately frozen graph/reply replay.", "Java managed-tool probe needs explicit build-tool selection.", "Explicit Maven without a Java version override fails compilation; the diagnostic snapshot audit also rejects ignored outputs despite an empty tracked diff.", "The first C build failure lacked recursive source details; a separate frozen probe records the actual tracked submodule changes.", "The first C++ replay assertion also compared display labels. It rejects the expected () suffix when a conversion symbol becomes a method. v2 permits only that kind-24 display normalization; IDs, file identity, selections and all edges remain exact." ], "fixed_query_replay": { "queries": 8, "original_replies_reproduced": 8, "available_graphs_before": 5, "available_graphs_after": 7, "additional_physical_lines": 146, "additional_annotated_lines": 0, "novel_annotated_languages": [], "online_novelty_gate_passed": false, "rows": [ { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "response_identical": false, "new_physical_lines": 108, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 108, "added_annotated_over_control": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "response_identical": true, "new_physical_lines": 37, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 0, "added_annotated_over_control": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "response_identical": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 0, "added_annotated_over_control": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "response_identical": false, "new_physical_lines": 1, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 1, "added_annotated_over_control": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "response_identical": true, "new_physical_lines": 5, "new_annotated_lines": 5, "new_annotated_absent_native": 0, "added_over_control": 0, "added_annotated_over_control": 0 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "response_identical": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 0, "added_annotated_over_control": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "response_identical": false, "new_physical_lines": 37, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 37, "added_annotated_over_control": 0 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "response_identical": true, "new_physical_lines": 0, "new_annotated_lines": 0, "new_annotated_absent_native": 0, "added_over_control": 0, "added_annotated_over_control": 0 } ] }, "validation": { "pre_fix_regression_failures": 13, "targeted_unit_passed": 198, "cpp_integration_passed": 3, "compiled_native_binding_rebuilt": true, "pinned_implementation_hooks_passed": true }, "all_inputs_match": true, "model_calls": 0, "sdk_estimate_usd": 0, "new_API_charges_usd": 0, "official_scores_unchanged": true, "official_500_started": false, "decision": "Keep the general compiler correctness fixes. Availability and added source do not establish relevance: the unchanged eight-query replay adds 146 physical lines and zero annotated lines. No new online batch is justified by this diagnostic; independent-32 and earlier failed gates remain intact." }, "sparse_discovery_followup": { "scope": "Zero-model discovery feasibility on all 24 disclosed tasks, using the authenticated existing bounded source corpus and current production BM25. No graph or labels select candidates. This is not an online tool trial or an independent result.", "runtime_commit": "c893aefd6fc987c36d5fe94cdd6f7f99d44228c9", "protocol_sha256": "b39b7a80bc2d55131f063f4a0de4aea928c632578a96a89358493fa1c08ef72d", "candidate_receipts_sha256": "9bbe10258afaa8b3f9e6ab41a413e5b57b3e967605277608ea7de1a113caae63", "analysis_sha256": "7799e62f6e3a6225e1821b6bdd485ccbd41c99d513dd26eec2c5062931fe68cc", "completed_tasks": 24, "corpus_nodes": 352494, "query": "Original issue prefix capped at the production 16000-character search limit; one query, no rewrites.", "retrieval": { "candidates": 40, "filter_test": false, "packed_nodes": 10, "reply_tokens": 2048, "content": "Existing source-only packer, whole lines, no repeated lines. All corpus source checked before retrieval." }, "packages": { "rank-bm25": "0.2.2", "tiktoken": "0.9.0", "numpy": "1.26.4" }, "annotation_contract": "Freeze every candidate pool and bounded reply before reading canonical annotations or native source sets. Preserve official paths; unavailable annotation source receives no credit.", "primary_static_addition_gate": { "new_annotated_source_absent_from_native_in_at_least_two_languages": true, "positive_macro_recall_in_both_cohorts": true, "nondecreasing_macro_f1_in_both_cohorts": true }, "optional_discovery_opportunity_gate": "At least three languages have annotated source absent from full native among the first five results. This gate justifies inspecting a selective discovery tool, not automatic source injection or a quality claim.", "static_addition_gate_passed": false, "optional_discovery_opportunity_gate_passed": false, "summaries": [ { "cohort": "first_source_24", "tasks": 24, "metrics": { "baseline": { "recall": 0.5335256710568391, "precision": 0.3162405037274296, "f1": 0.31176405157915116 }, "static_addition": { "recall": 0.535697144630244, "precision": 0.2305213243523746, "f1": 0.2662291669102204 } }, "added_annotated_lines": 38, "novel_annotated_languages": [ "c", "cpp" ], "top5_novel_annotated_languages": [ "c", "cpp" ] }, { "cohort": "late_opposite_8", "tasks": 8, "metrics": { "baseline": { "recall": 0.7789736557074383, "precision": 0.3054811929734058, "f1": 0.3367010488676228 }, "static_addition": { "recall": 0.7789736557074383, "precision": 0.24185985641402824, "f1": 0.29514328845435506 } }, "added_annotated_lines": 0, "novel_annotated_languages": [], "top5_novel_annotated_languages": [] } ], "rows": [ { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "baseline": { "recall": 0.3651452282157676, "precision": 0.704, "f1": 0.4808743169398907 }, "static_addition": { "recall": 0.3651452282157676, "precision": 0.2913907284768212, "f1": 0.3241252302025783 }, "added_physical_lines": 177, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "baseline": { "recall": 0.40816326530612246, "precision": 0.10854816824966079, "f1": 0.1714898177920686 }, "static_addition": { "recall": 0.40816326530612246, "precision": 0.0866738894907909, "f1": 0.14298480786416443 }, "added_physical_lines": 186, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "static_addition": { "recall": 1.0, "precision": 0.05432780847145488, "f1": 0.10305676855895196 }, "added_physical_lines": 172, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.3169642857142857, "f1": 0.48135593220338985 }, "static_addition": { "recall": 1.0, "precision": 0.21385542168674698, "f1": 0.3523573200992556 }, "added_physical_lines": 108, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "baseline": { "recall": 0.36303871439006574, "precision": 0.3066008636644047, "f1": 0.33244147157190634 }, "static_addition": { "recall": 0.3754565376186998, "precision": 0.2903954802259887, "f1": 0.32749283211213764 }, "added_physical_lines": 149, "added_annotated_lines": 17, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 17, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "status": "ok", "baseline": { "recall": 0.8706140350877193, "precision": 0.5251322751322751, "f1": 0.6551155115511551 }, "static_addition": { "recall": 0.8706140350877193, "precision": 0.45423340961098396, "f1": 0.5969924812030075 }, "added_physical_lines": 118, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "baseline": { "recall": 0.5362694300518135, "precision": 0.5036496350364964, "f1": 0.5194479297365119 }, "static_addition": { "recall": 0.5362694300518135, "precision": 0.3683274021352313, "f1": 0.43670886075949367 }, "added_physical_lines": 151, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "baseline": { "recall": 0.425, "precision": 0.8326530612244898, "f1": 0.5627586206896552 }, "static_addition": { "recall": 0.425, "precision": 0.5930232558139535, "f1": 0.49514563106796117 }, "added_physical_lines": 99, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "baseline": { "recall": 0.14889123548046462, "precision": 0.2256, "f1": 0.17938931297709923 }, "static_addition": { "recall": 0.14889123548046462, "precision": 0.18479685452162517, "f1": 0.1649122807017544 }, "added_physical_lines": 138, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 }, "static_addition": { "recall": 1.0, "precision": 0.029806259314456036, "f1": 0.05788712011577424 }, "added_physical_lines": 161, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "status": "ok", "baseline": { "recall": 0.29678638941398866, "precision": 0.48606811145510836, "f1": 0.3685446009389671 }, "static_addition": { "recall": 0.33648393194706994, "precision": 0.39732142857142855, "f1": 0.36438075742067555 }, "added_physical_lines": 125, "added_annotated_lines": 21, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 21, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "status": "ok", "baseline": { "recall": 0.5, "precision": 0.23148148148148148, "f1": 0.31645569620253167 }, "static_addition": { "recall": 0.5, "precision": 0.1358695652173913, "f1": 0.21367521367521367 }, "added_physical_lines": 152, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "status": "ok", "baseline": { "recall": 0.2047244094488189, "precision": 0.19030192131747484, "f1": 0.19724988146040778 }, "static_addition": { "recall": 0.2047244094488189, "precision": 0.17007358953393295, "f1": 0.18579723090665476 }, "added_physical_lines": 130, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "baseline": { "recall": 0.3528399311531842, "precision": 0.2074898785425101, "f1": 0.26131293817718293 }, "static_addition": { "recall": 0.3528399311531842, "precision": 0.1843525179856115, "f1": 0.24217365623154163 }, "added_physical_lines": 124, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "baseline": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 }, "static_addition": { "recall": 0.5795795795795796, "precision": 0.1461014383043149, "f1": 0.23337363966142685 }, "added_physical_lines": 147, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "status": "ok", "baseline": { "recall": 0.5652173913043478, "precision": 0.03117505995203837, "f1": 0.05909090909090909 }, "static_addition": { "recall": 0.5652173913043478, "precision": 0.022727272727272728, "f1": 0.043697478991596636 }, "added_physical_lines": 155, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "status": "ok", "baseline": { "recall": 0.8996282527881041, "precision": 0.36890243902439024, "f1": 0.5232432432432432 }, "static_addition": { "recall": 0.8996282527881041, "precision": 0.31145431145431146, "f1": 0.4627151051625239 }, "added_physical_lines": 121, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "status": "ok", "baseline": { "recall": 0.7195767195767195, "precision": 0.1348314606741573, "f1": 0.22710826607291956 }, "static_addition": { "recall": 0.7195767195767195, "precision": 0.12846347607052896, "f1": 0.21800694629975956 }, "added_physical_lines": 150, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "status": "ok", "baseline": { "recall": 0.46819787985865724, "precision": 0.828125, "f1": 0.5981941309255079 }, "static_addition": { "recall": 0.46819787985865724, "precision": 0.5888888888888889, "f1": 0.5216535433070866 }, "added_physical_lines": 130, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 }, "static_addition": { "recall": 1.0, "precision": 0.034739454094292806, "f1": 0.0671462829736211 }, "added_physical_lines": 62, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "status": "ok", "baseline": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "static_addition": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "added_physical_lines": 137, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "status": "ok", "baseline": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.1780693533270853 }, "static_addition": { "recall": 0.2745664739884393, "precision": 0.10638297872340426, "f1": 0.15334947538337368 }, "added_physical_lines": 172, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "status": "ok", "baseline": { "recall": 0.23621323529411764, "precision": 0.7427745664739884, "f1": 0.3584379358437936 }, "static_addition": { "recall": 0.23621323529411764, "precision": 0.4732965009208103, "f1": 0.31514408338442673 }, "added_physical_lines": 197, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "status": "ok", "baseline": { "recall": 0.5901639344262295, "precision": 0.4044943820224719, "f1": 0.48 }, "static_addition": { "recall": 0.5901639344262295, "precision": 0.2660098522167488, "f1": 0.366723259762309 }, "added_physical_lines": 139, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.04223335719398712, "f1": 0.08104395604395605 }, "static_addition": { "recall": 1.0, "precision": 0.03760356915232632, "f1": 0.07248157248157248 }, "added_physical_lines": 172, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.21846153846153846, "f1": 0.35858585858585856 }, "static_addition": { "recall": 1.0, "precision": 0.17149758454106281, "f1": 0.2927835051546392 }, "added_physical_lines": 89, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "status": "ok", "baseline": { "recall": 0.8706140350877193, "precision": 0.5357624831309041, "f1": 0.6633249791144528 }, "static_addition": { "recall": 0.8706140350877193, "precision": 0.46216530849825377, "f1": 0.6038022813688213 }, "added_physical_lines": 118, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "status": "ok", "baseline": { "recall": 0.2570888468809074, "precision": 0.3646112600536193, "f1": 0.30155210643015523 }, "static_addition": { "recall": 0.2570888468809074, "precision": 0.2918454935622318, "f1": 0.2733668341708543 }, "added_physical_lines": 93, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "status": "ok", "baseline": { "recall": 0.7391304347826086, "precision": 0.04028436018957346, "f1": 0.07640449438202247 }, "static_addition": { "recall": 0.7391304347826086, "precision": 0.029513888888888888, "f1": 0.05676126878130217 }, "added_physical_lines": 154, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "status": "ok", "baseline": { "recall": 0.8773234200743495, "precision": 0.36251920122887865, "f1": 0.5130434782608696 }, "static_addition": { "recall": 0.8773234200743495, "precision": 0.30569948186528495, "f1": 0.45341018251681076 }, "added_physical_lines": 121, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "status": "ok", "baseline": { "recall": 0.4876325088339223, "precision": 0.8363636363636363, "f1": 0.6160714285714286 }, "static_addition": { "recall": 0.4876325088339223, "precision": 0.6, "f1": 0.5380116959064327 }, "added_physical_lines": 130, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "status": "ok", "baseline": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.08358208955223881 }, "static_addition": { "recall": 1.0, "precision": 0.03655352480417755, "f1": 0.07052896725440806 }, "added_physical_lines": 62, "added_annotated_lines": 0, "added_annotated_in_unseen_files": 0, "top5_annotated_absent_native": 0, "official_baseline_counts_exact": true } ], "candidate_source_receipts": [ { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "source_commit": "a505463710aa34bccafd268c44064c129cdfb3e2", "corpus_sha256": "49f210d74b969b94234c53969a497a3eb3ba9fd638088ba607547e1a8fb94d61", "corpus_nodes": 8404, "corpus_files": 253, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2042, "status": "ok", "candidate_sha256": "612b002240fdeb716dd30465e682ffeb33dfbbb23f29f54a1a540c16555153c2" }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "source_commit": "25fa3b158281bee9fd0778de618ad8968d5ed026", "corpus_sha256": "093f3c15cb892fc100f7551d570eb027447765917f6b19e95e28cc99ce08e9a2", "corpus_nodes": 9073, "corpus_files": 401, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2042, "status": "ok", "candidate_sha256": "5f9754eb034f4d329f4824c38efc8915a3e4422275b5614242b188d28da60753" }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "source_commit": "7f547827e47b5ade563a293329deb4226496d72f", "corpus_sha256": "b2c11803454e7467f0eee1d86b84e1134f93d19737374cfccc24bfbb75007951", "corpus_nodes": 2481, "corpus_files": 81, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2032, "status": "ok", "candidate_sha256": "dcb820cb31f6c3d103dc3088643dcd396a6535387ef80894fc3e24f56ac62d21" }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "source_commit": "2d48a4d9c5e8f0a5ce914922eb2a45dc0ec93ee3", "corpus_sha256": "3ad72f66deb77d9ce3c701a581282eafa0dde8818aba9feae2162ff83787baa9", "corpus_nodes": 6427, "corpus_files": 414, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2035, "status": "ok", "candidate_sha256": "5a681f0cfa555da22fd599c4531d9812f3b3ebb7d62240f1639ac44eed386f2a" }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "source_commit": "cfc05e05f08dc0b6fe619831c96f00ee27d99613", "corpus_sha256": "a348bde3b4a846b51d794d26b580c1399705abd0ba24d228d68aa1d4bc438046", "corpus_nodes": 6478, "corpus_files": 71, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2030, "status": "ok", "candidate_sha256": "5028cb676f24a0c4662b6d83bd5a34cce2f42df31e68a11d1e3cef7e71571aec" }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "source_commit": "979bbf03bb00bc55ca09783791b5091a2247df68", "corpus_sha256": "28f649ca51c5627bae36aca5a94234acf96fb1e6e8404ed9cb2f39227f67b9d9", "corpus_nodes": 3176, "corpus_files": 265, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2048, "status": "ok", "candidate_sha256": "fd6be5a0e0307cea01539c3c069040b6bd47ed86ee400784d4563b1bc5cefcff" }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "source_commit": "e6ff77ce73c201b0ee36d2b802ea45e9e1ad1822", "corpus_sha256": "929f30209c599d9174ca7f50cbcea7699f0f64612b485ec9d50e1e5acbdda436", "corpus_nodes": 4000, "corpus_files": 372, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2047, "status": "ok", "candidate_sha256": "ba593293f9dc915654ae45fe4908e4e8144c901d76a137cc2047649cdca9d185" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "source_commit": "7c9e7c1e2acd9c55926b9d2592fc65234d9c3ff7", "corpus_sha256": "95bdb186587d6cd0b5e6a982b2fcb7a330e1f49994f079fd327075d743ee5028", "corpus_nodes": 32820, "corpus_files": 1183, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2025, "status": "ok", "candidate_sha256": "7a173f8feda62d2076f99a972e425da9ca8f05dd33b5b09148c5395970712072" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "source_commit": "3f4ac29f9112799a7374a99b18acabd0232ff075", "corpus_sha256": "52473777a00786f79460ab8d1650634b06aa84d653271a63aa84e4940e257518", "corpus_nodes": 4821, "corpus_files": 207, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2048, "status": "ok", "candidate_sha256": "133e9964e536939cd491ac47a00326676220329ceef1f0144b071176200ac6da" }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "source_commit": "12b40c7ba3e7c30e35977195770c80beb34715c5", "corpus_sha256": "d169e4c2efc857c1da9269175e82bb40a75bbb9e5dfbf91ccd8a4eb07df2c8b8", "corpus_nodes": 66078, "corpus_files": 4130, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2041, "status": "ok", "candidate_sha256": "20b41e05d25b998067e93c6a75734c73dafc8573a8e81050a90153ce62337633" }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "source_commit": "0abc70564746496eb211bbd951041b4655aec268", "corpus_sha256": "72a4f6227939971f90ba7f3e9dc420a03ea32a7f03f7b6390eb444c1d4bda9a2", "corpus_nodes": 821, "corpus_files": 137, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2045, "status": "ok", "candidate_sha256": "10c1f2b0b4f057967132e94f6142bbcf792ca9877cca3cf1268198549fe18b96" }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "source_commit": "9965a04b5684079bb614addd750340ffc165a9f5", "corpus_sha256": "99bd2cef9b881a0c80d8622a50818d04980e5272431891319325ca3fb8e4f2cb", "corpus_nodes": 1122, "corpus_files": 33, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2044, "status": "ok", "candidate_sha256": "ed6f76e8c2479cbfd4e8c2ef95675524982d727d4f26514193b8fd224932a2e2" }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "source_commit": "5417c89387b67c3192ae9043473b556cd669ee15", "corpus_sha256": "423615ab75fbd283ace6c06af3d86db1d884f3dcf6d7f43a63a69db0ea7e881a", "corpus_nodes": 24841, "corpus_files": 1342, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2029, "status": "ok", "candidate_sha256": "6d087a0d4cb9f6df55b1654fdd218c48ca5957653c8063c27cd4ebda756b0b83" }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "source_commit": "135b15467eda2394b017f2a7d25cda1417c0feec", "corpus_sha256": "5f18806de2eb66936273cde26dff21bd6620ce51133a778fe500ae63e864525e", "corpus_nodes": 4134, "corpus_files": 250, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2034, "status": "ok", "candidate_sha256": "8e7d2a0f2964ee3eb4e8e6de4f247f313212996de2104cda4d4877bcb851486e" }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "source_commit": "991883df4d5910851130e3dc0e21fcbce604ea7d", "corpus_sha256": "153a0f5f415caec09bb498ef058282754020a996a36c6cfea25870307895b397", "corpus_nodes": 2121, "corpus_files": 243, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2047, "status": "ok", "candidate_sha256": "7a506eb5bfb1b97a4036b8ccdc45e66424345222135f35c2f1b547259be44158" }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "source_commit": "d276a4f3e914aaccc291f7b2513e5d978919d0f9", "corpus_sha256": "22b1c68e52ff6d1b611406e916e9db4a26d7113cbf9dcb34a00a7741990e1596", "corpus_nodes": 4359, "corpus_files": 499, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2041, "status": "ok", "candidate_sha256": "b44b41076e65a964c92c9766949d4ff46a201f0aee00656737b8de09cae3d59d" }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "source_commit": "43b46cb324f64076e4d9e807c0b60c4b9ce11a82", "corpus_sha256": "7539b5da1dd3dac6eabadefc3ba691d4f057fc2be2eb79e833c6c05a0eb233d0", "corpus_nodes": 2147, "corpus_files": 138, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2046, "status": "ok", "candidate_sha256": "c1d379677ff3fb69e97d8065c49ef4612fbebb81e99bfd6dad7572f0eb36355c" }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "source_commit": "e432032cf2d11e3bb6f558748a5ee41a90640daa", "corpus_sha256": "f556f35fc18107bcc43fd672fe9c7dc6e6b98111b3992f161912cfeecd2783f8", "corpus_nodes": 2008, "corpus_files": 73, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2045, "status": "ok", "candidate_sha256": "49d35b3c0843b3313070ad1f3ab436f255746a3e27d8cf98b7645e4246622ab8" }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "source_commit": "1e137b07052bc3ea0da44ed201702c94055b8ad2", "corpus_sha256": "600de0ecac2df9f06a8ca1d6e9290c45f06c34ee21963c3fbc1c62e01cc3c3c6", "corpus_nodes": 9158, "corpus_files": 761, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2037, "status": "ok", "candidate_sha256": "17b5374a224abb9c43b1f0ee800a6b46882dae4e1471ab455018d55a2b2a2e1e" }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "source_commit": "1e55ae64624d28c5fe8b63ad7979880ee2e6ef3f", "corpus_sha256": "f559e6e8c331d2167567c09415f011b790adc17beec9599773351e9647c5268d", "corpus_nodes": 9223, "corpus_files": 784, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2032, "status": "ok", "candidate_sha256": "e644eb5b279f1f3920e236fdc4859d7a53c62b1e7f763723c0e2466855dbd2a6" }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "source_commit": "7c4e2ac83f7b4306296ff9b7b51aaf016e5ad614", "corpus_sha256": "ce6b07a46e21f39b23c72910d4b350da2356c2d9c170526bd33701c07ebfc2ac", "corpus_nodes": 9066, "corpus_files": 149, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2037, "status": "ok", "candidate_sha256": "d63e0bd7e659d5f3f31ff6dae47b70b395e4205b19366cdaf07ed8349f04897f" }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "source_commit": "37c5b8c07be104fd5288cd87f101e48cb7a40298", "corpus_sha256": "5a8ee8bdc34a5913bc7e376ce46e0e9c9950579c32d2425e226b9571295da9cc", "corpus_nodes": 53731, "corpus_files": 2207, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2024, "status": "ok", "candidate_sha256": "3e9d7a4b95e87419f886c85c8e48a2ccd3d697d3933d9d8059e6409e1740742a" }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "source_commit": "fea269947104889165e3ebb0f64638921caf2052", "corpus_sha256": "ed205c618b2955db6e5a37b1fc8e8c17041c4d4beebf3deddee36b13256ec65d", "corpus_nodes": 2200, "corpus_files": 268, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2048, "status": "ok", "candidate_sha256": "ae3f69d634710f65cbce13ee06fb147c17ca63d1e1c6c4474cb223582b0b34f4" }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "source_commit": "a993ada9300a531c09dfa6a1be772b1abc1492ad", "corpus_sha256": "a3b1b4b148827510f46fe93a54316c6eb6a74eb485f14c20485ab514fbcac944", "corpus_nodes": 83805, "corpus_files": 3602, "all_corpus_source_exact": true, "query_was_clipped": false, "reply_tokens": 2043, "status": "ok", "candidate_sha256": "ae11204f682bf1f1b7a68d6fa63ead1da57b3a1a231d274dcd49dcef3bebafac" } ], "all_inputs_match": true, "model_calls": 0, "new_api_charges_usd": 0, "conclusion": "Both declared gates fail. No automatic injection, new BM25 agent tool, online comparison, revised official score or independent result follows from this evidence." }, "solver_controlled_search": { "protocol_sha256": "f021546301b2de156824eb1124cabea163b7b95aa63515d73cbfb59fb2e98c2a", "scope": "Instrumentation-only scripted replies in the native Docker solver loop. No LLM calls, annotation inspection, scoring, policy comparison or repair success claim.", "scripted_native_container": { "simulated_model_calls": 8, "real_model_calls": 0, "original_search_complete_lines": 20, "large_search_returned_matches": 113, "large_search_complete_visible_lines": 94, "large_search_limits_reached": [ "output_limit" ], "test_search_complete_lines": 6, "changed_file_visible_lines": 20, "changed_file_base_credit": 0, "ordinary_grep_credit": 0, "unconfirmed_search_credit": 0, "terminal_diff_credit": 0, "base_source_line_count": 120, "source_attribution_status": "partial", "container_python": "Python 3.11.2" }, "all_frozen_hashes_match": true, "all_owned_containers_removed": true, "model_calls": 0, "new_api_charges_usd": 0, "passed": true, "image": "sha256:0c36ed672e54cd9a7f09b03d60a29d3731fd34e5250227f9154306aaf81013bf", "source_tree": "c793e859f77f2ef96bcc8f1b1804f06ebce087aa", "upstream_revision": "1436c28a8eb95496da4ea69ad458b9f8a8eb7d61", "implementation_sha256": { "scripts/benchmark_context_solver.py": "007f29c32d324606f9de013744548d4ac1aa68dbac25ee4adca7140c39c6632b", "scripts/contextbench_solver_sources.py": "1a503d2c233b0006e0c63dc5357567ffac71110476a977fd1a0c55afd3157faa", "scripts/_contextbench_solver_read.py": "c10cabdcf9a4fb69339686e7e92564914dbc375d22b6e7680f2644ce84c9915f" }, "interface": "codenib-grep PATTERN PATH [LIMIT], standalone only; Python regex, tracked and non-ignored untracked text including tests.", "limits": { "default_matches": 100, "max_matches": 200, "scanned_paths": 512, "match_output_characters": 12000, "file_bytes": 2097152 }, "failed_prior_attempt": { "passed": false, "protocol_sha256": "c08cf1fc50581ff0036e6d1e83b5612d1be80dffab068f85c9146861623e7adb", "reason": "The scripted edit contained an unescaped newline in a Python string literal, so the changed-file precondition was not reached. Search commands succeeded; no instrumentation pass claimed.", "container_removed": true, "model_calls": 0, "new_api_charges_usd": 0 }, "verification": { "related_unit_passed": 104, "pinned_upstream_integration_passed": 3, "actual_model_adoption_tested": false }, "open_gate": "Fresh matched solver protocol must include the common reader/search instructions; ordinary shell output and modified-file base coordinates remain unattributed. Partial coverage is not an official score.", "implementation_commit": "8d26e841c07ead8ca4b0d05d7d6016faba289b97" }, "grep_only_completion_ablation": { "protocol_sha256": "c0373a831212094b922b5f70b7ac6446f4daabbde7ff66617c54184d0a0e3b72", "replay_receipts_sha256": "6a3d6bdca63b75509e34cc440ad11e1cdb347fd646cd29d46c693d169f289181", "scope": "Fixed recorded graph-arm tool traces on disclosed development tasks. Remove only automatic read-completion suffixes; preserve original read source, all grep completion, graph output and subsequent messages. This is a source-set feasibility diagnostic, not a valid counterfactual agent trajectory, causal policy outcome or new official score.", "gate_passed": false, "summaries": [ { "cohort": "first_source_24", "tasks": 24, "metrics": { "native": { "recall": 0.5335256710568391, "precision": 0.3162405037274296, "f1": 0.31176405157915116 }, "original": { "recall": 0.5401686329564104, "precision": 0.2860920178829964, "f1": 0.30148152970892245 }, "candidate": { "recall": 0.5313964438609067, "precision": 0.28740996971858807, "f1": 0.301154612398371 } }, "changed_replies": 44, "removed_characters": 70978, "removed_physical_lines": 1346, "removed_annotated_lines": 219, "removed_annotated_absent_native": 150 }, { "cohort": "late_opposite_8", "tasks": 8, "metrics": { "native": { "recall": 0.7789736557074383, "precision": 0.3054811929734058, "f1": 0.3367010488676228 }, "original": { "recall": 0.8056646767366582, "precision": 0.31515054386714864, "f1": 0.3755068231351938 }, "candidate": { "recall": 0.8026165604198368, "precision": 0.3207649472344307, "f1": 0.3813707795344773 } }, "changed_replies": 10, "removed_characters": 7640, "removed_physical_lines": 178, "removed_annotated_lines": 10, "removed_annotated_absent_native": 7 } ], "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "cohort": "first_source_24", "language": "python", "original": { "recall": 0.5684647302904564, "precision": 0.6116071428571429, "f1": 0.589247311827957 }, "candidate": { "recall": 0.5518672199170125, "precision": 0.6045454545454545, "f1": 0.5770065075921909 }, "native": { "recall": 0.3651452282157676, "precision": 0.704, "f1": 0.4808743169398907 }, "changed_replies": 1, "removed_characters": 226, "removed_physical_lines": 4, "removed_annotated_lines": 4, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 57, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "cohort": "first_source_24", "language": "typescript", "original": { "recall": 0.25510204081632654, "precision": 0.06297229219143577, "f1": 0.10101010101010101 }, "candidate": { "recall": 0.25510204081632654, "precision": 0.07215007215007214, "f1": 0.1124859392575928 }, "native": { "recall": 0.40816326530612246, "precision": 0.10854816824966079, "f1": 0.1714898177920686 }, "changed_replies": 1, "removed_characters": 3317, "removed_physical_lines": 101, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "cohort": "first_source_24", "language": "typescript", "original": { "recall": 1.0, "precision": 0.09104938271604938, "f1": 0.1669024045261669 }, "candidate": { "recall": 1.0, "precision": 0.10085470085470086, "f1": 0.18322981366459629 }, "native": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "changed_replies": 2, "removed_characters": 3273, "removed_physical_lines": 63, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "cohort": "first_source_24", "language": "python", "original": { "recall": 1.0, "precision": 0.3641025641025641, "f1": 0.5338345864661654 }, "candidate": { "recall": 1.0, "precision": 0.3641025641025641, "f1": 0.5338345864661654 }, "native": { "recall": 1.0, "precision": 0.3169642857142857, "f1": 0.48135593220338985 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "cohort": "first_source_24", "language": "c", "original": { "recall": 0.4616508400292184, "precision": 0.36155606407322655, "f1": 0.4055181264035932 }, "candidate": { "recall": 0.3345507669831994, "precision": 0.29934640522875816, "f1": 0.31597102449120384 }, "native": { "recall": 0.36303871439006574, "precision": 0.3066008636644047, "f1": 0.33244147157190634 }, "changed_replies": 3, "removed_characters": 6410, "removed_physical_lines": 218, "removed_annotated_lines": 174, "removed_annotated_absent_native": 119, "retained_annotated_absent_native": 40, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "cohort": "first_source_24", "language": "c", "original": { "recall": 0.5460526315789473, "precision": 0.41089108910891087, "f1": 0.4689265536723164 }, "candidate": { "recall": 0.5460526315789473, "precision": 0.41089108910891087, "f1": 0.4689265536723164 }, "native": { "recall": 0.8706140350877193, "precision": 0.5251322751322751, "f1": 0.6551155115511551 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 11, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "cohort": "first_source_24", "language": "cpp", "original": { "recall": 0.5362694300518135, "precision": 0.552, "f1": 0.5440210249671484 }, "candidate": { "recall": 0.5362694300518135, "precision": 0.552, "f1": 0.5440210249671484 }, "native": { "recall": 0.5362694300518135, "precision": 0.5036496350364964, "f1": 0.5194479297365119 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "cohort": "first_source_24", "language": "go", "original": { "recall": 0.4875, "precision": 0.45792563600782776, "f1": 0.4722502522704339 }, "candidate": { "recall": 0.48541666666666666, "precision": 0.4613861386138614, "f1": 0.4730964467005076 }, "native": { "recall": 0.425, "precision": 0.8326530612244898, "f1": 0.5627586206896552 }, "changed_replies": 2, "removed_characters": 910, "removed_physical_lines": 6, "removed_annotated_lines": 1, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 50, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "cohort": "first_source_24", "language": "java", "original": { "recall": 0.19640971488912354, "precision": 0.3089700996677741, "f1": 0.24015493867010976 }, "candidate": { "recall": 0.1700105596620908, "precision": 0.2927272727272727, "f1": 0.21509686038744155 }, "native": { "recall": 0.14889123548046462, "precision": 0.2256, "f1": 0.17938931297709923 }, "changed_replies": 3, "removed_characters": 3365, "removed_physical_lines": 52, "removed_annotated_lines": 25, "removed_annotated_absent_native": 23, "retained_annotated_absent_native": 50, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "cohort": "first_source_24", "language": "typescript", "original": { "recall": 1.0, "precision": 0.047619047619047616, "f1": 0.09090909090909091 }, "candidate": { "recall": 1.0, "precision": 0.04819277108433735, "f1": 0.09195402298850575 }, "native": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 }, "changed_replies": 1, "removed_characters": 143, "removed_physical_lines": 5, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "cohort": "first_source_24", "language": "cpp", "original": { "recall": 0.2495274102079395, "precision": 0.5076923076923077, "f1": 0.33460076045627374 }, "candidate": { "recall": 0.2495274102079395, "precision": 0.5076923076923077, "f1": 0.33460076045627374 }, "native": { "recall": 0.29678638941398866, "precision": 0.48606811145510836, "f1": 0.3685446009389671 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 20, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "cohort": "first_source_24", "language": "cpp", "original": { "recall": 0.44, "precision": 0.16356877323420074, "f1": 0.23848238482384823 }, "candidate": { "recall": 0.44, "precision": 0.16356877323420074, "f1": 0.23848238482384823 }, "native": { "recall": 0.5, "precision": 0.23148148148148148, "f1": 0.31645569620253167 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "cohort": "first_source_24", "language": "c", "original": { "recall": 0.17716535433070865, "precision": 0.14551333872271624, "f1": 0.15978695073235685 }, "candidate": { "recall": 0.17716535433070865, "precision": 0.1694915254237288, "f1": 0.17324350336862368 }, "native": { "recall": 0.2047244094488189, "precision": 0.19030192131747484, "f1": 0.19724988146040778 }, "changed_replies": 4, "removed_characters": 7847, "removed_physical_lines": 175, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "cohort": "first_source_24", "language": "javascript", "original": { "recall": 0.36316695352839934, "precision": 0.18672566371681415, "f1": 0.24663939216832262 }, "candidate": { "recall": 0.35628227194492257, "precision": 0.18956043956043955, "f1": 0.2474596533173939 }, "native": { "recall": 0.3528399311531842, "precision": 0.2074898785425101, "f1": 0.26131293817718293 }, "changed_replies": 3, "removed_characters": 2379, "removed_physical_lines": 38, "removed_annotated_lines": 4, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 7, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "cohort": "first_source_24", "language": "rust", "original": { "recall": 0.5645645645645646, "precision": 0.1551155115511551, "f1": 0.24336569579288025 }, "candidate": { "recall": 0.5495495495495496, "precision": 0.16038562664329536, "f1": 0.24830393487109906 }, "native": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 }, "changed_replies": 4, "removed_characters": 4663, "removed_physical_lines": 71, "removed_annotated_lines": 5, "removed_annotated_absent_native": 5, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "cohort": "first_source_24", "language": "rust", "original": { "recall": 0.6521739130434783, "precision": 0.02697841726618705, "f1": 0.05181347150259067 }, "candidate": { "recall": 0.6521739130434783, "precision": 0.027675276752767528, "f1": 0.05309734513274336 }, "native": { "recall": 0.5652173913043478, "precision": 0.03117505995203837, "f1": 0.05909090909090909 }, "changed_replies": 3, "removed_characters": 782, "removed_physical_lines": 14, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 2, "official_baseline_counts_exact": true }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "cohort": "first_source_24", "language": "javascript", "original": { "recall": 0.8773234200743495, "precision": 0.36932707355242567, "f1": 0.5198237885462555 }, "candidate": { "recall": 0.8661710037174721, "precision": 0.36808846761453395, "f1": 0.516629711751663 }, "native": { "recall": 0.8996282527881041, "precision": 0.36890243902439024, "f1": 0.5232432432432432 }, "changed_replies": 1, "removed_characters": 178, "removed_physical_lines": 6, "removed_annotated_lines": 3, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 1, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "cohort": "first_source_24", "language": "rust", "original": { "recall": 0.7248677248677249, "precision": 0.1282371294851794, "f1": 0.21792152704135737 }, "candidate": { "recall": 0.7248677248677249, "precision": 0.14885910901847157, "f1": 0.24699519230769232 }, "native": { "recall": 0.7195767195767195, "precision": 0.1348314606741573, "f1": 0.22710826607291956 }, "changed_replies": 6, "removed_characters": 21374, "removed_physical_lines": 444, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 31, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "cohort": "first_source_24", "language": "java", "original": { "recall": 0.44876325088339225, "precision": 0.8639455782312925, "f1": 0.5906976744186047 }, "candidate": { "recall": 0.44346289752650175, "precision": 0.8625429553264605, "f1": 0.5857642940490082 }, "native": { "recall": 0.46819787985865724, "precision": 0.828125, "f1": 0.5981941309255079 }, "changed_replies": 1, "removed_characters": 428, "removed_physical_lines": 3, "removed_annotated_lines": 3, "removed_annotated_absent_native": 3, "retained_annotated_absent_native": 21, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "cohort": "first_source_24", "language": "go", "original": { "recall": 1.0, "precision": 0.036458333333333336, "f1": 0.07035175879396985 }, "candidate": { "recall": 1.0, "precision": 0.04375, "f1": 0.08383233532934131 }, "native": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 }, "changed_replies": 2, "removed_characters": 2929, "removed_physical_lines": 64, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "cohort": "first_source_24", "language": "java", "original": { "recall": 0.375, "precision": 0.05647058823529412, "f1": 0.09815950920245399 }, "candidate": { "recall": 0.375, "precision": 0.06015037593984962, "f1": 0.10367170626349892 }, "native": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "changed_replies": 2, "removed_characters": 2131, "removed_physical_lines": 26, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 24, "official_baseline_counts_exact": true }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "cohort": "first_source_24", "language": "go", "original": { "recall": 0.22832369942196531, "precision": 0.11318051575931232, "f1": 0.15134099616858238 }, "candidate": { "recall": 0.22832369942196531, "precision": 0.11634756995581738, "f1": 0.15414634146341463 }, "native": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.1780693533270853 }, "changed_replies": 4, "removed_characters": 8457, "removed_physical_lines": 19, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "cohort": "first_source_24", "language": "javascript", "original": { "recall": 0.2270220588235294, "precision": 0.5678160919540229, "f1": 0.32435981615233095 }, "candidate": { "recall": 0.2270220588235294, "precision": 0.5678160919540229, "f1": 0.32435981615233095 }, "native": { "recall": 0.23621323529411764, "precision": 0.7427745664739884, "f1": 0.3584379358437936 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "cohort": "first_source_24", "language": "python", "original": { "recall": 0.5846994535519126, "precision": 0.27648578811369506, "f1": 0.37543859649122807 }, "candidate": { "recall": 0.5846994535519126, "precision": 0.3057142857142857, "f1": 0.40150093808630394 }, "native": { "recall": 0.5901639344262295, "precision": 0.4044943820224719, "f1": 0.48 }, "changed_replies": 1, "removed_characters": 2166, "removed_physical_lines": 37, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 8, "official_baseline_counts_exact": true }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "cohort": "late_opposite_8", "language": "typescript", "original": { "recall": 1.0, "precision": 0.08885542168674698, "f1": 0.1632088520055325 }, "candidate": { "recall": 1.0, "precision": 0.09161490683229814, "f1": 0.1678520625889047 }, "native": { "recall": 1.0, "precision": 0.04223335719398712, "f1": 0.08104395604395605 }, "changed_replies": 1, "removed_characters": 701, "removed_physical_lines": 20, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "cohort": "late_opposite_8", "language": "python", "original": { "recall": 1.0, "precision": 0.2305194805194805, "f1": 0.37467018469656993 }, "candidate": { "recall": 1.0, "precision": 0.25177304964539005, "f1": 0.40226628895184136 }, "native": { "recall": 1.0, "precision": 0.21846153846153846, "f1": 0.35858585858585856 }, "changed_replies": 1, "removed_characters": 1177, "removed_physical_lines": 26, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "cohort": "late_opposite_8", "language": "c", "original": { "recall": 0.43201754385964913, "precision": 0.36822429906542054, "f1": 0.3975782038345106 }, "candidate": { "recall": 0.43201754385964913, "precision": 0.36822429906542054, "f1": 0.3975782038345106 }, "native": { "recall": 0.8706140350877193, "precision": 0.5357624831309041, "f1": 0.6633249791144528 }, "changed_replies": 0, "removed_characters": 0, "removed_physical_lines": 0, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "cohort": "late_opposite_8", "language": "cpp", "original": { "recall": 0.4706994328922495, "precision": 0.5220125786163522, "f1": 0.49502982107355864 }, "candidate": { "recall": 0.45746691871455575, "precision": 0.5159914712153518, "f1": 0.4849699398797595 }, "native": { "recall": 0.2570888468809074, "precision": 0.3646112600536193, "f1": 0.30155210643015523 }, "changed_replies": 2, "removed_characters": 292, "removed_physical_lines": 8, "removed_annotated_lines": 7, "removed_annotated_absent_native": 7, "retained_annotated_absent_native": 106, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "cohort": "late_opposite_8", "language": "rust", "original": { "recall": 0.6521739130434783, "precision": 0.027624309392265192, "f1": 0.053003533568904596 }, "candidate": { "recall": 0.6521739130434783, "precision": 0.029880478087649404, "f1": 0.05714285714285714 }, "native": { "recall": 0.7391304347826086, "precision": 0.04028436018957346, "f1": 0.07640449438202247 }, "changed_replies": 3, "removed_characters": 2239, "removed_physical_lines": 41, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "cohort": "late_opposite_8", "language": "javascript", "original": { "recall": 0.8921933085501859, "precision": 0.35502958579881655, "f1": 0.5079365079365079 }, "candidate": { "recall": 0.8810408921933085, "precision": 0.3537313432835821, "f1": 0.5047923322683706 }, "native": { "recall": 0.8773234200743495, "precision": 0.36251920122887865, "f1": 0.5130434782608696 }, "changed_replies": 1, "removed_characters": 178, "removed_physical_lines": 6, "removed_annotated_lines": 3, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 5, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "cohort": "late_opposite_8", "language": "java", "original": { "recall": 0.9982332155477032, "precision": 0.8925750394944708, "f1": 0.9424520433694745 }, "candidate": { "recall": 0.9982332155477032, "precision": 0.9112903225806451, "f1": 0.9527824620573356 }, "native": { "recall": 0.4876325088339223, "precision": 0.8363636363636363, "f1": 0.6160714285714286 }, "changed_replies": 1, "removed_characters": 708, "removed_physical_lines": 13, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 289, "official_baseline_counts_exact": true }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "cohort": "late_opposite_8", "language": "go", "original": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 }, "candidate": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.08358208955223881 }, "native": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.08358208955223881 }, "changed_replies": 1, "removed_characters": 2345, "removed_physical_lines": 64, "removed_annotated_lines": 0, "removed_annotated_absent_native": 0, "retained_annotated_absent_native": 0, "official_baseline_counts_exact": true } ], "all_frozen_hashes_match": true, "model_calls": 0, "new_api_charges_usd": 0, "runtime_commit": "e10b3d6e31623e6b42a605611822635720b78541", "motivation": "Existing attribution suggests grep fragment completion supplies useful source; read commands already request substantial explicit source ranges. Earlier raw annotation counts cannot support an ablation decision, so all comparisons use canonical official spans and baseline-count parity.", "transformation": "For successful read replies containing the exact existing function-completion marker, retain the original prefix and remove the complete suffix. No replacement source, query, graph reply or later model text is generated. All original tool calls and other responses remain byte-identical.", "acceptance": "At least one original read-completion suffix changes. Preserve every originally observed canonical annotated line in both cohorts, and do not decrease macro F1 in either cohort, with strict F1 improvement in at least one. A pass only supports a fresh matched development hypothesis; a failure does not justify removing all read completion.", "annotation_use": "Freeze all transformed source sets and reply receipts before reading canonical annotations or official baseline totals. Exact baseline gold/prediction/intersection parity is required before comparison.", "analysis_sha256": "f75fa2f743870059a502aa589b8b431a0b4e855292e52d5e1fd6f2ee75d6ca38", "decision": "Do not remove read completion wholesale. Canonical-source loss fails the predeclared gate; no implementation change or online ablation follows. Earlier scores remain unchanged." }, "solver_search_adoption": { "protocol_sha256": "050dd033f6ce718b8d5df39c53fc4fcbabb609e071902c7396f6bfd5730020b5", "scope": "One repeated disclosed C development task to verify actual adoption of common controlled search. No new retrieval, graph-policy comparison, independent validation or official score.", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "model_calls": 9, "actual_models": [ "claude-opus-5-5" ], "controlled_search_adopted": false, "confirmed_search_executions": 0, "confirmed_search_files": [], "confirmed_unique_search_source_lines": 0, "search_source_absent_initial_context": 0, "initial_source_lines": 1025, "base_source_line_count": 1095, "source_coverage_status": "partial", "missing_execution_receipt_indexes": [], "unattributed_delivered_shell_events": 6, "all_model_inputs_confirmed": true, "all_frozen_hashes_match": true, "sdk_estimate_usd": 2.2842187, "new_api_charges_usd": 0, "solver_container_removed": true, "solver_status": "Submitted", "native_grading": { "status": "graded", "resolved": false, "error": null, "container_removed": true, "process_exit_code": 0, "failed_tests": 11 }, "all_frozen_hashes_match_after_grading": true, "confirmed_controlled_reads": 2, "controlled_read_source_absent_initial_context": 70 }, "jev_rerank_development": { "scope": "Jev candidate-order control on all 24 disclosed development tasks. Same frozen top-40 BM25 pools and 2K/top-10 packing. Source-set results only; no live agent, repair, independent validation or leaderboard result.", "protocol_sha256": "056973a8b33efe9fe7666713cd1f5e1ad0ffcbb6217e75c1500794ce17e38989", "ranking_receipts_sha256": "72ea69055c0c51998b8888937fb5e49bad54fbeb44adcaf026aa55c0922180bc", "known_api_cost_usd": 0.025743857999999994, "api_calls": 96, "unknown_usage_calls": 0, "all_frozen_hashes_match": true, "summaries": [ { "cohort": "first_source_24", "tasks": 24, "repositories": 24, "metrics": { "bm25": { "recall": 0.005313384874787362, "precision": 0.02102471477410686, "f1": 0.008322141527011596 }, "jev": { "recall": 0.0911478747829688, "precision": 0.16229329667683842, "f1": 0.07771143619669552 }, "native": { "recall": 0.5335256710568391, "precision": 0.31624050372742957, "f1": 0.31176405157915105 }, "native_bm25": { "recall": 0.535697144630244, "precision": 0.23052132435237457, "f1": 0.26622916691022036 }, "native_jev": { "recall": 0.5403169993271247, "precision": 0.24965917898088988, "f1": 0.27952350069978876 } }, "jev_minus_bm25_95pct": { "recall": [ 0.02432292078380427, 0.1798168310377241 ], "f1": [ 0.029698925931874176, 0.11391032612836739 ] }, "jev_added_annotated": 143, "novel_annotated_languages": [ "c", "cpp", "java", "javascript" ] }, { "cohort": "late_opposite_8", "tasks": 8, "repositories": 8, "metrics": { "bm25": { "recall": 0.013468809073724008, "precision": 0.04398148148148148, "f1": 0.020622286541244574 }, "jev": { "recall": 0.23035097353898942, "precision": 0.25866431945575435, "f1": 0.16181052903287596 }, "native": { "recall": 0.7789736557074384, "precision": 0.3054811929734058, "f1": 0.33670104886762275 }, "native_bm25": { "recall": 0.7789736557074384, "precision": 0.24185985641402824, "f1": 0.2951432884543551 }, "native_jev": { "recall": 0.7876600840893573, "precision": 0.2693304186951592, "f1": 0.31992740941672704 } }, "jev_minus_bm25_95pct": { "recall": [ 0.06270664667450601, 0.44373035720307696 ], "f1": [ 0.06391897957896489, 0.2261172284107745 ] }, "jev_added_annotated": 35, "novel_annotated_languages": [ "c", "cpp" ] } ], "rows": [ { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "repo": "pylint-dev/pylint", "language": "python", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.3651452282157676, "precision": 0.704, "f1": 0.4808743169398907 }, "native_bm25": { "recall": 0.3651452282157676, "precision": 0.2913907284768212, "f1": 0.3241252302025783 }, "native_jev": { "recall": 0.3651452282157676, "precision": 0.3044982698961938, "f1": 0.3320754716981132 }, "jev_added_physical": 164, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "repo": "vuejs/core", "language": "typescript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.40816326530612246, "precision": 0.10854816824966079, "f1": 0.1714898177920686 }, "native_bm25": { "recall": 0.40816326530612246, "precision": 0.0866738894907909, "f1": 0.14298480786416443 }, "native_jev": { "recall": 0.40816326530612246, "precision": 0.09080590238365494, "f1": 0.14856081708449395 }, "jev_added_physical": 144, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "repo": "microsoft/vscode", "language": "typescript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "native_bm25": { "recall": 1.0, "precision": 0.05432780847145488, "f1": 0.10305676855895196 }, "native_jev": { "recall": 1.0, "precision": 0.05795677799607073, "f1": 0.1095636025998143 }, "jev_added_physical": 104, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "repo": "django/django", "language": "python", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.2112676056338028, "precision": 0.09868421052631579, "f1": 0.13452914798206278 }, "native": { "recall": 1.0, "precision": 0.3169642857142857, "f1": 0.48135593220338985 }, "native_bm25": { "recall": 1.0, "precision": 0.21385542168674698, "f1": 0.3523573200992556 }, "native_jev": { "recall": 1.0, "precision": 0.22327044025157233, "f1": 0.36503856041131105 }, "jev_added_physical": 94, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "repo": "ponylang/ponyc", "language": "c", "bm25": { "recall": 0.01241782322863404, "precision": 0.11409395973154363, "f1": 0.022397891963109356 }, "jev": { "recall": 0.13513513513513514, "precision": 1.0, "f1": 0.23809523809523808 }, "native": { "recall": 0.36303871439006574, "precision": 0.3066008636644047, "f1": 0.33244147157190634 }, "native_bm25": { "recall": 0.3754565376186998, "precision": 0.2903954802259887, "f1": 0.32749283211213764 }, "native_jev": { "recall": 0.42658875091307524, "precision": 0.34192037470725994, "f1": 0.37959051023724405 }, "jev_added_physical": 87, "jev_added_annotated": 87, "bm25_added_annotated": 17, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "repo": "jqlang/jq", "language": "c", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.1600877192982456, "precision": 0.5615384615384615, "f1": 0.24914675767918087 }, "native": { "recall": 0.8706140350877193, "precision": 0.5251322751322751, "f1": 0.6551155115511551 }, "native_bm25": { "recall": 0.8706140350877193, "precision": 0.45423340961098396, "f1": 0.5969924812030075 }, "native_jev": { "recall": 0.8947368421052632, "precision": 0.49514563106796117, "f1": 0.6375 }, "jev_added_physical": 68, "jev_added_annotated": 11, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "repo": "fmtlib/fmt", "language": "cpp", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.5362694300518135, "precision": 0.5036496350364964, "f1": 0.5194479297365119 }, "native_bm25": { "recall": 0.5362694300518135, "precision": 0.3683274021352313, "f1": 0.43670886075949367 }, "native_jev": { "recall": 0.5362694300518135, "precision": 0.3670212765957447, "f1": 0.4357894736842105 }, "jev_added_physical": 153, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "repo": "future-architect/vuls", "language": "go", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.004166666666666667, "precision": 0.02197802197802198, "f1": 0.0070052539404553416 }, "native": { "recall": 0.425, "precision": 0.8326530612244898, "f1": 0.5627586206896552 }, "native_bm25": { "recall": 0.425, "precision": 0.5930232558139535, "f1": 0.49514563106796117 }, "native_jev": { "recall": 0.425, "precision": 0.6107784431137725, "f1": 0.5012285012285013 }, "jev_added_physical": 89, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "repo": "fasterxml/jackson", "language": "java", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.14889123548046462, "precision": 0.2256, "f1": 0.17938931297709923 }, "native_bm25": { "recall": 0.14889123548046462, "precision": 0.18479685452162517, "f1": 0.1649122807017544 }, "native_jev": { "recall": 0.14889123548046462, "precision": 0.17780580075662042, "f1": 0.16206896551724137 }, "jev_added_physical": 168, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "repo": "darkreader/darkreader", "language": "typescript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 }, "native_bm25": { "recall": 1.0, "precision": 0.029806259314456036, "f1": 0.05788712011577424 }, "native_jev": { "recall": 1.0, "precision": 0.030816640986132512, "f1": 0.059790732436472344 }, "jev_added_physical": 139, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "repo": "nlohmann/json", "language": "cpp", "bm25": { "recall": 0.10775047258979206, "precision": 0.35185185185185186, "f1": 0.1649782923299566 }, "jev": { "recall": 0.15500945179584122, "precision": 0.5061728395061729, "f1": 0.23733719247467439 }, "native": { "recall": 0.29678638941398866, "precision": 0.48606811145510836, "f1": 0.3685446009389671 }, "native_bm25": { "recall": 0.33648393194706994, "precision": 0.39732142857142855, "f1": 0.36438075742067555 }, "native_jev": { "recall": 0.33648393194706994, "precision": 0.4258373205741627, "f1": 0.3759239704329461 }, "jev_added_physical": 95, "jev_added_annotated": 21, "bm25_added_annotated": 21, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "repo": "catchorg/Catch2", "language": "cpp", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.5, "precision": 0.23148148148148148, "f1": 0.31645569620253167 }, "native_bm25": { "recall": 0.5, "precision": 0.1358695652173913, "f1": 0.21367521367521367 }, "native_jev": { "recall": 0.5, "precision": 0.14450867052023122, "f1": 0.2242152466367713 }, "jev_added_physical": 130, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "repo": "facebook/zstd", "language": "c", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.2047244094488189, "precision": 0.19030192131747484, "f1": 0.19724988146040778 }, "native_bm25": { "recall": 0.2047244094488189, "precision": 0.17007358953393295, "f1": 0.18579723090665476 }, "native_jev": { "recall": 0.2047244094488189, "precision": 0.16979591836734695, "f1": 0.1856314145470772 }, "jev_added_physical": 132, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "repo": "NodeBB/NodeBB", "language": "javascript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.1686746987951807, "precision": 0.5975609756097561, "f1": 0.26308724832214764 }, "native": { "recall": 0.3528399311531842, "precision": 0.2074898785425101, "f1": 0.26131293817718293 }, "native_bm25": { "recall": 0.3528399311531842, "precision": 0.1843525179856115, "f1": 0.24217365623154163 }, "native_jev": { "recall": 0.3528399311531842, "precision": 0.20398009950248755, "f1": 0.25851197982345525 }, "jev_added_physical": 17, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "repo": "clap-rs/clap", "language": "rust", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 }, "native_bm25": { "recall": 0.5795795795795796, "precision": 0.1461014383043149, "f1": 0.23337363966142685 }, "native_jev": { "recall": 0.5795795795795796, "precision": 0.15691056910569107, "f1": 0.24696097248880358 }, "jev_added_physical": 56, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "repo": "tokio-rs/bytes", "language": "rust", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.5652173913043478, "precision": 0.03117505995203837, "f1": 0.05909090909090909 }, "native_bm25": { "recall": 0.5652173913043478, "precision": 0.022727272727272728, "f1": 0.043697478991596636 }, "native_jev": { "recall": 0.5652173913043478, "precision": 0.02594810379241517, "f1": 0.04961832061068702 }, "jev_added_physical": 84, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "repo": "serverless/serverless", "language": "javascript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.3048327137546468, "precision": 0.47126436781609193, "f1": 0.37020316027088035 }, "native": { "recall": 0.8996282527881041, "precision": 0.36890243902439024, "f1": 0.5232432432432432 }, "native_bm25": { "recall": 0.8996282527881041, "precision": 0.31145431145431146, "f1": 0.4627151051625239 }, "native_jev": { "recall": 0.8996282527881041, "precision": 0.3644578313253012, "f1": 0.5187566988210075 }, "jev_added_physical": 8, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "repo": "nushell/nushell", "language": "rust", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.7195767195767195, "precision": 0.1348314606741573, "f1": 0.22710826607291956 }, "native_bm25": { "recall": 0.7195767195767195, "precision": 0.12846347607052896, "f1": 0.21800694629975956 }, "native_jev": { "recall": 0.7195767195767195, "precision": 0.12882854436375119, "f1": 0.2185324049276915 }, "jev_added_physical": 141, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "repo": "google/gson", "language": "java", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.08303886925795052, "precision": 0.3381294964028777, "f1": 0.13333333333333333 }, "native": { "recall": 0.46819787985865724, "precision": 0.828125, "f1": 0.5981941309255079 }, "native_bm25": { "recall": 0.46819787985865724, "precision": 0.5888888888888889, "f1": 0.5216535433070866 }, "native_jev": { "recall": 0.49646643109540634, "precision": 0.6565420560747663, "f1": 0.5653923541247485 }, "jev_added_physical": 108, "jev_added_annotated": 16, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "repo": "cli/cli", "language": "go", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.9285714285714286, "precision": 0.09352517985611511, "f1": 0.16993464052287582 }, "native": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 }, "native_bm25": { "recall": 1.0, "precision": 0.034739454094292806, "f1": 0.0671462829736211 }, "native_jev": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 }, "jev_added_physical": 0, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "repo": "alibaba/fastjson2", "language": "java", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native_bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native_jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev_added_physical": 151, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "repo": "flipt-io/flipt", "language": "go", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.1780693533270853 }, "native_bm25": { "recall": 0.2745664739884393, "precision": 0.10638297872340426, "f1": 0.15334947538337368 }, "native_jev": { "recall": 0.2745664739884393, "precision": 0.10590858416945373, "f1": 0.15285599356395815 }, "jev_added_physical": 176, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "repo": "axios/axios", "language": "javascript", "bm25": { "recall": 0.007352941176470588, "precision": 0.03864734299516908, "f1": 0.012355212355212355 }, "jev": { "recall": 0.03676470588235294, "precision": 0.20618556701030927, "f1": 0.062402496099843996 }, "native": { "recall": 0.23621323529411764, "precision": 0.7427745664739884, "f1": 0.3584379358437936 }, "native_bm25": { "recall": 0.23621323529411764, "precision": 0.4732965009208103, "f1": 0.31514408338442673 }, "native_jev": { "recall": 0.24356617647058823, "precision": 0.6162790697674418, "f1": 0.34914361001317523 }, "jev_added_physical": 84, "jev_added_annotated": 8, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "repo": "pydata/xarray", "language": "python", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.5901639344262295, "precision": 0.4044943820224719, "f1": 0.48 }, "native_bm25": { "recall": 0.5901639344262295, "precision": 0.2660098522167488, "f1": 0.366723259762309 }, "native_jev": { "recall": 0.5901639344262295, "precision": 0.2517482517482518, "f1": 0.35294117647058826 }, "jev_added_physical": 162, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "repo": "microsoft/vscode", "language": "typescript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 1.0, "precision": 0.04223335719398712, "f1": 0.08104395604395605 }, "native_bm25": { "recall": 1.0, "precision": 0.03760356915232632, "f1": 0.07248157248157248 }, "native_jev": { "recall": 1.0, "precision": 0.03930712858094604, "f1": 0.07564102564102564 }, "jev_added_physical": 104, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "repo": "django/django", "language": "python", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.2112676056338028, "precision": 0.09868421052631579, "f1": 0.13452914798206278 }, "native": { "recall": 1.0, "precision": 0.21846153846153846, "f1": 0.35858585858585856 }, "native_bm25": { "recall": 1.0, "precision": 0.17149758454106281, "f1": 0.2927835051546392 }, "native_jev": { "recall": 1.0, "precision": 0.1775, "f1": 0.30148619957537154 }, "jev_added_physical": 75, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "repo": "jqlang/jq", "language": "c", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.1600877192982456, "precision": 0.5615384615384615, "f1": 0.24914675767918087 }, "native": { "recall": 0.8706140350877193, "precision": 0.5357624831309041, "f1": 0.6633249791144528 }, "native_bm25": { "recall": 0.8706140350877193, "precision": 0.46216530849825377, "f1": 0.6038022813688213 }, "native_jev": { "recall": 0.8947368421052632, "precision": 0.5043263288009888, "f1": 0.6450592885375495 }, "jev_added_physical": 68, "jev_added_annotated": 11, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "repo": "nlohmann/json", "language": "cpp", "bm25": { "recall": 0.10775047258979206, "precision": 0.35185185185185186, "f1": 0.1649782923299566 }, "jev": { "recall": 0.15500945179584122, "precision": 0.5061728395061729, "f1": 0.23733719247467439 }, "native": { "recall": 0.2570888468809074, "precision": 0.3646112600536193, "f1": 0.30155210643015523 }, "native_bm25": { "recall": 0.2570888468809074, "precision": 0.2918454935622318, "f1": 0.2733668341708543 }, "native_jev": { "recall": 0.30245746691871456, "precision": 0.34408602150537637, "f1": 0.32193158953722334 }, "jev_added_physical": 92, "jev_added_annotated": 24, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "repo": "tokio-rs/bytes", "language": "rust", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.7391304347826086, "precision": 0.04028436018957346, "f1": 0.07640449438202247 }, "native_bm25": { "recall": 0.7391304347826086, "precision": 0.029513888888888888, "f1": 0.05676126878130217 }, "native_jev": { "recall": 0.7391304347826086, "precision": 0.033663366336633666, "f1": 0.06439393939393939 }, "jev_added_physical": 83, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "repo": "serverless/serverless", "language": "javascript", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.3048327137546468, "precision": 0.47126436781609193, "f1": 0.37020316027088035 }, "native": { "recall": 0.8773234200743495, "precision": 0.36251920122887865, "f1": 0.5130434782608696 }, "native_bm25": { "recall": 0.8773234200743495, "precision": 0.30569948186528495, "f1": 0.45341018251681076 }, "native_jev": { "recall": 0.8773234200743495, "precision": 0.3581183611532625, "f1": 0.5086206896551724 }, "jev_added_physical": 8, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "repo": "google/gson", "language": "java", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.08303886925795052, "precision": 0.3381294964028777, "f1": 0.13333333333333333 }, "native": { "recall": 0.4876325088339223, "precision": 0.8363636363636363, "f1": 0.6160714285714286 }, "native_bm25": { "recall": 0.4876325088339223, "precision": 0.6, "f1": 0.5380116959064327 }, "native_jev": { "recall": 0.4876325088339223, "precision": 0.6540284360189573, "f1": 0.5587044534412956 }, "jev_added_physical": 92, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "repo": "cli/cli", "language": "go", "bm25": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.9285714285714286, "precision": 0.09352517985611511, "f1": 0.16993464052287582 }, "native": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.08358208955223881 }, "native_bm25": { "recall": 1.0, "precision": 0.03655352480417755, "f1": 0.07052896725440806 }, "native_jev": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.08358208955223881 }, "jev_added_physical": 0, "jev_added_annotated": 0, "bm25_added_annotated": 0, "official_baseline_counts_exact": true } ], "rerank_development_gate_passed": true, "static_injection_gate_passed": false, "integrity_audit": { "scope": "Post-run source/permutation/cost integrity audit and candidate coverage diagnostic. No policy or selection change.", "all_candidate_permutations_exact": true, "all_delivered_source_exact": true, "all_requests_and_cost_receipts_exact": true, "all_frozen_hashes_match": true, "tasks": 24, "candidate_pool_macro_recall": 0.11994866004421378, "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "pool_recall": 0.0, "candidate_union_lines": 1214, "gold_lines": 241, "ranked_delivery_lines": 164 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "pool_recall": 0.11734693877551021, "candidate_union_lines": 1214, "gold_lines": 196, "ranked_delivery_lines": 200 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "pool_recall": 0.0, "candidate_union_lines": 3435, "gold_lines": 59, "ranked_delivery_lines": 104 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "pool_recall": 0.2112676056338028, "candidate_union_lines": 702, "gold_lines": 71, "ranked_delivery_lines": 152 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "pool_recall": 0.16362308254200147, "candidate_union_lines": 2033, "gold_lines": 1369, "ranked_delivery_lines": 185 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "pool_recall": 0.1600877192982456, "candidate_union_lines": 1779, "gold_lines": 456, "ranked_delivery_lines": 130 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "pool_recall": 0.0, "candidate_union_lines": 1641, "gold_lines": 386, "ranked_delivery_lines": 153 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "pool_recall": 0.004166666666666667, "candidate_union_lines": 1325, "gold_lines": 480, "ranked_delivery_lines": 91 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "pool_recall": 0.0, "candidate_union_lines": 1101, "gold_lines": 947, "ranked_delivery_lines": 168 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "pool_recall": 0.0, "candidate_union_lines": 1062, "gold_lines": 20, "ranked_delivery_lines": 139 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "pool_recall": 0.19659735349716445, "candidate_union_lines": 1307, "gold_lines": 529, "ranked_delivery_lines": 162 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "pool_recall": 0.0, "candidate_union_lines": 1485, "gold_lines": 100, "ranked_delivery_lines": 162 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "pool_recall": 0.0, "candidate_union_lines": 2076, "gold_lines": 1016, "ranked_delivery_lines": 132 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "pool_recall": 0.1686746987951807, "candidate_union_lines": 1861, "gold_lines": 581, "ranked_delivery_lines": 164 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "pool_recall": 0.0, "candidate_union_lines": 817, "gold_lines": 333, "ranked_delivery_lines": 109 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "pool_recall": 0.0, "candidate_union_lines": 1324, "gold_lines": 23, "ranked_delivery_lines": 150 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "pool_recall": 0.7323420074349443, "candidate_union_lines": 2967, "gold_lines": 269, "ranked_delivery_lines": 174 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "pool_recall": 0.0, "candidate_union_lines": 1236, "gold_lines": 567, "ranked_delivery_lines": 141 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "pool_recall": 0.08303886925795052, "candidate_union_lines": 1213, "gold_lines": 566, "ranked_delivery_lines": 139 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "pool_recall": 0.9285714285714286, "candidate_union_lines": 2022, "gold_lines": 14, "ranked_delivery_lines": 139 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "pool_recall": 0.0, "candidate_union_lines": 1414, "gold_lines": 64, "ranked_delivery_lines": 151 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "pool_recall": 0.0, "candidate_union_lines": 524, "gold_lines": 346, "ranked_delivery_lines": 176 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "pool_recall": 0.1130514705882353, "candidate_union_lines": 1018, "gold_lines": 1088, "ranked_delivery_lines": 194 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "pool_recall": 0.0, "candidate_union_lines": 2549, "gold_lines": 183, "ranked_delivery_lines": 163 } ], "response_models": [ "typesafe/jev-1.13-20260917" ], "max_request_cost_usd": 0.000651168, "new_model_calls": 0, "new_api_charges_usd": 0 }, "model": "typesafe/jev-1.13", "response_models": [ "typesafe/jev-1.13-20260917" ], "delivery": { "candidates": 40, "top_k": 10, "token_budget": 2048, "whole_source_lines": true, "tests_included": true }, "ranking": "Existing four-level code relevance rubric, issue prefix <=16000 chars, candidate content <=3000 chars, batches of ten, descending mean score, original order on ties. No thresholds, gold, patch or native observations in ranking.", "budget": { "api_reservation_usd": 1, "per_call_reservation_usd": 0.01, "max_calls": 96, "retries": 0, "timeout_s": 30, "on_error_or_unknown_usage": "Persist receipt and stop all later calls; no model or ordering fallback counted as Jev." }, "uncertainty": { "method": "repository-cluster paired bootstrap, exploratory 95%", "draws": 5000, "seed": 20261017 }, "runtime_commit": "e10b3d6e31623e6b42a605611822635720b78541", "implementation_sha256": { "scripts/benchmark_context_agent.py": "7cf8372c78449d30c327e014ac04f9e54599bdb8586c1bd0b68904d3f6f75621", "scripts/benchmark_context_claude.py": "9e3ec04a05e4e87c7f8885c4cfb99a2aa920dad5b0a6fd2f66c243e072a105de", "scripts/contextbench_graph_navigation.py": "d188b4c0edeaba024d8aef958c00b237d212a3f67a76923c9e6d4a446be03bac", "scripts/benchmark_context_serving.py": "6fa40f7ad145938efcb1edceddcad864b2febe0c1c61dda315d9cd5122048fe2", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "1d6d1718d24abaccb46e4f6743ac8033e145628fbc55ceec85d07c0390b39609", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/eval/benchmarks/_contextbench_evaluate.py": "26b82cc9e800759469c52d972ac2f4b5cfda69f052601841f026f05e28cf222c", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "5a29bc267bc4d8946d7983590e87f14b838bb2f34ccba41927d4cc3302f1f9ef", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "conclusion": "Jev improves the same 2K candidate payload, but the frozen BM25 pool covers only 11.99% of canonical annotated source on average. Appending the ranked payload still reduces F1 versus native on both cohorts, so static injection is rejected. Prioritize a separately frozen grep-candidate ranking control before any new online policy; no independent or leaderboard gain is established." }, "jev_grep_development": { "scope": "Jev candidate-order control on all 24 disclosed development tasks. Same frozen top-40 planned-grep pools and 4K/top-10 packing. Source-set results only; no live agent, repair, independent validation or leaderboard result.", "protocol_sha256": "680e669f4b40a798a441ffed6052a6e08279ec255713ca0dab1a1c766cbd46a9", "ranking_receipts_sha256": "222e9d34284a9f555a307bb5f71a569843219deaa2310ed0b9b7990be031b89e", "known_api_cost_usd": 0.012341112, "api_calls": 55, "unknown_usage_calls": 0, "all_frozen_hashes_match": true, "summaries": [ { "cohort": "first_source_24", "tasks": 24, "repositories": 24, "metrics": { "grep": { "recall": 0.32458311738472695, "precision": 0.261706256169623, "f1": 0.22900656367776848 }, "jev": { "recall": 0.3655804028334009, "precision": 0.3544915727971922, "f1": 0.2852113816077193 }, "native": { "recall": 0.5335256710568391, "precision": 0.31624050372742957, "f1": 0.31176405157915105 }, "native_grep": { "recall": 0.5717884203180778, "precision": 0.2509067873956268, "f1": 0.29116504241742264 }, "native_jev": { "recall": 0.5767756518829088, "precision": 0.2629296939776763, "f1": 0.30164330185834987 } }, "jev_minus_grep_95pct": { "recall": [ 0.00841718963544605, 0.07800411633637323 ], "f1": [ 0.017020366750142114, 0.10018795250053998 ] }, "jev_added_annotated": 382, "novel_annotated_languages": [ "c", "cpp", "java", "python" ] }, { "cohort": "late_opposite_8", "tasks": 8, "repositories": 8, "metrics": { "grep": { "recall": 0.5805245534139435, "precision": 0.3347305906283551, "f1": 0.31511161775638946 }, "jev": { "recall": 0.6118958196371223, "precision": 0.3569689810439398, "f1": 0.34900013157280524 }, "native": { "recall": 0.7789736557074384, "precision": 0.3054811929734058, "f1": 0.33670104886762275 }, "native_grep": { "recall": 0.7912646249591897, "precision": 0.2668793567875476, "f1": 0.3117284129518817 }, "native_jev": { "recall": 0.7939691674492438, "precision": 0.2716904637264399, "f1": 0.31820507275387483 } }, "jev_minus_grep_95pct": { "recall": [ -0.009046052631578955, 0.10315985130111521 ], "f1": [ -0.01018261765281364, 0.09811952045740427 ] }, "jev_added_annotated": 58, "novel_annotated_languages": [ "c", "java", "javascript" ] } ], "rows": [ { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "repo": "pylint-dev/pylint", "language": "python", "grep": { "recall": 0.5186721991701245, "precision": 0.49019607843137253, "f1": 0.5040322580645161 }, "jev": { "recall": 0.5020746887966805, "precision": 0.5330396475770925, "f1": 0.5170940170940171 }, "native": { "recall": 0.3651452282157676, "precision": 0.704, "f1": 0.4808743169398907 }, "native_grep": { "recall": 0.5892116182572614, "precision": 0.461038961038961, "f1": 0.517304189435337 }, "native_jev": { "recall": 0.5684647302904564, "precision": 0.48928571428571427, "f1": 0.525911708253359 }, "jev_added_physical": 155, "jev_added_annotated": 49, "grep_added_annotated": 54, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "repo": "vuejs/core", "language": "typescript", "grep": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 0.40816326530612246, "precision": 0.10854816824966079, "f1": 0.1714898177920686 }, "native_grep": { "recall": 0.40816326530612246, "precision": 0.08791208791208792, "f1": 0.14466546112115733 }, "native_jev": { "recall": 0.40816326530612246, "precision": 0.08465608465608465, "f1": 0.14022787028921999 }, "jev_added_physical": 208, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "repo": "microsoft/vscode", "language": "typescript", "grep": { "recall": 0.9491525423728814, "precision": 0.1696969696969697, "f1": 0.2879177377892031 }, "jev": { "recall": 0.9491525423728814, "precision": 0.21052631578947367, "f1": 0.3446153846153846 }, "native": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "native_grep": { "recall": 1.0, "precision": 0.05060034305317324, "f1": 0.0963265306122449 }, "native_jev": { "recall": 1.0, "precision": 0.06230200633579725, "f1": 0.1172962226640159 }, "jev_added_physical": 33, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "repo": "django/django", "language": "python", "grep": { "recall": 1.0, "precision": 0.27099236641221375, "f1": 0.4264264264264264 }, "jev": { "recall": 1.0, "precision": 0.27099236641221375, "f1": 0.4264264264264264 }, "native": { "recall": 1.0, "precision": 0.3169642857142857, "f1": 0.48135593220338985 }, "native_grep": { "recall": 1.0, "precision": 0.19086021505376344, "f1": 0.32054176072234764 }, "native_jev": { "recall": 1.0, "precision": 0.19086021505376344, "f1": 0.32054176072234764 }, "jev_added_physical": 148, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "repo": "ponylang/ponyc", "language": "c", "grep": { "recall": 0.13732651570489407, "precision": 0.5236768802228412, "f1": 0.2175925925925926 }, "jev": { "recall": 0.24908692476260044, "precision": 1.0, "f1": 0.39883040935672515 }, "native": { "recall": 0.36303871439006574, "precision": 0.3066008636644047, "f1": 0.33244147157190634 }, "native_grep": { "recall": 0.36303871439006574, "precision": 0.27749860413176997, "f1": 0.31455696202531647 }, "native_jev": { "recall": 0.44923301680058436, "precision": 0.35365152386428983, "f1": 0.39575289575289574 }, "jev_added_physical": 118, "jev_added_annotated": 118, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "repo": "jqlang/jq", "language": "c", "grep": { "recall": 0.4451754385964912, "precision": 0.697594501718213, "f1": 0.5435073627844712 }, "jev": { "recall": 0.42105263157894735, "precision": 0.6056782334384858, "f1": 0.49676584734799484 }, "native": { "recall": 0.8706140350877193, "precision": 0.5251322751322751, "f1": 0.6551155115511551 }, "native_grep": { "recall": 0.8947368421052632, "precision": 0.5319426336375489, "f1": 0.6672117743254292 }, "native_jev": { "recall": 0.8903508771929824, "precision": 0.530718954248366, "f1": 0.665028665028665 }, "jev_added_physical": 9, "jev_added_annotated": 9, "grep_added_annotated": 11, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "repo": "fmtlib/fmt", "language": "cpp", "grep": { "recall": 0.09326424870466321, "precision": 0.15789473684210525, "f1": 0.11726384364820847 }, "jev": { "recall": 0.09326424870466321, "precision": 0.15789473684210525, "f1": 0.11726384364820847 }, "native": { "recall": 0.5362694300518135, "precision": 0.5036496350364964, "f1": 0.5194479297365119 }, "native_grep": { "recall": 0.5854922279792746, "precision": 0.44313725490196076, "f1": 0.5044642857142857 }, "native_jev": { "recall": 0.5854922279792746, "precision": 0.44313725490196076, "f1": 0.5044642857142857 }, "jev_added_physical": 99, "jev_added_annotated": 19, "grep_added_annotated": 19, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "repo": "future-architect/vuls", "language": "go", "grep": { "recall": 0.225, "precision": 0.3898916967509025, "f1": 0.285336856010568 }, "jev": { "recall": 0.3104166666666667, "precision": 0.4901315789473684, "f1": 0.38010204081632654 }, "native": { "recall": 0.425, "precision": 0.8326530612244898, "f1": 0.5627586206896552 }, "native_grep": { "recall": 0.425, "precision": 0.4939467312348668, "f1": 0.4568868980963046 }, "native_jev": { "recall": 0.425, "precision": 0.5354330708661418, "f1": 0.4738675958188153 }, "jev_added_physical": 136, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "repo": "fasterxml/jackson", "language": "java", "grep": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.13199577613516367, "precision": 0.5605381165919282, "f1": 0.21367521367521367 }, "native": { "recall": 0.14889123548046462, "precision": 0.2256, "f1": 0.17938931297709923 }, "native_grep": { "recall": 0.14889123548046462, "precision": 0.18359375, "f1": 0.16443148688046647 }, "native_jev": { "recall": 0.2006335797254488, "precision": 0.2672292545710267, "f1": 0.22919179734620024 }, "jev_added_physical": 86, "jev_added_annotated": 49, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "repo": "darkreader/darkreader", "language": "typescript", "grep": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 }, "native_grep": { "recall": 1.0, "precision": 0.02717391304347826, "f1": 0.05291005291005291 }, "native_jev": { "recall": 1.0, "precision": 0.027247956403269755, "f1": 0.05305039787798409 }, "jev_added_physical": 224, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "repo": "nlohmann/json", "language": "cpp", "grep": { "recall": 0.23629489603024575, "precision": 0.4370629370629371, "f1": 0.3067484662576687 }, "jev": { "recall": 0.23629489603024575, "precision": 0.4370629370629371, "f1": 0.3067484662576687 }, "native": { "recall": 0.29678638941398866, "precision": 0.48606811145510836, "f1": 0.3685446009389671 }, "native_grep": { "recall": 0.33648393194706994, "precision": 0.3869565217391304, "f1": 0.35995955510616784 }, "native_jev": { "recall": 0.33648393194706994, "precision": 0.3869565217391304, "f1": 0.35995955510616784 }, "jev_added_physical": 137, "jev_added_annotated": 21, "grep_added_annotated": 21, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "repo": "catchorg/Catch2", "language": "cpp", "grep": { "recall": 0.5, "precision": 0.15151515151515152, "f1": 0.23255813953488372 }, "jev": { "recall": 0.5, "precision": 0.15151515151515152, "f1": 0.23255813953488372 }, "native": { "recall": 0.5, "precision": 0.23148148148148148, "f1": 0.31645569620253167 }, "native_grep": { "recall": 0.5, "precision": 0.14285714285714285, "f1": 0.2222222222222222 }, "native_jev": { "recall": 0.5, "precision": 0.14285714285714285, "f1": 0.2222222222222222 }, "jev_added_physical": 134, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "repo": "facebook/zstd", "language": "c", "grep": { "recall": 0.08366141732283465, "precision": 0.34274193548387094, "f1": 0.13449367088607594 }, "jev": { "recall": 0.06791338582677166, "precision": 0.2987012987012987, "f1": 0.11066559743384122 }, "native": { "recall": 0.2047244094488189, "precision": 0.19030192131747484, "f1": 0.19724988146040778 }, "native_grep": { "recall": 0.2155511811023622, "precision": 0.18496621621621623, "f1": 0.1990909090909091 }, "native_jev": { "recall": 0.22244094488188976, "precision": 0.18880534670008353, "f1": 0.20424762765476728 }, "jev_added_physical": 104, "jev_added_annotated": 18, "grep_added_annotated": 11, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "repo": "NodeBB/NodeBB", "language": "javascript", "grep": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "jev": { "recall": 0.23580034423407917, "precision": 0.4756944444444444, "f1": 0.31530494821634064 }, "native": { "recall": 0.3528399311531842, "precision": 0.2074898785425101, "f1": 0.26131293817718293 }, "native_grep": { "recall": 0.3528399311531842, "precision": 0.16748366013071894, "f1": 0.22714681440443213 }, "native_jev": { "recall": 0.3528399311531842, "precision": 0.19052044609665428, "f1": 0.24743512371756185 }, "jev_added_physical": 88, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "repo": "clap-rs/clap", "language": "rust", "grep": { "recall": 0.23723723723723725, "precision": 0.31225296442687744, "f1": 0.2696245733788396 }, "jev": { "recall": 0.21021021021021022, "precision": 0.35353535353535354, "f1": 0.263653483992467 }, "native": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 }, "native_grep": { "recall": 0.5795795795795796, "precision": 0.1634208298052498, "f1": 0.25495376486129456 }, "native_jev": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 }, "jev_added_physical": 0, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "repo": "tokio-rs/bytes", "language": "rust", "grep": { "recall": 0.34782608695652173, "precision": 0.037209302325581395, "f1": 0.06722689075630252 }, "jev": { "recall": 0.34782608695652173, "precision": 0.05, "f1": 0.08743169398907104 }, "native": { "recall": 0.5652173913043478, "precision": 0.03117505995203837, "f1": 0.05909090909090909 }, "native_grep": { "recall": 0.5652173913043478, "precision": 0.023423423423423424, "f1": 0.04498269896193772 }, "native_jev": { "recall": 0.5652173913043478, "precision": 0.026530612244897958, "f1": 0.050682261208576995 }, "jev_added_physical": 73, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "repo": "serverless/serverless", "language": "javascript", "grep": { "recall": 0.40148698884758366, "precision": 0.3103448275862069, "f1": 0.3500810372771475 }, "jev": { "recall": 0.6765799256505576, "precision": 0.527536231884058, "f1": 0.5928338762214984 }, "native": { "recall": 0.8996282527881041, "precision": 0.36890243902439024, "f1": 0.5232432432432432 }, "native_grep": { "recall": 0.8996282527881041, "precision": 0.275, "f1": 0.4212358572671889 }, "native_jev": { "recall": 0.8996282527881041, "precision": 0.3055555555555556, "f1": 0.45617342130065974 }, "jev_added_physical": 136, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "repo": "nushell/nushell", "language": "rust", "grep": { "recall": 0.07054673721340388, "precision": 0.125, "f1": 0.09019165727170236 }, "jev": { "recall": 0.30158730158730157, "precision": 0.534375, "f1": 0.3855693348365276 }, "native": { "recall": 0.7195767195767195, "precision": 0.1348314606741573, "f1": 0.22710826607291956 }, "native_grep": { "recall": 0.7195767195767195, "precision": 0.131697869593286, "f1": 0.22264665757162347 }, "native_jev": { "recall": 0.7195767195767195, "precision": 0.13229571984435798, "f1": 0.22350041084634348 }, "jev_added_physical": 58, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "repo": "google/gson", "language": "java", "grep": { "recall": 0.33568904593639576, "precision": 0.7116104868913857, "f1": 0.4561824729891957 }, "jev": { "recall": 0.33568904593639576, "precision": 0.7116104868913857, "f1": 0.4561824729891957 }, "native": { "recall": 0.46819787985865724, "precision": 0.828125, "f1": 0.5981941309255079 }, "native_grep": { "recall": 0.5706713780918727, "precision": 0.7783132530120482, "f1": 0.6585117227319062 }, "native_jev": { "recall": 0.5706713780918727, "precision": 0.7783132530120482, "f1": 0.6585117227319062 }, "jev_added_physical": 95, "jev_added_annotated": 58, "grep_added_annotated": 58, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "repo": "cli/cli", "language": "go", "grep": { "recall": 0.9285714285714286, "precision": 0.043333333333333335, "f1": 0.08280254777070063 }, "jev": { "recall": 0.9285714285714286, "precision": 0.04234527687296417, "f1": 0.08099688473520249 }, "native": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 }, "native_grep": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 }, "native_jev": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 }, "jev_added_physical": 44, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "repo": "alibaba/fastjson2", "language": "java", "grep": { "recall": 0.375, "precision": 0.16216216216216217, "f1": 0.22641509433962265 }, "jev": { "recall": 0.375, "precision": 0.16216216216216217, "f1": 0.22641509433962265 }, "native": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "native_grep": { "recall": 0.375, "precision": 0.07079646017699115, "f1": 0.11910669975186104 }, "native_jev": { "recall": 0.375, "precision": 0.07079646017699115, "f1": 0.11910669975186104 }, "jev_added_physical": 88, "jev_added_annotated": 24, "grep_added_annotated": 24, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "repo": "flipt-io/flipt", "language": "go", "grep": { "recall": 0.20809248554913296, "precision": 0.3037974683544304, "f1": 0.2469982847341338 }, "jev": { "recall": 0.20809248554913296, "precision": 0.3037974683544304, "f1": 0.2469982847341338 }, "native": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.1780693533270853 }, "native_grep": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.1780693533270853 }, "native_jev": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.1780693533270853 }, "jev_added_physical": 0, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "repo": "axios/axios", "language": "javascript", "grep": { "recall": 0.030330882352941176, "precision": 0.08943089430894309, "f1": 0.045298558682223745 }, "jev": { "recall": 0.02665441176470588, "precision": 0.07611548556430446, "f1": 0.039482641252552755 }, "native": { "recall": 0.23621323529411764, "precision": 0.7427745664739884, "f1": 0.3584379358437936 }, "native_grep": { "recall": 0.23621323529411764, "precision": 0.4269102990033223, "f1": 0.30414201183431955 }, "native_jev": { "recall": 0.23621323529411764, "precision": 0.4165316045380875, "f1": 0.30146627565982403 }, "jev_added_physical": 271, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "first_source_24", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "repo": "pydata/xarray", "language": "python", "grep": { "recall": 0.6666666666666666, "precision": 0.5545454545454546, "f1": 0.6054590570719603 }, "jev": { "recall": 0.6666666666666666, "precision": 0.5545454545454546, "f1": 0.6054590570719603 }, "native": { "recall": 0.5901639344262295, "precision": 0.4044943820224719, "f1": 0.48 }, "native_grep": { "recall": 0.6830601092896175, "precision": 0.35410764872521244, "f1": 0.4664179104477612 }, "native_jev": { "recall": 0.6830601092896175, "precision": 0.35410764872521244, "f1": 0.4664179104477612 }, "jev_added_physical": 86, "jev_added_annotated": 17, "grep_added_annotated": 17, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "repo": "microsoft/vscode", "language": "typescript", "grep": { "recall": 0.9491525423728814, "precision": 0.1696969696969697, "f1": 0.2879177377892031 }, "jev": { "recall": 0.9491525423728814, "precision": 0.21052631578947367, "f1": 0.3446153846153846 }, "native": { "recall": 1.0, "precision": 0.04223335719398712, "f1": 0.08104395604395605 }, "native_grep": { "recall": 1.0, "precision": 0.03626306084818685, "f1": 0.06998813760379596 }, "native_jev": { "recall": 1.0, "precision": 0.04134548002803083, "f1": 0.07940780619111709 }, "jev_added_physical": 30, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "repo": "django/django", "language": "python", "grep": { "recall": 1.0, "precision": 0.27099236641221375, "f1": 0.4264264264264264 }, "jev": { "recall": 1.0, "precision": 0.27099236641221375, "f1": 0.4264264264264264 }, "native": { "recall": 1.0, "precision": 0.21846153846153846, "f1": 0.35858585858585856 }, "native_grep": { "recall": 1.0, "precision": 0.16173120728929385, "f1": 0.2784313725490196 }, "native_jev": { "recall": 1.0, "precision": 0.16173120728929385, "f1": 0.2784313725490196 }, "jev_added_physical": 114, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "repo": "jqlang/jq", "language": "c", "grep": { "recall": 0.4451754385964912, "precision": 0.697594501718213, "f1": 0.5435073627844712 }, "jev": { "recall": 0.42105263157894735, "precision": 0.6056782334384858, "f1": 0.49676584734799484 }, "native": { "recall": 0.8706140350877193, "precision": 0.5357624831309041, "f1": 0.6633249791144528 }, "native_grep": { "recall": 0.8947368421052632, "precision": 0.5425531914893617, "f1": 0.6754966887417219 }, "native_jev": { "recall": 0.8903508771929824, "precision": 0.5413333333333333, "f1": 0.6733001658374793 }, "jev_added_physical": 9, "jev_added_annotated": 9, "grep_added_annotated": 11, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "repo": "nlohmann/json", "language": "cpp", "grep": { "recall": 0.23629489603024575, "precision": 0.4370629370629371, "f1": 0.3067484662576687 }, "jev": { "recall": 0.23629489603024575, "precision": 0.4370629370629371, "f1": 0.3067484662576687 }, "native": { "recall": 0.2570888468809074, "precision": 0.3646112600536193, "f1": 0.30155210643015523 }, "native_grep": { "recall": 0.2570888468809074, "precision": 0.27530364372469635, "f1": 0.26588465298142716 }, "native_jev": { "recall": 0.2570888468809074, "precision": 0.27530364372469635, "f1": 0.26588465298142716 }, "jev_added_physical": 121, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "repo": "tokio-rs/bytes", "language": "rust", "grep": { "recall": 0.34782608695652173, "precision": 0.037209302325581395, "f1": 0.06722689075630252 }, "jev": { "recall": 0.34782608695652173, "precision": 0.05, "f1": 0.08743169398907104 }, "native": { "recall": 0.7391304347826086, "precision": 0.04028436018957346, "f1": 0.07640449438202247 }, "native_grep": { "recall": 0.7391304347826086, "precision": 0.030357142857142857, "f1": 0.058319039451114926 }, "native_jev": { "recall": 0.7391304347826086, "precision": 0.033663366336633666, "f1": 0.06439393939393939 }, "jev_added_physical": 83, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "repo": "serverless/serverless", "language": "javascript", "grep": { "recall": 0.40148698884758366, "precision": 0.3103448275862069, "f1": 0.3500810372771475 }, "jev": { "recall": 0.6765799256505576, "precision": 0.527536231884058, "f1": 0.5928338762214984 }, "native": { "recall": 0.8773234200743495, "precision": 0.36251920122887865, "f1": 0.5130434782608696 }, "native_grep": { "recall": 0.8773234200743495, "precision": 0.2806183115338882, "f1": 0.4252252252252252 }, "native_jev": { "recall": 0.9033457249070632, "precision": 0.3119383825417201, "f1": 0.4637404580152672 }, "jev_added_physical": 128, "jev_added_annotated": 7, "grep_added_annotated": 0, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "repo": "google/gson", "language": "java", "grep": { "recall": 0.33568904593639576, "precision": 0.7116104868913857, "f1": 0.4561824729891957 }, "jev": { "recall": 0.33568904593639576, "precision": 0.7116104868913857, "f1": 0.4561824729891957 }, "native": { "recall": 0.4876325088339223, "precision": 0.8363636363636363, "f1": 0.6160714285714286 }, "native_grep": { "recall": 0.5618374558303887, "precision": 0.7718446601941747, "f1": 0.6503067484662577 }, "native_jev": { "recall": 0.5618374558303887, "precision": 0.7718446601941747, "f1": 0.6503067484662577 }, "jev_added_physical": 82, "jev_added_annotated": 42, "grep_added_annotated": 42, "official_baseline_counts_exact": true }, { "cohort": "late_opposite_8", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "repo": "cli/cli", "language": "go", "grep": { "recall": 0.9285714285714286, "precision": 0.043333333333333335, "f1": 0.08280254777070063 }, "jev": { "recall": 0.9285714285714286, "precision": 0.04234527687296417, "f1": 0.08099688473520249 }, "native": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.08358208955223881 }, "native_grep": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 }, "native_jev": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 }, "jev_added_physical": 64, "jev_added_annotated": 0, "grep_added_annotated": 0, "official_baseline_counts_exact": true } ], "rerank_development_gate_passed": true, "static_injection_gate_passed": false, "integrity_audit": { "scope": "Post-run source/permutation/cost integrity audit and candidate coverage diagnostic. No policy or selection change.", "all_candidate_permutations_exact": true, "all_delivered_source_exact": true, "all_requests_and_cost_receipts_exact": true, "all_frozen_hashes_match": true, "tasks": 24, "candidate_pool_macro_recall": 0.4088892470451529, "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "pool_recall": 0.5394190871369294, "candidate_union_lines": 341, "gold_lines": 241, "ranked_delivery_lines": 227 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "pool_recall": 0.0, "candidate_union_lines": 1607, "gold_lines": 196, "ranked_delivery_lines": 369 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "pool_recall": 0.9491525423728814, "candidate_union_lines": 2004, "gold_lines": 59, "ranked_delivery_lines": 266 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "pool_recall": 1.0, "candidate_union_lines": 262, "gold_lines": 71, "ranked_delivery_lines": 262 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "pool_recall": 0.42804967129291455, "candidate_union_lines": 1280, "gold_lines": 1369, "ranked_delivery_lines": 341 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "pool_recall": 0.5197368421052632, "candidate_union_lines": 373, "gold_lines": 456, "ranked_delivery_lines": 317 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "pool_recall": 0.09326424870466321, "candidate_union_lines": 228, "gold_lines": 386, "ranked_delivery_lines": 228 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "pool_recall": 0.43333333333333335, "candidate_union_lines": 1398, "gold_lines": 480, "ranked_delivery_lines": 304 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "pool_recall": 0.2006335797254488, "candidate_union_lines": 747, "gold_lines": 947, "ranked_delivery_lines": 223 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "pool_recall": 0.0, "candidate_union_lines": 1289, "gold_lines": 20, "ranked_delivery_lines": 269 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "pool_recall": 0.23629489603024575, "candidate_union_lines": 286, "gold_lines": 529, "ranked_delivery_lines": 286 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "pool_recall": 0.5, "candidate_union_lines": 330, "gold_lines": 100, "ranked_delivery_lines": 330 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "pool_recall": 0.2017716535433071, "candidate_union_lines": 1283, "gold_lines": 1016, "ranked_delivery_lines": 231 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "pool_recall": 0.35800344234079173, "candidate_union_lines": 1774, "gold_lines": 581, "ranked_delivery_lines": 288 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "pool_recall": 0.27627627627627627, "candidate_union_lines": 350, "gold_lines": 333, "ranked_delivery_lines": 198 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "pool_recall": 0.34782608695652173, "candidate_union_lines": 284, "gold_lines": 23, "ranked_delivery_lines": 160 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "pool_recall": 0.7657992565055762, "candidate_union_lines": 1337, "gold_lines": 269, "ranked_delivery_lines": 345 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "pool_recall": 0.4056437389770723, "candidate_union_lines": 1014, "gold_lines": 567, "ranked_delivery_lines": 320 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "pool_recall": 0.33568904593639576, "candidate_union_lines": 267, "gold_lines": 566, "ranked_delivery_lines": 267 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "pool_recall": 0.9285714285714286, "candidate_union_lines": 436, "gold_lines": 14, "ranked_delivery_lines": 307 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "pool_recall": 0.375, "candidate_union_lines": 148, "gold_lines": 64, "ranked_delivery_lines": 148 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "pool_recall": 0.20809248554913296, "candidate_union_lines": 237, "gold_lines": 346, "ranked_delivery_lines": 237 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "pool_recall": 0.04411764705882353, "candidate_union_lines": 526, "gold_lines": 1088, "ranked_delivery_lines": 381 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "pool_recall": 0.6666666666666666, "candidate_union_lines": 220, "gold_lines": 183, "ranked_delivery_lines": 220 } ], "response_models": [ "typesafe/jev-1.13-20260917" ], "max_request_cost_usd": 0.000427476, "new_model_calls": 0, "new_api_charges_usd": 0 }, "model": "typesafe/jev-1.13", "response_models": [ "typesafe/jev-1.13-20260917" ], "delivery": { "candidates": 40, "top_k": 10, "token_budget": 4096, "whole_source_lines": true, "tests_included": true }, "ranking": "Existing four-level code relevance rubric, issue prefix <=16000 chars, candidate content <=3000 chars, batches of ten, descending mean score, original order on ties. No thresholds, gold, patch or native observations in ranking.", "budget": { "api_reservation_usd": 1, "per_call_reservation_usd": 0.01, "max_calls": 96, "retries": 0, "timeout_s": 30, "on_error_or_unknown_usage": "Persist receipt and stop all later calls; no model or ordering fallback counted as Jev." }, "uncertainty": { "method": "repository-cluster paired bootstrap, exploratory 95%", "draws": 5000, "seed": 20261017 }, "runtime_commit": "77968aaa70ca58164216d488f25cb24642b206b2", "implementation_sha256": { "scripts/benchmark_context_agent.py": "7cf8372c78449d30c327e014ac04f9e54599bdb8586c1bd0b68904d3f6f75621", "scripts/benchmark_context_claude.py": "9e3ec04a05e4e87c7f8885c4cfb99a2aa920dad5b0a6fd2f66c243e072a105de", "scripts/contextbench_graph_navigation.py": "d188b4c0edeaba024d8aef958c00b237d212a3f67a76923c9e6d4a446be03bac", "scripts/benchmark_context_serving.py": "6fa40f7ad145938efcb1edceddcad864b2febe0c1c61dda315d9cd5122048fe2", "codenib/graph/code_graph.py": "70b393a0218b17598e33df9727b6def9ee13325470a1bcd3f2a5e7d68c9f2a27", "codenib/graph/traverse_graph.py": "4da1112c051a76276e1e5414c07fe8cae0acc94aadce86b9fd5c50c24f797854", "scripts/benchmark_decision_loop.py": "39c016e92ac4d8adf6aad96ff1fbffffc6df47f8b6d35fbe3afe50811a3ca138", "scripts/benchmark_graph_decisions.py": "56d3c2db81a716f46bec603620e51c3ac5ee3bf4ef790023e362e859e4a5216d", "codenib/agent/runner.py": "4036c53f8df96b41090b3592f15fe4bad737cef29409552fffb8afff0def9479", "codenib/agent/agent_types.py": "abf5997210ddbb4893ccba2625343058fe8da3cc781afe135337924379821869", "codenib/agent/history.py": "a5029f6750085be346de9f4401ba514eae34360e0cbe1d3638d05c6e9b7c7487", "codenib/agent/tools/defaults.py": "58c27573798677fc83caec0791ebf8c75058adfcc037f8205fcd3abc38d3e3ca", "codenib/agent/boundary.py": "5a2abe8543dcc4b63a2277e1a490eb41796b57b5af6a3f27d74738a212c70ba4", "codenib/agent/tool_schema.py": "537b63bf07d7d1e1d304499a5f949e6b94eafb9da29a757445feeb89fb453302", "codenib/eval/benchmarks/contextbench.py": "1d6d1718d24abaccb46e4f6743ac8033e145628fbc55ceec85d07c0390b39609", "codenib/eval/benchmarks/contextbench_runner.py": "6e7e40026c6d7006153fe3cc69acf72393340010d72c78c96f43c83164656fa3", "codenib/eval/benchmarks/_contextbench_evaluate.py": "26b82cc9e800759469c52d972ac2f4b5cfda69f052601841f026f05e28cf222c", "codenib/types.py": "8addb60456bf079e09e27160667800b909ed340962eca6fda953118cccba7129", "codenib/agent/rerank_agent.py": "d72b3799d9ff0a4326a1ed78969862966a8724d6e98c8a4111635d3316bea465", "codenib/agent/decision_rerank.py": "fbfb3d30a0d9b2da364d5c46d52e8dba642204f0a93698f30ce84052a6d9edc8", "codenib/llm/decisions.py": "89d5c79ee8005d503cf7f0fbb152d4f9d2ece60dd1ca0cdac16b7de36cca7ed2", "scripts/benchmark_jev_retrieval.py": "7ec3248e96c323f0914df0f54ebaa41fa8577f8df7428257fd0aaf9005522be4", "codenib/code_chunking/__init__.py": "e49eb5d1ff9d2328dd1a9bb4191457594557bde46d7e5d4143380202b615ac68", "codenib/code_chunking/base.py": "32e8637d8f3888bfa15ce33892faca4d0ca4da32be1806b0d8288dc86fe92b6e", "codenib/code_chunking/cpp_chunker.py": "5a29bc267bc4d8946d7983590e87f14b838bb2f34ccba41927d4cc3302f1f9ef", "codenib/code_chunking/csharp_chunker.py": "82990bce7e260c434d50d6be16e786991f9c314cf058287f791ab6e4f92f6f54", "codenib/code_chunking/go_chunker.py": "c9948c5209ee68c6fdd1609e8cb2485ed424ddfbf9248f87c4593fb14fb970a6", "codenib/code_chunking/java_chunker.py": "7d53f12d8e99b91d0a23bae30b85cdb780ad67cc30c82d2c6d17eae046940852", "codenib/code_chunking/js_chunker.py": "b4436ae1df3902e47b0a97b84f0275fb760ca902c7e09c32791dba76a1de8989", "codenib/code_chunking/kotlin_chunker.py": "a1d9456e99953d589ca8d3a053d1d2b7afb987ebbd6706cbbcfc3ccd6dbeea6d", "codenib/code_chunking/lua_chunker.py": "58cd2ea7fed1997d3d1834405ca91d47c138e836d821ac7df5dbc3a511f67333", "codenib/code_chunking/php_chunker.py": "5a0b7990ce5c0b0e941d2953e1aeac31197835c8ffb61a34213b8413555b153b", "codenib/code_chunking/python_chunker.py": "cffd6a7d80dec2842012e664dca26dedc0d84093d4239ec064f4111328bea81a", "codenib/code_chunking/ruby_chunker.py": "fc9b9dbe4dd59d8be98980801bd545efcf31fc9756d97b8b1d42522d11038761", "codenib/code_chunking/rust_chunker.py": "a828e28038a59aba026be4cd15ce4f468786486213130f18fc9a61d8dc8f5e1a", "codenib/code_chunking/scala_chunker.py": "ac778f886df536f3b3f9e0220c582cfb4145c9fe72d642a323ebe11c48d46d06", "codenib/code_chunking/swift_chunker.py": "b81496687cc3faff865005f02fc77d7ad691d945602f0ae318e665b7b34db135", "codenib/code_chunker.py": "06e170ff40486c04011204ad26d9aad5b6bc730b10c816a32529b72f3a60c4c6", "codenib/languages.py": "ff2986d883523071ec3473a7ebde07c663a3b8fd4b971ba780af7882a5776d99" }, "candidate_origin": "Frozen original Opus 100-task one-shot grep plans/pools; no new planner, graph or adaptive agent. Only the same disclosed 24 IDs.", "original_planner_models": [ "anthropic/claude-opus-5.5" ], "candidate_count": 484, "new_planner_calls": 0, "conclusion": "The 4K same-pool planned-grep comparison passes the ranking development gate. Keep the positive selection signal separate from an online effect: static source addition still reduces F1. Stop BM25 variants and prioritize selective grep/Jev delivery inside a fresh matched agent protocol without graphs. Independent 32 remains sealed; no full-500 launch." }, "online_grep_jev_development": { "scope": "Fresh online retrieval-agent development controls. Not repair Pass@1, independent validation, or an official leaderboard submission.", "protocol_sha256": "c720694d303ee1ca7fa2170f935766ceeb2791282951454c56f6b88f025b7fe2", "output_receipts_sha256": "45a0c3a6ff65d4441ef5eb42b6e0b32f4d6d584dddec0c0c4fefc9764d05bb9d", "summaries": { "native": { "n": 8, "arms": { "trajectory": { "recall": 0.42359182915762567, "precision": 0.3788476649053235, "f1": 0.3679707734297141, "official_valid": 8 }, "served_16384": { "recall": 0.4228580106589719, "precision": 0.39068074172096096, "f1": 0.3744360668181256, "official_valid": 8 }, "served_32768": { "recall": 0.42307315695845554, "precision": 0.3787438157720305, "f1": 0.3676268618345067, "official_valid": 8 } }, "usage": { "billing_route": "anthropic_api_key", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.6448555, "missing_metering_cases": 0, "jev_api_cost_usd": 0, "jev_api_calls": 0, "jev_unknown_usage_calls": 0, "statuses": { "ok": 8 }, "cases": 8 } }, "chunks": { "n": 8, "arms": { "trajectory": { "recall": 0.4728708743584402, "precision": 0.3449488654405801, "f1": 0.3501512339935523, "official_valid": 8 }, "served_16384": { "recall": 0.4575496150612773, "precision": 0.351683719781227, "f1": 0.3534863444494245, "official_valid": 8 }, "served_32768": { "recall": 0.4728708743584402, "precision": 0.34504094205055214, "f1": 0.3502337347062103, "official_valid": 8 } }, "usage": { "billing_route": "anthropic_api_key", "actual_bill_usd": null, "sdk_reported_cost_usd": 2.0563018, "missing_metering_cases": 0, "jev_api_cost_usd": 0.0, "jev_api_calls": 0, "jev_unknown_usage_calls": 0, "statuses": { "ok": 8 }, "cases": 8 } }, "jev": { "n": 8, "arms": { "trajectory": { "recall": 0.4097126239086658, "precision": 0.29461198855957627, "f1": 0.31940212027763265, "official_valid": 8 }, "served_16384": { "recall": 0.40949747760918215, "precision": 0.29556292868064216, "f1": 0.3202382236508917, "official_valid": 8 }, "served_32768": { "recall": 0.4097126239086658, "precision": 0.294656105424624, "f1": 0.31945249212695315, "official_valid": 8 } }, "usage": { "billing_route": "anthropic_api_key", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.7896918999999998, "missing_metering_cases": 0, "jev_api_cost_usd": 0.005520479999999999, "jev_api_calls": 55, "jev_unknown_usage_calls": 0, "statuses": { "ok": 8 }, "cases": 8 } } }, "comparisons": { "native": { "recall": { "mean_difference": -0.013879205248959844, "exploratory_95_interval": [ -0.08253134629314507, 0.03802743521026851 ] }, "precision": { "mean_difference": -0.08423567634574723, "exploratory_95_interval": [ -0.18409407795540872, -0.006579258699113934 ] }, "f1": { "mean_difference": -0.04856865315208148, "exploratory_95_interval": [ -0.09045826302253063, -0.014770475107669214 ] } }, "chunks": { "recall": { "mean_difference": -0.0631582504497744, "exploratory_95_interval": [ -0.11545866100396786, -0.02162870483786111 ] }, "precision": { "mean_difference": -0.05033687688100384, "exploratory_95_interval": [ -0.16803406423013373, 0.033291948816028955 ] }, "f1": { "mean_difference": -0.030749113715919634, "exploratory_95_interval": [ -0.0763934273997886, 0.0014634188299206888 ] } } }, "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "metrics": { "recall": 0.42738589211618255, "precision": 0.6912751677852349, "f1": 0.5282051282051282 }, "grep_calls": 3, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.0835748, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 10, "sdk_reported_turns": 8, "tool_calls": { "grep": 3, "read": 3, "glob": 1 }, "wall_s": 22.508819550042972, "source_lines": 149 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "metrics": { "recall": 0.25510204081632654, "precision": 0.072992700729927, "f1": 0.11350737797956868 }, "grep_calls": 5, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.19352370000000002, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 22, "sdk_reported_turns": 15, "tool_calls": { "grep": 5, "read": 9 }, "wall_s": 52.83518237003591, "source_lines": 685 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "metrics": { "recall": 0.5237399561723886, "precision": 0.4076179647527004, "f1": 0.4584398976982097 }, "grep_calls": 10, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.3881817, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 39, "sdk_reported_turns": 26, "tool_calls": { "grep": 10, "glob": 1, "read": 14 }, "wall_s": 75.0879469870124, "source_lines": 1759 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "metrics": { "recall": 0.5362694300518135, "precision": 0.5123762376237624, "f1": 0.5240506329113925 }, "grep_calls": 3, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.10767510000000001, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 9, "sdk_reported_turns": 6, "tool_calls": { "grep": 3, "read": 2 }, "wall_s": 24.71202497498598, "source_lines": 404 }, { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "metrics": { "recall": 0.4875, "precision": 0.7335423197492164, "f1": 0.5857321652065082 }, "grep_calls": 4, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1183334, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 14, "sdk_reported_turns": 11, "tool_calls": { "grep": 4, "read": 6 }, "wall_s": 31.601712243049406, "source_lines": 319 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "metrics": { "recall": 0.20485744456177402, "precision": 0.26216216216216215, "f1": 0.22999407231772376 }, "grep_calls": 6, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1850284, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 21, "sdk_reported_turns": 15, "tool_calls": { "read": 8, "grep": 6 }, "wall_s": 47.26539044594392, "source_lines": 740 }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "metrics": { "recall": 0.35628227194492257, "precision": 0.1830238726790451, "f1": 0.24182242990654207 }, "grep_calls": 10, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.2955288, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 28, "sdk_reported_turns": 20, "tool_calls": { "read": 9, "grep": 10 }, "wall_s": 61.976043528993614, "source_lines": 1131 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "metrics": { "recall": 0.5975975975975976, "precision": 0.16779089376053963, "f1": 0.26201448321263987 }, "grep_calls": 8, "ranked_views": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.27300959999999996, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 28, "sdk_reported_turns": 20, "tool_calls": { "grep": 8, "read": 11 }, "wall_s": 56.14357372093946, "source_lines": 1186 }, { "arm": "chunks", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "metrics": { "recall": 0.5269709543568465, "precision": 0.8410596026490066, "f1": 0.6479591836734694 }, "grep_calls": 2, "ranked_views": 2, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.0806578, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 5, "sdk_reported_turns": 3, "tool_calls": { "grep": 2 }, "wall_s": 19.846476412029006, "source_lines": 151 }, { "arm": "chunks", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "metrics": { "recall": 0.5153061224489796, "precision": 0.0832646331409728, "f1": 0.1433640880056778 }, "grep_calls": 8, "ranked_views": 7, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.3167561, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 21, "sdk_reported_turns": 15, "tool_calls": { "grep": 8, "read": 5, "glob": 1 }, "wall_s": 53.519105145009235, "source_lines": 1213 }, { "arm": "chunks", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "metrics": { "recall": 0.34550766983199416, "precision": 0.2986111111111111, "f1": 0.3203521842194379 }, "grep_calls": 12, "ranked_views": 9, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.39048150000000004, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 31, "sdk_reported_turns": 21, "tool_calls": { "grep": 12, "read": 6, "glob": 2 }, "wall_s": 68.226258983952, "source_lines": 1584 }, { "arm": "chunks", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "metrics": { "recall": 0.5362694300518135, "precision": 0.5061124694376528, "f1": 0.5207547169811321 }, "grep_calls": 3, "ranked_views": 2, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.10545469999999998, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 7, "sdk_reported_turns": 5, "tool_calls": { "grep": 3, "read": 1 }, "wall_s": 24.01812545605935, "source_lines": 409 }, { "arm": "chunks", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "metrics": { "recall": 0.55625, "precision": 0.37186629526462395, "f1": 0.44574290484140233 }, "grep_calls": 4, "ranked_views": 3, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1564024, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 10, "sdk_reported_turns": 6, "tool_calls": { "grep": 4, "read": 1 }, "wall_s": 31.407650568988174, "source_lines": 718 }, { "arm": "chunks", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "metrics": { "recall": 0.20380147835269272, "precision": 0.3634651600753296, "f1": 0.26116373477672533 }, "grep_calls": 7, "ranked_views": 6, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.2300915, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 20, "sdk_reported_turns": 12, "tool_calls": { "grep": 7, "read": 3, "glob": 1 }, "wall_s": 42.20836838602554, "source_lines": 531 }, { "arm": "chunks", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "metrics": { "recall": 0.49225473321858865, "precision": 0.16361556064073227, "f1": 0.24559896951481322 }, "grep_calls": 12, "ranked_views": 9, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.42820639999999993, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 30, "sdk_reported_turns": 22, "tool_calls": { "read": 8, "grep": 12, "glob": 1 }, "wall_s": 91.48264361603651, "source_lines": 1748 }, { "arm": "chunks", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "metrics": { "recall": 0.6066066066066066, "precision": 0.13159609120521173, "f1": 0.2162740899357602 }, "grep_calls": 10, "ranked_views": 9, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.34825139999999993, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 26, "sdk_reported_turns": 16, "tool_calls": { "grep": 10, "read": 5 }, "wall_s": 71.15520703000948, "source_lines": 1535 }, { "arm": "jev", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "metrics": { "recall": 0.5269709543568465, "precision": 0.42474916387959866, "f1": 0.4703703703703703 }, "grep_calls": 3, "ranked_views": 1, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1014428, "jev_api_calls": 1, "jev_cost_usd": 0.000139566, "assistant_events": 11, "sdk_reported_turns": 8, "tool_calls": { "grep": 3, "glob": 1, "read": 3 }, "wall_s": 23.36581815604586, "source_lines": 299 }, { "arm": "jev", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "metrics": { "recall": 0.30612244897959184, "precision": 0.07083825265643448, "f1": 0.11505273250239693 }, "grep_calls": 5, "ranked_views": 4, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.2104763, "jev_api_calls": 7, "jev_cost_usd": 0.0007204259999999999, "assistant_events": 21, "sdk_reported_turns": 15, "tool_calls": { "grep": 5, "read": 9 }, "wall_s": 46.1273370210547, "source_lines": 847 }, { "arm": "jev", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "metrics": { "recall": 0.295836376917458, "precision": 0.420997920997921, "f1": 0.34749034749034746 }, "grep_calls": 8, "ranked_views": 6, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.28681049999999997, "jev_api_calls": 9, "jev_cost_usd": 0.0008709119999999999, "assistant_events": 24, "sdk_reported_turns": 15, "tool_calls": { "grep": 8, "read": 5, "glob": 1 }, "wall_s": 57.471231493982486, "source_lines": 962 }, { "arm": "jev", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "metrics": { "recall": 0.5362694300518135, "precision": 0.49640287769784175, "f1": 0.5155666251556662 }, "grep_calls": 3, "ranked_views": 3, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1118277, "jev_api_calls": 3, "jev_cost_usd": 0.00024242399999999998, "assistant_events": 8, "sdk_reported_turns": 5, "tool_calls": { "grep": 3, "read": 1 }, "wall_s": 26.252606361056678, "source_lines": 417 }, { "arm": "jev", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "metrics": { "recall": 0.48125, "precision": 0.38372093023255816, "f1": 0.42698706099815165 }, "grep_calls": 3, "ranked_views": 3, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1423074, "jev_api_calls": 3, "jev_cost_usd": 0.000296982, "assistant_events": 9, "sdk_reported_turns": 6, "tool_calls": { "grep": 3, "read": 2 }, "wall_s": 30.61699365102686, "source_lines": 602 }, { "arm": "jev", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "metrics": { "recall": 0.19535374868004224, "precision": 0.24601063829787234, "f1": 0.21777516185991758 }, "grep_calls": 4, "ranked_views": 3, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.20420149999999998, "jev_api_calls": 4, "jev_cost_usd": 0.0004898459999999999, "assistant_events": 15, "sdk_reported_turns": 11, "tool_calls": { "read": 6, "grep": 4 }, "wall_s": 35.341841981047764, "source_lines": 752 }, { "arm": "jev", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "metrics": { "recall": 0.3683304647160069, "precision": 0.17174959871589085, "f1": 0.234263820470717 }, "grep_calls": 10, "ranked_views": 9, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.36851009999999995, "jev_api_calls": 14, "jev_cost_usd": 0.0016803359999999997, "assistant_events": 28, "sdk_reported_turns": 20, "tool_calls": { "read": 9, "grep": 10 }, "wall_s": 79.9044428669149, "source_lines": 1246 }, { "arm": "jev", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "metrics": { "recall": 0.5675675675675675, "precision": 0.14242652599849284, "f1": 0.22771084337349398 }, "grep_calls": 10, "ranked_views": 9, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.3641156, "jev_api_calls": 14, "jev_cost_usd": 0.0010799879999999998, "assistant_events": 26, "sdk_reported_turns": 17, "tool_calls": { "grep": 10, "read": 6 }, "wall_s": 81.08305041294079, "source_lines": 1327 } ], "gates": { "all_complete": true, "known_usage": true, "one_model_and_api_route": true, "all_replies_followed_by_assistant": true, "jev_greater_recall_vs_both": false, "jev_nondecreasing_f1_vs_both": false }, "development_gate_passed": false, "known_anthropic_sdk_usd": 5.4908491999999995, "known_jev_api_usd": 0.005520479999999999, "models": [ "claude-opus-5-5" ], "routes": [ "ANTHROPIC_API_KEY" ], "all_hashes_and_source_checks_passed": true, "protocol": { "selection": "First task per language in the existing disclosed 24-task order, independent of scores or source availability; eight tasks, no replacements. Sealed independent 32 excluded.", "implementation_commit": "cf4bc60b84622ebe5bf9c89c9d0f47fbc18463c0", "model": "claude-opus-5-5", "effort": "medium", "max_turns": 20, "case_timeout_s": 600, "source_policy": "native", "grep": { "candidate_limit": 40, "block_limit": 10, "response_tokens": 4096, "response_chars": 15900, "candidate_chars": 3000, "candidate_lines": 100 }, "selection_policy": "Actual native grep hits select files and bounded source chunks. Chunks preserves that order; Jev scores the identical per-query candidate construction with the original issue, batches of ten, stable ties. Read/glob unchanged; no graph, index, BM25 or planner. Agent queries may diverge after feedback.", "primary": "Jev versus native: full source-selection policy effect. Jev versus chunks: online effect of ranking with identical tools and packing; adaptive queries can diverge.", "development_gate": "All 24 positions finish with known usage and one model/route, verified delivered source, strictly higher macro trajectory Recall and nondecreasing F1 for Jev versus both controls. Eight-task intervals are exploratory; no automatic promotion to 500.", "next_e2e_gate": "A passing candidate must be checked through the existing mature solver and native tests on predeclared compatible development tasks; retrieval scores alone are not end-to-end gains.", "analysis": "Freeze outputs before official scoring. Keep all eight tasks, failures and not-run slots. Paired repository bootstrap 5000 draws, seed 20261017. Exact source only; canonical official gold paths." }, "billing": { "route": "anthropic_api_key", "batch_reservation_usd": 10, "account_bill_or_credit_balance_verified": false }, "selection_diagnostic": { "scope": "Post-result fixed-query selection audit; no rerun, new policy, changed metric or counterfactual agent claim.", "primary_analysis_sha256": "0ad541a9fe264906bc641c5d0eacda1e9fe3aba4a666ae212754e7d0620b2f0d", "diagnostic_driver_sha256": "92ccf2cda065a202f57da5ebce80c6e79d2dac34291608d0592930fa6020247b", "views": 38, "reordered_views": 31, "views_with_changed_source_selection": 12, "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "candidate_count": 9, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 141, "original_order_returned_lines": 141 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "candidate_count": 24, "reordered": true, "source_selection_changed": true, "added_lines": 137, "removed_lines": 162, "returned_lines": 190, "original_order_returned_lines": 215 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "candidate_count": 4, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 4, "original_order_returned_lines": 4 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "candidate_count": 16, "reordered": true, "source_selection_changed": true, "added_lines": 4, "removed_lines": 13, "returned_lines": 109, "original_order_returned_lines": 118 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "candidate_count": 2, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 112, "original_order_returned_lines": 112 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "candidate_count": 3, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 3, "original_order_returned_lines": 3 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "candidate_count": 3, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 159, "original_order_returned_lines": 159 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "candidate_count": 4, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 175, "original_order_returned_lines": 175 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "candidate_count": 1, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 83, "original_order_returned_lines": 83 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "candidate_count": 4, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 237, "original_order_returned_lines": 237 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 11, "reordered": true, "source_selection_changed": true, "added_lines": 1, "removed_lines": 1, "returned_lines": 142, "original_order_returned_lines": 142 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 32, "reordered": true, "source_selection_changed": true, "added_lines": 221, "removed_lines": 23, "returned_lines": 240, "original_order_returned_lines": 42 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 2, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 37, "original_order_returned_lines": 37 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 1, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 51, "original_order_returned_lines": 51 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 3, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 150, "original_order_returned_lines": 150 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 2, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 17, "original_order_returned_lines": 17 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 3, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 7, "original_order_returned_lines": 7 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 15, "reordered": true, "source_selection_changed": true, "added_lines": 4, "removed_lines": 4, "returned_lines": 10, "original_order_returned_lines": 10 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "candidate_count": 5, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 138, "original_order_returned_lines": 138 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 20, "reordered": true, "source_selection_changed": true, "added_lines": 235, "removed_lines": 232, "returned_lines": 244, "original_order_returned_lines": 241 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 6, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 187, "original_order_returned_lines": 187 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 10, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 10, "original_order_returned_lines": 10 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 15, "reordered": true, "source_selection_changed": true, "added_lines": 154, "removed_lines": 171, "returned_lines": 239, "original_order_returned_lines": 256 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 31, "reordered": true, "source_selection_changed": true, "added_lines": 7, "removed_lines": 7, "returned_lines": 10, "original_order_returned_lines": 10 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 6, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 6, "original_order_returned_lines": 6 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 7, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 7, "original_order_returned_lines": 7 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 3, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 3, "original_order_returned_lines": 3 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "candidate_count": 1, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 1, "original_order_returned_lines": 1 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "candidate_count": 7, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 110, "original_order_returned_lines": 110 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "candidate_count": 2, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 43, "original_order_returned_lines": 43 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "candidate_count": 8, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 8, "original_order_returned_lines": 8 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "candidate_count": 14, "reordered": true, "source_selection_changed": true, "added_lines": 57, "removed_lines": 30, "returned_lines": 130, "original_order_returned_lines": 103 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "candidate_count": 8, "reordered": true, "source_selection_changed": true, "added_lines": 65, "removed_lines": 66, "returned_lines": 124, "original_order_returned_lines": 125 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "candidate_count": 1, "reordered": false, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 15, "original_order_returned_lines": 15 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "candidate_count": 40, "reordered": true, "source_selection_changed": true, "added_lines": 10, "removed_lines": 10, "returned_lines": 10, "original_order_returned_lines": 10 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "candidate_count": 8, "reordered": true, "source_selection_changed": true, "added_lines": 168, "removed_lines": 154, "returned_lines": 208, "original_order_returned_lines": 194 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "candidate_count": 8, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 120, "original_order_returned_lines": 120 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "candidate_count": 6, "reordered": true, "source_selection_changed": false, "added_lines": 0, "removed_lines": 0, "returned_lines": 6, "original_order_returned_lines": 6 } ], "runtime_and_output_hashes_match": true }, "e2e_status": "Complete: four preselected attempts, both C++ native tests pass, both C SDK budget stops retain zero. No independent or comparative repair gain.", "e2e_analysis": { "scope": "Two repeated development tasks, fresh matched retrieval-to-repair pipelines. Four attempts maximum, native versus Jev. No independent Pass@1 or leaderboard claim.", "protocol_sha256": "223f487c75dc7dd0243c819e09051d50d163a75decbf05854232fd22c5de3b6f", "implementation_commit": "cf4bc60b84622ebe5bf9c89c9d0f47fbc18463c0", "rows": [ { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "native", "status": "Submitted", "resolved": true, "model_calls": 6, "repair_sdk_estimate_usd": 0.7711020000000001, "known_repair_sdk_estimate_usd": 0.7711020000000001, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.10767510000000001, "jev_api_usd": 0, "pipeline_meter_estimate_usd": 0.8787771000000001, "pipeline_wall_s_excluding_grading": 254.42253226798493, "repair_wall_s": 229.71050729299895, "grading_wall_s": 193.3281993549317, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 404, "initial_base_source_lines": 404, "new_original_source_lines": 0, "controlled_reads": 1, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": true, "native_exit_code": 0, "container_removed": true, "source_snapshot_verified": true, "native_log_sha256": "fd880ade9d5cc1be77433c3df0fa6216f2ebf92bb38798f3dcfb3cdfb715fbf4" }, "model_patch_sha256": "9cb9cfa65c12062a8df2b8ccceace08b7549c66bf7655f52051e0bce30cab912" }, { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "jev", "status": "Submitted", "resolved": true, "model_calls": 3, "repair_sdk_estimate_usd": 0.36135439999999996, "known_repair_sdk_estimate_usd": 0.36135439999999996, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.1118277, "jev_api_usd": 0.00024242399999999998, "pipeline_meter_estimate_usd": 0.47342452399999996, "pipeline_wall_s_excluding_grading": 78.85783060011454, "repair_wall_s": 52.60522423905786, "grading_wall_s": 194.60446256399155, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 417, "initial_base_source_lines": 417, "new_original_source_lines": 0, "controlled_reads": 1, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": true, "native_exit_code": 0, "container_removed": true, "source_snapshot_verified": true, "native_log_sha256": "e2d4c65035e2d74738e23653f855a10dbd1db44c63964c4cbc7a68d89f338271" }, "model_patch_sha256": "6586e05c941ba3954bba47642028f5476e054508dc48b663581e0ab53a84be28" }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "native", "status": "error", "resolved": false, "model_calls": 9, "repair_sdk_estimate_usd": 3.0819029, "known_repair_sdk_estimate_usd": 3.0819029, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.3881817, "jev_api_usd": 0, "pipeline_meter_estimate_usd": 3.4700846, "pipeline_wall_s_excluding_grading": 266.39263455499895, "repair_wall_s": 191.30468756798655, "grading_wall_s": null, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 1425, "initial_base_source_lines": 1425, "new_original_source_lines": 0, "controlled_reads": 0, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "no_prediction", "resolved": false, "native_exit_code": null, "container_removed": null, "source_snapshot_verified": null, "native_log_sha256": null }, "model_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "jev", "status": "error", "resolved": false, "model_calls": 13, "repair_sdk_estimate_usd": 3.0282820000000004, "known_repair_sdk_estimate_usd": 3.0282820000000004, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.28681049999999997, "jev_api_usd": 0.0008709119999999999, "pipeline_meter_estimate_usd": 3.3159634120000003, "pipeline_wall_s_excluding_grading": 255.33172728097998, "repair_wall_s": 197.8604957869975, "grading_wall_s": null, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 1092, "initial_base_source_lines": 962, "new_original_source_lines": 130, "controlled_reads": 4, "controlled_searches": 3, "solver_container_removed": true, "native_grading": { "status": "no_prediction", "resolved": false, "native_exit_code": null, "container_removed": null, "source_snapshot_verified": null, "native_log_sha256": null }, "model_patch_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" } ], "resolved": { "native": { "resolved": 1, "selected_tasks": 2 }, "jev": { "resolved": 1, "selected_tasks": 2 } }, "known_repair_sdk_usd": 7.242641300000001, "repair_api_key_calls": 0, "missing_metering_calls": 0, "execution_amendment": { "sha256": "09baa070f5a6a8d5b3e3d45d64f0db14cef2cd2e4cbc181b076f5b5f1c577582", "reason": "Original generic error stop prevented the second arm after a fully metered terminal SDK budget error. Complete the already selected, previously unrun Jev arm once at the unchanged budget; do not retry native or inspect reference/grade content. This post-stop execution amendment is explicit and does not replace original records.", "invariants": "Same task, model, initial source receipt, common solver prompt, pinned image, 20 turns and $3 SDK guard. Baseline failure remains zero. No new API route, retry, source-policy change or budget increase. Original maximum total repair SDK reservation remains $10.", "grading_note": "Original scheduling-only grading already began after the primary driver stopped. Its two C no-prediction records are retained. Grade the new candidate only after this attempt ends, in the same separate native harness; no grading content enters model inputs." }, "matched_common_prompts_and_configs": true, "all_frozen_hashes_match": true, "all_owned_containers_removed_and_absent": true, "development_resolved_difference": 0, "independent_end_to_end_gain_established": false, "interpretation": "Report paired outcomes and costs; two repeated compatible development tasks cannot establish population or independent end-to-end gain. Source coverage remains partial for arbitrary shell output.", "analysis_sha256": "29616d5f2d6c7611cbaa30cd62dc25757ca58bdccc2276a0acebe62423f2fcd4" }, "budget_patch_collection_followup": { "scope": "Prospective harness correction, not a revised result or an agent policy gain.", "failure": "SDK budget exceptions bypassed patch export before container cleanup; both historical C attempts have no captured prediction.", "policy": "Collect executed working-tree edits only after normal agent termination or fully metered SDK budget exhaustion. Keep the original error and cost, make no extra model request, and exclude terminal export from source observations.", "unknown_usage_and_unrelated_errors_excluded": true, "historical_attempts_unchanged": true, "validation": "57 solver/source tests pass, including three pinned-upstream integrations; deterministic budget/error export regression makes no model or container calls.", "implementation_commit": "f607ea7c55054d57f9a340b6109a6a1aaf29f432" } }, "latest_evidence_date": "2026-10-17", "online_grep_hints_development": { "scope": "Fresh online retrieval-agent development controls. Not repair Pass@1, independent validation, or an official leaderboard submission.", "protocol_sha256": "3b44b7ee65f75ce9f33613c37440bcd8a113227168108182918300f7ca576b3a", "output_receipts_sha256": "12eaa2f700ef565270518256b71d8e3cbe9f580e3108849337d4c8bd50d69bf7", "summaries": { "native": { "n": 8, "arms": { "trajectory": { "recall": 0.4329153656708359, "precision": 0.38509373320009405, "f1": 0.3708023564283629, "official_valid": 8 }, "served_16384": { "recall": 0.4216541044095746, "precision": 0.3857054754532709, "f1": 0.37056358943719114, "official_valid": 8 }, "served_32768": { "recall": 0.4329153656708359, "precision": 0.38551346058592684, "f1": 0.3710326823030195, "official_valid": 8 } }, "usage": { "billing_route": "anthropic_api_key", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.6075559, "missing_metering_cases": 0, "jev_api_cost_usd": 0, "jev_api_calls": 0, "jev_unknown_usage_calls": 0, "statuses": { "ok": 8 }, "cases": 8 } }, "hints": { "n": 8, "arms": { "trajectory": { "recall": 0.43206076362381596, "precision": 0.3768703408334222, "f1": 0.3634494147634172, "official_valid": 8 }, "served_16384": { "recall": 0.4313317009980352, "precision": 0.38018969063948904, "f1": 0.3654732660207942, "official_valid": 8 }, "served_32768": { "recall": 0.4313317009980352, "precision": 0.3767629579817152, "f1": 0.3632171917105076, "official_valid": 8 } }, "usage": { "billing_route": "anthropic_api_key", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.6213457999999998, "missing_metering_cases": 0, "jev_api_cost_usd": 0.0, "jev_api_calls": 0, "jev_unknown_usage_calls": 0, "statuses": { "ok": 8 }, "cases": 8 } }, "jev-hints": { "n": 8, "arms": { "trajectory": { "recall": 0.3873026388731309, "precision": 0.3637346019334589, "f1": 0.3416844146844938, "official_valid": 8 }, "served_16384": { "recall": 0.3866309549549474, "precision": 0.36913333833024653, "f1": 0.34461642596120734, "official_valid": 8 }, "served_32768": { "recall": 0.3873026388731309, "precision": 0.36401200409126366, "f1": 0.3418552267327215, "official_valid": 8 } }, "usage": { "billing_route": "anthropic_api_key", "actual_bill_usd": null, "sdk_reported_cost_usd": 1.6099298, "missing_metering_cases": 0, "jev_api_cost_usd": 0.009593262000000002, "jev_api_calls": 75, "jev_unknown_usage_calls": 0, "statuses": { "ok": 8 }, "cases": 8 } } }, "comparisons": { "native": { "recall": { "mean_difference": -0.045612726797705026, "exploratory_95_interval": [ -0.10076157928474531, -0.006377551020408163 ] }, "precision": { "mean_difference": -0.021359131266635162, "exploratory_95_interval": [ -0.1158581828615073, 0.06896391648671565 ] }, "f1": { "mean_difference": -0.029117941743869148, "exploratory_95_interval": [ -0.10297905608615839, 0.028927799646731332 ] } }, "hints": { "recall": { "mean_difference": -0.04475812475068512, "exploratory_95_interval": [ -0.1304085513057926, 0.024540506095089013 ] }, "precision": { "mean_difference": -0.013135738899963313, "exploratory_95_interval": [ -0.11227135341910374, 0.09906432420779722 ] }, "f1": { "mean_difference": -0.021765000078923454, "exploratory_95_interval": [ -0.1106232686194129, 0.0573524443654226 ] } } }, "secondary_unranked_hints_vs_native": { "recall": { "mean_difference": -0.0008546020470199013, "exploratory_95_interval": [ -0.04338181658355898, 0.04006094433954543 ] }, "precision": { "mean_difference": -0.008223392366671849, "exploratory_95_interval": [ -0.04220182622038603, 0.02667057391070715 ] }, "f1": { "mean_difference": -0.007352941664945694, "exploratory_95_interval": [ -0.04426975368392229, 0.02651494613059313 ] } }, "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "metrics": { "recall": 0.4896265560165975, "precision": 0.5244444444444445, "f1": 0.5064377682403433 }, "grep_calls": 2, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.10044059999999999, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 11, "sdk_reported_turns": 9, "tool_calls": { "grep": 2, "read": 5, "glob": 1 }, "wall_s": 26.471280466997996, "source_lines": 225 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "metrics": { "recall": 0.30612244897959184, "precision": 0.12195121951219512, "f1": 0.17441860465116277 }, "grep_calls": 5, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1469511, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 20, "sdk_reported_turns": 13, "tool_calls": { "grep": 5, "read": 7 }, "wall_s": 39.37248671601992, "source_lines": 492 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "metrics": { "recall": 0.5091307523739956, "precision": 0.5349194167306216, "f1": 0.5217065868263472 }, "grep_calls": 8, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.33174750000000003, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 28, "sdk_reported_turns": 20, "tool_calls": { "grep": 8, "read": 9, "glob": 2 }, "wall_s": 80.63994976202957, "source_lines": 1303 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "metrics": { "recall": 0.5362694300518135, "precision": 0.5149253731343284, "f1": 0.5253807106598986 }, "grep_calls": 3, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.12088910000000001, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 9, "sdk_reported_turns": 6, "tool_calls": { "grep": 3, "read": 2 }, "wall_s": 27.989065212896094, "source_lines": 402 }, { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "metrics": { "recall": 0.45416666666666666, "precision": 0.8074074074074075, "f1": 0.5813333333333333 }, "grep_calls": 4, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.10846000000000001, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 14, "sdk_reported_turns": 10, "tool_calls": { "grep": 4, "read": 5 }, "wall_s": 34.30429421795998, "source_lines": 270 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "metrics": { "recall": 0.146779303062302, "precision": 0.25272727272727274, "f1": 0.18570474281897129 }, "grep_calls": 3, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1439565, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 14, "sdk_reported_turns": 11, "tool_calls": { "read": 7, "grep": 3 }, "wall_s": 36.47857246699277, "source_lines": 550 }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "metrics": { "recall": 0.35456110154905335, "precision": 0.18933823529411764, "f1": 0.24685440383463148 }, "grep_calls": 5, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.25795009999999996, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 22, "sdk_reported_turns": 15, "tool_calls": { "read": 9, "grep": 5 }, "wall_s": 55.99795773799997, "source_lines": 1088 }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "metrics": { "recall": 0.6666666666666666, "precision": 0.13503649635036497, "f1": 0.22458270106221548 }, "grep_calls": 12, "ranked_views": 0, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.397161, "jev_api_calls": 0, "jev_cost_usd": 0, "assistant_events": 40, "sdk_reported_turns": 26, "tool_calls": { "grep": 12, "read": 13 }, "wall_s": 77.99639331304934, "source_lines": 1644 }, { "arm": "hints", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "metrics": { "recall": 0.3983402489626556, "precision": 0.44036697247706424, "f1": 0.4183006535947713 }, "grep_calls": 3, "ranked_views": 2, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 0, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.090115, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 10, "sdk_reported_turns": 8, "tool_calls": { "grep": 3, "read": 4 }, "wall_s": 25.114816530956887, "source_lines": 218 }, { "arm": "hints", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "metrics": { "recall": 0.37244897959183676, "precision": 0.09959072305593451, "f1": 0.1571582346609257 }, "grep_calls": 7, "ranked_views": 6, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 3, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.2038759, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 25, "sdk_reported_turns": 17, "tool_calls": { "grep": 7, "read": 9 }, "wall_s": 63.38308473699726, "source_lines": 733 }, { "arm": "hints", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "metrics": { "recall": 0.5821767713659606, "precision": 0.5202349869451697, "f1": 0.5494657014822475 }, "grep_calls": 9, "ranked_views": 8, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 6, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.35840970000000005, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 35, "sdk_reported_turns": 23, "tool_calls": { "grep": 9, "read": 12, "glob": 1 }, "wall_s": 82.52649485203438, "source_lines": 1532 }, { "arm": "hints", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "metrics": { "recall": 0.5362694300518135, "precision": 0.5085995085995086, "f1": 0.5220680958385877 }, "grep_calls": 4, "ranked_views": 3, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 5, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1226363, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 12, "sdk_reported_turns": 8, "tool_calls": { "grep": 4, "read": 3 }, "wall_s": 28.825861321995035, "source_lines": 407 }, { "arm": "hints", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "metrics": { "recall": 0.3729166666666667, "precision": 0.7396694214876033, "f1": 0.4958448753462604 }, "grep_calls": 4, "ranked_views": 4, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 4, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.093556, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 11, "sdk_reported_turns": 9, "tool_calls": { "grep": 4, "read": 4 }, "wall_s": 27.490284716011956, "source_lines": 242 }, { "arm": "hints", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "metrics": { "recall": 0.20485744456177402, "precision": 0.33856893542757416, "f1": 0.2552631578947368 }, "grep_calls": 5, "ranked_views": 4, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 4, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1704246, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 19, "sdk_reported_turns": 13, "tool_calls": { "grep": 5, "read": 7 }, "wall_s": 45.17950349801686, "source_lines": 573 }, { "arm": "hints", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "metrics": { "recall": 0.3528399311531842, "precision": 0.22282608695652173, "f1": 0.27315123251165885 }, "grep_calls": 9, "ranked_views": 9, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 5, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.23132049999999998, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 24, "sdk_reported_turns": 18, "tool_calls": { "read": 8, "grep": 9 }, "wall_s": 56.74839012802113, "source_lines": 920 }, { "arm": "hints", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "metrics": { "recall": 0.6366366366366366, "precision": 0.14510609171800137, "f1": 0.2363433667781494 }, "grep_calls": 11, "ranked_views": 9, "views_reordered": 0, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 15, "jev_request_latency_s": 0, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.3510078, "jev_api_calls": 0, "jev_cost_usd": 0.0, "assistant_events": 35, "sdk_reported_turns": 25, "tool_calls": { "grep": 11, "read": 13 }, "wall_s": 79.8583371900022, "source_lines": 1461 }, { "arm": "jev-hints", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "metrics": { "recall": 0.5020746887966805, "precision": 0.7423312883435583, "f1": 0.5990099009900989 }, "grep_calls": 2, "ranked_views": 2, "views_reordered": 2, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 3, "jev_request_latency_s": 0.652294336934574, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.0830826, "jev_api_calls": 2, "jev_cost_usd": 0.000247044, "assistant_events": 8, "sdk_reported_turns": 6, "tool_calls": { "grep": 2, "read": 3 }, "wall_s": 22.706850082962774, "source_lines": 163 }, { "arm": "jev-hints", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "metrics": { "recall": 0.25510204081632654, "precision": 0.07363770250368189, "f1": 0.11428571428571428 }, "grep_calls": 6, "ranked_views": 6, "views_reordered": 5, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 3, "jev_request_latency_s": 4.220684670028277, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.22069070000000002, "jev_api_calls": 15, "jev_cost_usd": 0.0021610260000000004, "assistant_events": 24, "sdk_reported_turns": 15, "tool_calls": { "grep": 6, "read": 8 }, "wall_s": 67.90257290098816, "source_lines": 679 }, { "arm": "jev-hints", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "metrics": { "recall": 0.2914536157779401, "precision": 0.24767225325884543, "f1": 0.2677852348993289 }, "grep_calls": 8, "ranked_views": 7, "views_reordered": 6, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 2, "jev_request_latency_s": 4.064416881883517, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.34919130000000004, "jev_api_calls": 16, "jev_cost_usd": 0.00226863, "assistant_events": 29, "sdk_reported_turns": 19, "tool_calls": { "grep": 8, "read": 9, "glob": 1 }, "wall_s": 82.25347500597127, "source_lines": 1611 }, { "arm": "jev-hints", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "metrics": { "recall": 0.5362694300518135, "precision": 0.5798319327731093, "f1": 0.5572005383580081 }, "grep_calls": 3, "ranked_views": 3, "views_reordered": 3, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 3, "jev_request_latency_s": 1.045982429990545, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.10642380000000001, "jev_api_calls": 4, "jev_cost_usd": 0.000407358, "assistant_events": 9, "sdk_reported_turns": 6, "tool_calls": { "grep": 3, "read": 2 }, "wall_s": 27.520248775021173, "source_lines": 357 }, { "arm": "jev-hints", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "metrics": { "recall": 0.41458333333333336, "precision": 0.7007042253521126, "f1": 0.5209424083769634 }, "grep_calls": 5, "ranked_views": 5, "views_reordered": 2, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 5, "jev_request_latency_s": 1.2183346850797534, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1086078, "jev_api_calls": 5, "jev_cost_usd": 0.000418992, "assistant_events": 16, "sdk_reported_turns": 11, "tool_calls": { "grep": 5, "read": 5 }, "wall_s": 32.47138429002371, "source_lines": 284 }, { "arm": "jev-hints", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "metrics": { "recall": 0.146779303062302, "precision": 0.2202852614896989, "f1": 0.17617237008871992 }, "grep_calls": 5, "ranked_views": 4, "views_reordered": 3, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 3, "jev_request_latency_s": 1.8323559809941798, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.1652135, "jev_api_calls": 7, "jev_cost_usd": 0.00079548, "assistant_events": 19, "sdk_reported_turns": 13, "tool_calls": { "grep": 5, "read": 6, "glob": 1 }, "wall_s": 44.18462630896829, "source_lines": 631 }, { "arm": "jev-hints", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "metrics": { "recall": 0.35456110154905335, "precision": 0.1794425087108014, "f1": 0.23828802776171196 }, "grep_calls": 6, "ranked_views": 6, "views_reordered": 6, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 2, "jev_request_latency_s": 3.12084668118041, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.2813371, "jev_api_calls": 12, "jev_cost_usd": 0.0018798780000000006, "assistant_events": 23, "sdk_reported_turns": 17, "tool_calls": { "read": 10, "grep": 6 }, "wall_s": 69.46998837997671, "source_lines": 1148 }, { "arm": "jev-hints", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "metrics": { "recall": 0.5975975975975976, "precision": 0.16597164303586323, "f1": 0.2597911227154047 }, "grep_calls": 9, "ranked_views": 8, "views_reordered": 8, "unread_source_credit_added": 0, "suggestions_followed_by_actual_later_read": 15, "jev_request_latency_s": 3.539928643964231, "unconfirmed_tool_replies": 0, "sdk_cost_usd": 0.295383, "jev_api_calls": 14, "jev_cost_usd": 0.0014148540000000001, "assistant_events": 29, "sdk_reported_turns": 20, "tool_calls": { "grep": 9, "read": 10 }, "wall_s": 67.93161414808128, "source_lines": 1199 } ], "gates": { "all_complete": true, "known_usage": true, "one_model_and_api_route": true, "all_replies_followed_by_assistant": true, "original_grep_preserved_and_hints_add_no_source_credit": true, "primary_jev_hints_greater_recall_vs_native": false, "primary_jev_hints_nondecreasing_f1_vs_native": false }, "development_gate_passed": false, "known_anthropic_sdk_usd": 4.838831499999999, "known_jev_api_usd": 0.009593262000000002, "models": [ "claude-opus-5-5" ], "routes": [ "ANTHROPIC_API_KEY" ], "all_hashes_and_source_checks_passed": true, "implementation_commit": "10d1a439039a68d87dc44df17fa9f8b2756e396b", "analysis_sha256": "70bf6bf90a3b8379a9b8217765c2b1d3f65783fa2482fd460ce836ea760e81c1", "policy": "Original grep text is preserved verbatim. Append at most three metadata-only read suggestions, at most 512 additional tokens; Jev scores issue plus current grep arguments. Original-order hints is the prespecified ranking ablation. Actual later reads supply source evidence.", "diagnostic": "41 Jev views yield 36 suggested ranges matching actual later read output; this is adoption matching, not proof those reads were caused by the hints. All original reply/candidate/source/score receipts verify; hints add zero source credit.", "e2e_status": "Complete: both fmt repairs pass 18 native tests; both C patches are graded unresolved. Each arm resolves 1/2 selected tasks; no end-to-end resolution gain.", "repair_sdk_estimate_usd": 6.0355762, "conclusion": "Primary and prespecified unranked secondary fail the development retrieval criteria. Do not expand the read-hint policy or keep trying Jev output-format variants.", "e2e_analysis": { "scope": "Two repeated development tasks, fresh matched retrieval-to-repair pipelines. Four attempts maximum, native versus Jev read hints. No independent Pass@1 or leaderboard claim.", "protocol_sha256": "774cb3ae0118483a19c59202435d246fff03dbf0d866db14ec0743926920859d", "implementation_commit": "10d1a439039a68d87dc44df17fa9f8b2756e396b", "rows": [ { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "native", "status": "Submitted", "resolved": true, "model_calls": 3, "repair_sdk_estimate_usd": 0.3408082, "known_repair_sdk_estimate_usd": 0.3408082, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.12088910000000001, "jev_api_usd": 0, "pipeline_meter_estimate_usd": 0.46169730000000003, "pipeline_wall_s_excluding_grading": 83.1007596498821, "repair_wall_s": 55.111694436986, "grading_wall_s": 196.60547913704067, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 402, "initial_base_source_lines": 402, "new_original_source_lines": 0, "controlled_reads": 1, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": true, "native_exit_code": 0, "container_removed": true, "source_snapshot_verified": true, "native_log_sha256": "628a0ffe3d8df04ce9b82c05532edadc9ecd7e3b234ed7053a4c3561e609c844" }, "model_patch_sha256": "b9df14e1da8b40d65f97d7be407ec4f27c767bcc56521e25baeb90c9be55a360" }, { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "jev-hints", "status": "Submitted", "resolved": true, "model_calls": 3, "repair_sdk_estimate_usd": 0.3406072, "known_repair_sdk_estimate_usd": 0.3406072, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.10642380000000001, "jev_api_usd": 0.000407358, "pipeline_meter_estimate_usd": 0.447438358, "pipeline_wall_s_excluding_grading": 90.15055774210487, "repair_wall_s": 62.63030896708369, "grading_wall_s": 196.62364138499834, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 356, "initial_base_source_lines": 356, "new_original_source_lines": 0, "controlled_reads": 1, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": true, "native_exit_code": 0, "container_removed": true, "source_snapshot_verified": true, "native_log_sha256": "5e067c829235a347d7dbc0e5b199844e9ef7e0ce9fe55055d64dbe85abfc2111" }, "model_patch_sha256": "9c251bb0d9ee39e58fea9b659ab9e2e3dffa476e121eb19940715127b95dc10b" }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "native", "status": "Submitted", "resolved": false, "model_calls": 9, "repair_sdk_estimate_usd": 2.4970695000000003, "known_repair_sdk_estimate_usd": 2.4970695000000003, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.33174750000000003, "jev_api_usd": 0, "pipeline_meter_estimate_usd": 2.8288170000000004, "pipeline_wall_s_excluding_grading": 253.09868140902836, "repair_wall_s": 172.45873164699879, "grading_wall_s": 115.37804004992358, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 1302, "initial_base_source_lines": 1302, "new_original_source_lines": 0, "controlled_reads": 1, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": false, "native_exit_code": 2, "container_removed": true, "source_snapshot_verified": true, "native_log_sha256": "57d38224c5a5c2415812bc01b41340e95040b952204aee575535cb939577c65b" }, "model_patch_sha256": "d826a64abc448866b6e7a66735324d8f31ede4c0b8523b4b356f478a158641b9" }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "jev-hints", "status": "Submitted", "resolved": false, "model_calls": 9, "repair_sdk_estimate_usd": 2.8570913000000004, "known_repair_sdk_estimate_usd": 2.8570913000000004, "missing_metering_calls": 0, "retrieval_api_sdk_usd": 0.34919130000000004, "jev_api_usd": 0.00226863, "pipeline_meter_estimate_usd": 3.2085512300000003, "pipeline_wall_s_excluding_grading": 265.13257093098946, "repair_wall_s": 182.8790959250182, "grading_wall_s": 115.22666374198161, "actual_models": [ "claude-opus-5-5" ], "all_model_inputs_confirmed": true, "source_coverage_status": "partial", "base_source_line_count": 1412, "initial_base_source_lines": 1392, "new_original_source_lines": 20, "controlled_reads": 2, "controlled_searches": 0, "solver_container_removed": true, "native_grading": { "status": "graded", "resolved": false, "native_exit_code": 2, "container_removed": true, "source_snapshot_verified": true, "native_log_sha256": "cdc6ca6c359781d7ab768e1abe24921a66737ae4899314e682f9b4bc3ff4cfba" }, "model_patch_sha256": "c23085e290ee2319cacdfa64275e1a91b56a1f348e72f7f27d94aaa9c4bcdc13" } ], "resolved": { "native": { "resolved": 1, "selected_tasks": 2 }, "jev-hints": { "resolved": 1, "selected_tasks": 2 } }, "known_repair_sdk_usd": 6.035576200000001, "repair_api_key_calls": 0, "missing_metering_calls": 0, "matched_common_prompts_and_configs": true, "all_frozen_hashes_match": true, "all_owned_containers_removed_and_absent": true, "development_resolved_difference": 0, "independent_end_to_end_gain_established": false, "interpretation": "Report paired outcomes and costs; two repeated compatible development tasks cannot establish population or independent end-to-end gain. Source coverage remains partial for arbitrary shell output.", "analysis_sha256": "f167ae2e86660eebfee20f176b7492021f813897a652ee661de520c04c984bf9" } }, "retrieval_notes_development": { "status": "Complete: source 1/3, notes 0/3; reject the notes handoff policy.", "protocol_sha256": "aa6e6148a22646727666108d39dd08b7b6584c5e4f80b04285d5a62b9fc8932e", "implementation_commit": "33d027dbd91f53a15e105fc4236712d34976fd30", "scope": "Three repeated development tasks (Pylint Python, ponyc C, fmt C++), six fresh repair attempts on shared frozen native retrieval. No new retrieval calls or independent Pass@1 claim.", "policy": "Both arms receive identical 16K source. The candidate adds the confirmed final retrieval answer as unverified notes, capped at 2048 tokens and excluded from source credit. Extra input is reported separately; source budget and SDK guards match, total input-token counts need not match.", "source_notes_tokens": { "pylint": 827, "ponyc": 1520, "fmt": 954 }, "route": "Existing Claude Max, no API fallback.", "repair_sdk_reservation_usd": 17, "new_api_key_inference_calls": 0, "validation": "59 solver/source tests pass, including three pinned-upstream integrations; pinned hooks pass. Code is isolated from the frozen read-hint runtime.", "pylint_readiness": { "model_calls": 0, "source_tree_matches": true, "empty_control_native_failures": 21, "reference_control_native_passes": 21, "preservation_tests_in_native_task": 0, "reference_production_patch_excluded_from_solver_inputs": true, "behavioral_baseline_executable": false, "collection_error": "Native test patch imports IS_PYPY, absent at the base and supplied by the official reference production patch." }, "analysis": { "scope": "Three repeated development tasks, shared frozen native retrieval, fresh matched repair attempts. Handoff ablation, not independent Pass@1 or new ContextBench retrieval scores.", "protocol_sha256": "aa6e6148a22646727666108d39dd08b7b6584c5e4f80b04285d5a62b9fc8932e", "implementation_commit": "33d027dbd91f53a15e105fc4236712d34976fd30", "rows": [ { "task": "pylint", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "source", "status": "Submitted", "resolved": false, "native_grade_status": "graded", "model_calls": 6, "repair_sdk_usd": 0.44844080000000003, "known_repair_sdk_usd": 0.44844080000000003, "missing_metering_calls": 0, "shared_retrieval_api_sdk_usd": 0.10044059999999999, "allocated_pipeline_meter_usd": 0.5488814, "notes_tokens": 0, "notes_truncated": false, "initial_base_source_lines": 224, "base_source_line_count": 224, "new_original_source_lines": 0, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "actual_models": [ "claude-opus-5-5" ], "budget_exhausted": false, "solver_container_removed": true, "repair_wall_s": 65.67515322298277, "grading_wall_s": null, "model_patch_sha256": "45c8b21e12c5c64b73db309c222369cd243e219ecc781fc428f050c9eb4b22a2" }, { "task": "pylint", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "source-notes", "status": "Submitted", "resolved": false, "native_grade_status": "graded", "model_calls": 9, "repair_sdk_usd": 0.8676517000000001, "known_repair_sdk_usd": 0.8676517000000001, "missing_metering_calls": 0, "shared_retrieval_api_sdk_usd": 0.10044059999999999, "allocated_pipeline_meter_usd": 0.9680923000000001, "notes_tokens": 827, "notes_truncated": false, "initial_base_source_lines": 224, "base_source_line_count": 224, "new_original_source_lines": 0, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "actual_models": [ "claude-opus-5-5" ], "budget_exhausted": false, "solver_container_removed": true, "repair_wall_s": 129.90341635595541, "grading_wall_s": null, "model_patch_sha256": "e547d89f3eb9d8fd8eefffd8b1af190bdee58a1a2757b16f9064ef49c2e509af" }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "source", "status": "error", "resolved": false, "native_grade_status": "graded", "model_calls": 12, "repair_sdk_usd": 3.2079216000000006, "known_repair_sdk_usd": 3.2079216000000006, "missing_metering_calls": 0, "shared_retrieval_api_sdk_usd": 0.33174750000000003, "allocated_pipeline_meter_usd": 3.5396691000000007, "notes_tokens": 0, "notes_truncated": false, "initial_base_source_lines": 1302, "base_source_line_count": 1302, "new_original_source_lines": 0, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "actual_models": [ "claude-opus-5-5" ], "budget_exhausted": true, "solver_container_removed": true, "repair_wall_s": 154.74466425401624, "grading_wall_s": 90.64230875403155, "model_patch_sha256": "855a0cd6457d0cd7fa0b5214c35607d9eb1559120a8cf9c2a9a38a4dd5cfbae8" }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "source-notes", "status": "error", "resolved": false, "native_grade_status": "graded", "model_calls": 10, "repair_sdk_usd": 3.1926371000000002, "known_repair_sdk_usd": 3.1926371000000002, "missing_metering_calls": 0, "shared_retrieval_api_sdk_usd": 0.33174750000000003, "allocated_pipeline_meter_usd": 3.5243846000000003, "notes_tokens": 1520, "notes_truncated": false, "initial_base_source_lines": 1302, "base_source_line_count": 1302, "new_original_source_lines": 0, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "actual_models": [ "claude-opus-5-5" ], "budget_exhausted": true, "solver_container_removed": true, "repair_wall_s": 170.98775800305884, "grading_wall_s": 3.108311725081876, "model_patch_sha256": "6bbdc1f09b62fd91273307c86c947734b583f503c6c429c53068832a014f36ce" }, { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "source", "status": "Submitted", "resolved": true, "native_grade_status": "graded", "model_calls": 4, "repair_sdk_usd": 0.38688120000000004, "known_repair_sdk_usd": 0.38688120000000004, "missing_metering_calls": 0, "shared_retrieval_api_sdk_usd": 0.12088910000000001, "allocated_pipeline_meter_usd": 0.5077703, "notes_tokens": 0, "notes_truncated": false, "initial_base_source_lines": 402, "base_source_line_count": 402, "new_original_source_lines": 0, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "actual_models": [ "claude-opus-5-5" ], "budget_exhausted": false, "solver_container_removed": true, "repair_wall_s": 161.5216594779631, "grading_wall_s": 200.11525097792037, "model_patch_sha256": "1c849808ca5ba029631a601cb4c7f5d54e136f577bbb7ebdf9b434aba6734361" }, { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "source-notes", "status": "Submitted", "resolved": false, "native_grade_status": "graded", "model_calls": 6, "repair_sdk_usd": 0.9548426, "known_repair_sdk_usd": 0.9548426, "missing_metering_calls": 0, "shared_retrieval_api_sdk_usd": 0.12088910000000001, "allocated_pipeline_meter_usd": 1.0757317, "notes_tokens": 954, "notes_truncated": false, "initial_base_source_lines": 402, "base_source_line_count": 402, "new_original_source_lines": 0, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "actual_models": [ "claude-opus-5-5" ], "budget_exhausted": false, "solver_container_removed": true, "repair_wall_s": 80.89998147008009, "grading_wall_s": 1.928598088095896, "model_patch_sha256": "256cdb043226c07388fe51ed8fbbb5e95909315c3feb263a3ef81cb701f7d299" } ], "totals": { "source": { "resolved": 1, "selected_tasks": 3, "repair_sdk_usd": 4.043243600000001, "model_calls": 22 }, "source-notes": { "resolved": 0, "selected_tasks": 3, "repair_sdk_usd": 5.0151314000000005, "model_calls": 25 } }, "known_repair_sdk_usd": 9.058375, "new_api_key_inference_calls": 0, "missing_metering_calls": 0, "all_selected_native_outcomes_valid": true, "all_frozen_hashes_match": true, "all_initial_sources_identical": true, "only_notes_and_advisory_change_initial_prompt": true, "all_owned_containers_absent": true, "independent_gain_established": false, "interpretation": "Report all repeated development outcomes. Extra note input is explicit; source and SDK guards match. Shared retrieval is allocated for arm comparisons but not billed again. Any positive result requires replication and independent validation." }, "analysis_sha256": "aa6fe70d341884a07544204026e43955c4b8f70872452d2506ae78cfe569b1bf", "equivalent_pr_runtime_commit": "a8fe2a02", "native_failure_diagnostics": [ { "task": "pylint", "arm": "source", "native_resolved": false, "failure_phase": "collection_error_missing_IS_PYPY", "log_sha256": "402ca14e0fc3d6a5c66c96ef75a16fa638a5353ce9fb4ed1cf13211f71ef9293" }, { "task": "pylint", "arm": "source-notes", "native_resolved": false, "failure_phase": "collection_error_missing_IS_PYPY", "log_sha256": "a009df98a47eea568e5d03f98953654d4964709b3e3c0304903c39d3776d92cd" }, { "task": "ponyc", "arm": "source", "native_resolved": false, "failure_phase": "native_test_execution", "log_sha256": "9c49c00d2eac53f7ebbc8a9d830d7f8b6c6f39db059972328cc0ce279535bf24" }, { "task": "ponyc", "arm": "source-notes", "native_resolved": false, "failure_phase": "model_patch_conflicts_with_native_test_patch", "log_sha256": "0fe8b04448f30fab2ece02dfdc8e504bbe6b04b4edf2a24d32d364cf5429d1c7" }, { "task": "fmt", "arm": "source", "native_resolved": true, "failure_phase": "native_test_execution", "log_sha256": "9ad0dab5a3b5bbeb113708c39ee530ae44ce9f74df0651267a8dd2b8276ca807" }, { "task": "fmt", "arm": "source-notes", "native_resolved": false, "failure_phase": "model_patch_conflicts_with_native_test_patch", "log_sha256": "41e42e5689b1391c62e34f9ec204502dcfa3e4c4a167c642ae6d639b7f1048aa" } ], "analysis_field_note": "The archived all_selected_native_outcomes_valid flag verifies available native report receipts, not that every native test executed. The separate failure-phase audit qualifies all unresolved outcomes." }, "haiku_initial_context_development": { "status": "Complete: issue-only and frozen-source each resolve 1/3 selected tasks; no resolution or allocated pipeline-cost gain. Native failure phases qualify interpretation.", "protocol_sha256": "83acf2b25a0af2c035b4cd03e870a2b59c34d01067c746993a37a1d434fc92c7", "implementation_commit": "59ef2e9fafcad23a977b798f4dc7f2ca2b3efa76", "scope": "Six fresh matched Haiku repair attempts on three repeated development tasks. Frozen retrieval was generated by Opus and is reused without new calls. This is repair context availability, not a cheap full-pipeline or independent gain claim.", "policy": "Identical issue, solver system/user templates, image and limits. Initial retrieved_source is the empty JSON list in none, or the frozen native 16K context in retrieved. No notes, reranking, graph or extra planner. Subsequent shell inspection is available in both.", "model": "claude-haiku-5-5", "route": "Explicit first-party Anthropic API, no subscription/provider/model fallback.", "repair_sdk_reservation_usd": 1, "repair_api_sdk_estimate_usd": 0.26029216, "repair_attempts": 6, "repair_sdk_queries": 55, "additional_retrieval_calls": 0, "missing_metering_calls": 0, "independent_gain_established": false, "analysis": { "scope": "Six fresh matched Haiku repair attempts on three repeated development tasks. Frozen retrieval was generated by Opus and is reused without new calls. This is repair context availability, not a cheap full-pipeline or independent gain claim.", "protocol_sha256": "83acf2b25a0af2c035b4cd03e870a2b59c34d01067c746993a37a1d434fc92c7", "implementation_commit": "59ef2e9fafcad23a977b798f4dc7f2ca2b3efa76", "rows": [ { "task": "pylint", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "none", "status": "Submitted", "resolved": false, "native_grade_status": "graded", "repair_api_sdk_usd": 0.012634925, "model_calls": 8, "actual_models": [ "claude-haiku-5-5" ], "missing_metering_calls": 0, "initial_base_source_lines": 0, "base_source_line_count": 56, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "budget_exhausted": false, "solver_container_removed": true, "prior_retrieval_api_sdk_usd": 0.10044059999999999, "allocated_pipeline_meter_usd": 0.012634925, "repair_wall_s": 39.7998473449843, "grading_wall_s": null }, { "task": "pylint", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "retrieved", "status": "Submitted", "resolved": false, "native_grade_status": "graded", "repair_api_sdk_usd": 0.008787975, "model_calls": 4, "actual_models": [ "claude-haiku-5-5" ], "missing_metering_calls": 0, "initial_base_source_lines": 224, "base_source_line_count": 224, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "budget_exhausted": false, "solver_container_removed": true, "prior_retrieval_api_sdk_usd": 0.10044059999999999, "allocated_pipeline_meter_usd": 0.109228575, "repair_wall_s": 26.795587124070153, "grading_wall_s": null }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "none", "status": "Submitted", "resolved": false, "native_grade_status": "graded", "repair_api_sdk_usd": 0.09553966, "model_calls": 20, "actual_models": [ "claude-haiku-5-5" ], "missing_metering_calls": 0, "initial_base_source_lines": 0, "base_source_line_count": 765, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "budget_exhausted": false, "solver_container_removed": true, "prior_retrieval_api_sdk_usd": 0.33174750000000003, "allocated_pipeline_meter_usd": 0.09553966, "repair_wall_s": 126.81422503001522, "grading_wall_s": 91.28787757398095 }, { "task": "ponyc", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "retrieved", "status": "error", "resolved": false, "native_grade_status": "graded", "repair_api_sdk_usd": 0.10082195, "model_calls": 11, "actual_models": [ "claude-haiku-5-5" ], "missing_metering_calls": 0, "initial_base_source_lines": 1302, "base_source_line_count": 1394, "source_coverage_status": "partial", "all_model_inputs_confirmed": false, "budget_exhausted": true, "solver_container_removed": true, "prior_retrieval_api_sdk_usd": 0.33174750000000003, "allocated_pipeline_meter_usd": 0.43256945, "repair_wall_s": 156.0411981480429, "grading_wall_s": 90.6421816929942 }, { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "none", "status": "Submitted", "resolved": true, "native_grade_status": "graded", "repair_api_sdk_usd": 0.022075400000000002, "model_calls": 7, "actual_models": [ "claude-haiku-5-5" ], "missing_metering_calls": 0, "initial_base_source_lines": 0, "base_source_line_count": 288, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "budget_exhausted": false, "solver_container_removed": true, "prior_retrieval_api_sdk_usd": 0.12088910000000001, "allocated_pipeline_meter_usd": 0.022075400000000002, "repair_wall_s": 74.1336342409486, "grading_wall_s": 188.14434883696958 }, { "task": "fmt", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "retrieved", "status": "Submitted", "resolved": true, "native_grade_status": "graded", "repair_api_sdk_usd": 0.02043225, "model_calls": 5, "actual_models": [ "claude-haiku-5-5" ], "missing_metering_calls": 0, "initial_base_source_lines": 402, "base_source_line_count": 402, "source_coverage_status": "partial", "all_model_inputs_confirmed": true, "budget_exhausted": false, "solver_container_removed": true, "prior_retrieval_api_sdk_usd": 0.12088910000000001, "allocated_pipeline_meter_usd": 0.14132135, "repair_wall_s": 102.58622005209327, "grading_wall_s": 187.96970002201851 } ], "totals": { "none": { "resolved": 1, "selected_tasks": 3, "repair_api_sdk_usd": 0.130249985, "allocated_pipeline_meter_usd": 0.130249985, "model_calls": 35 }, "retrieved": { "resolved": 1, "selected_tasks": 3, "repair_api_sdk_usd": 0.13004217499999998, "allocated_pipeline_meter_usd": 0.6831193750000001, "model_calls": 20 } }, "known_new_repair_api_sdk_usd": 0.26029216, "new_repair_attempts": 6, "new_api_sdk_sessions": 55, "new_retrieval_or_summary_calls": 0, "missing_metering_calls": 0, "all_selected_native_reports_available": true, "native_readiness_and_patch_methodology_gate_passed": false, "all_frozen_hashes_match": true, "only_source_availability_changes_initial_prompt": true, "all_owned_containers_absent": true, "independent_gain_established": false, "interpretation": "Repeated development screen only. Report every position and repair-only vs allocated pipeline cost separately. No comparison to Opus is matched. No independent validation or official500 advancement.", "native_failure_diagnostics": [ { "task": "pylint", "arm": "none", "native_resolved": false, "failure_phase": "collection_error_missing_IS_PYPY", "log_sha256": "378a3cdefdf74ce8c0cc2f9e7f6d666239b8a34a28bf040bc81e7471d16681ea" }, { "task": "pylint", "arm": "retrieved", "native_resolved": false, "failure_phase": "collection_error_missing_IS_PYPY", "log_sha256": "e353b2c3a39d9526a0e0fe4753369cd5742f995f04c278a2358de58b3819152a" }, { "task": "ponyc", "arm": "none", "native_resolved": false, "failure_phase": "native_test_execution", "log_sha256": "fee538629eaf8aad79108c2b321bf7d1852ff4f6d8b68abe46886a0f546b0ae2" }, { "task": "ponyc", "arm": "retrieved", "native_resolved": false, "failure_phase": "native_test_execution", "log_sha256": "30d4495a17bd6f6fe219d2fe76512bd8ddc9fb8b0c5fe627b3bffcc715c287c4" }, { "task": "fmt", "arm": "none", "native_resolved": true, "failure_phase": "native_test_execution", "log_sha256": "6537df58d4d74926b4fe50004a3f0ffeb751cafcfb673a90c19fedeff51038e9" }, { "task": "fmt", "arm": "retrieved", "native_resolved": true, "failure_phase": "native_test_execution", "log_sha256": "fdfafaa5341bd5dbcada1226a5341140e7fcfbba211d42e213fad5b7c7a999d9" } ] }, "analysis_sha256": "5fb5338c32df8ad3dc996250302b423b07737595ba67f90690517bc48772cae0", "native_readiness_and_patch_methodology_gate_passed": false, "c_native_tests": { "selected_native_test_count": 1017, "issue_only_passed": 1006, "issue_only_failed": 11, "frozen_source_passed": 1010, "frozen_source_failed": 7, "both_unresolved": true, "interpretation": "Single repeated development task; partial test progress, not a resolved-issue or independent gain." }, "cpp_native_tests": { "issue_only_resolved": true, "frozen_source_resolved": true } }, "current_priority": "Context metrics first,Recallprimary,F1/costreported. Preserve original strict attempt gatefailure andtheSDKrejectedaction/error;qualifyonlyactualregistered3tool observations withproofrejectedaction contributeszero source. Continueonly257unclaimed originals,reuse242prefixes/56pairs,retain1originalzero-source/unmeteredcase/all500denominators. Same1bmodel/prompt/source/compiler/budget,all3cohortssealed. NoGold/interimscore/repairs.443exactinputcount/fullreserve<=40beforeonepaidBatch.", "source_gap_diagnosis": { "scope": "Evaluation-only diagnosis of eight disclosed native trajectories. Canonical official paths and scores unchanged. No model calls; no annotation-driven runtime choices.", "totals": { "valid_gold_lines": 4408, "unavailable_gold_lines": 125, "valid_missing_in_observed_files": 1768, "valid_missing_in_unobserved_files": 773, "tool_steps_after_last_gold_gain": 21 }, "source_metrics_sha256": "018203c3452eadb573a2b5ce3ba100afafada10cd04790e46d9c2960baf97bbf", "model_calls": 0, "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "valid_gold_lines": 207, "unavailable_gold_lines": 34, "valid_missing_in_observed_files": 77, "valid_missing_in_unobserved_files": 12 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "valid_gold_lines": 195, "unavailable_gold_lines": 1, "valid_missing_in_observed_files": 41, "valid_missing_in_unobserved_files": 94 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "valid_gold_lines": 1366, "unavailable_gold_lines": 3, "valid_missing_in_observed_files": 669, "valid_missing_in_unobserved_files": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "valid_gold_lines": 386, "unavailable_gold_lines": 0, "valid_missing_in_observed_files": 179, "valid_missing_in_unobserved_files": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "valid_gold_lines": 480, "unavailable_gold_lines": 0, "valid_missing_in_observed_files": 113, "valid_missing_in_unobserved_files": 149 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "valid_gold_lines": 860, "unavailable_gold_lines": 87, "valid_missing_in_observed_files": 519, "valid_missing_in_unobserved_files": 202 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "valid_gold_lines": 581, "unavailable_gold_lines": 0, "valid_missing_in_observed_files": 59, "valid_missing_in_unobserved_files": 316 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "valid_gold_lines": 333, "unavailable_gold_lines": 0, "valid_missing_in_observed_files": 111, "valid_missing_in_unobserved_files": 0 } ] }, "file_map_opportunity": { "scope": "Addressability diagnostic only. No predicted retrieval gain or actual map-guided agent behavior.", "model_calls": 0, "all_original_source_credit_preserved": true, "frozen_maps_sha256": "b9b0b6ade034bd9b0b6629381244ef1d84f49da5ebf7a7563b01ec2d210013f5", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "maps": 2, "symbols": 82, "rejected_ranges": 1, "omitted_symbols": 0, "missing_gold_inside_map_callable_ranges": 76, "actual_source_credit_added": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "maps": 4, "symbols": 34, "rejected_ranges": 1, "omitted_symbols": 0, "missing_gold_inside_map_callable_ranges": 7, "actual_source_credit_added": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "maps": 8, "symbols": 153, "rejected_ranges": 1, "omitted_symbols": 0, "missing_gold_inside_map_callable_ranges": 597, "actual_source_credit_added": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "maps": 2, "symbols": 33, "rejected_ranges": 1, "omitted_symbols": 10, "missing_gold_inside_map_callable_ranges": 0, "actual_source_credit_added": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "maps": 4, "symbols": 43, "rejected_ranges": 0, "omitted_symbols": 0, "missing_gold_inside_map_callable_ranges": 78, "actual_source_credit_added": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "maps": 5, "symbols": 153, "rejected_ranges": 0, "omitted_symbols": 0, "missing_gold_inside_map_callable_ranges": 332, "actual_source_credit_added": 0 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "maps": 6, "symbols": 64, "rejected_ranges": 0, "omitted_symbols": 0, "missing_gold_inside_map_callable_ranges": 0, "actual_source_credit_added": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "maps": 9, "symbols": 458, "rejected_ranges": 0, "omitted_symbols": 70, "missing_gold_inside_map_callable_ranges": 12, "actual_source_credit_added": 0 } ] }, "file_map_retrieval_development": { "status": "Complete; primary gate failed. Do not expand this policy.", "runtime_commit": "57387c83724c06ca6cb5ac82c50d38341c2f6069", "protocol_sha256": "55896c5d1602b10bf3711ce00ba7192a0edadc40a595449666ca4693ec1bfbf9", "model": "claude-opus-5-5", "sdk_reservation_usd": 5, "selected_tasks_per_arm": 4, "arms": [ "native", "file-map" ], "source_credit_from_metadata": 0, "repair_calls": 0, "independent_gain_established": false, "analysis": { "scope": "Pure ContextBench retrieval-agent development pilot. No repairs, mini-swe-agent or native tests. Cumulative verified source is primary; final served context is secondary. No independent or leaderboard claim.", "protocol_sha256": "55896c5d1602b10bf3711ce00ba7192a0edadc40a595449666ca4693ec1bfbf9", "runtime_commit": "57387c83724c06ca6cb5ac82c50d38341c2f6069", "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.08478039999999999, "map_views": [], "later_read_replies_overlapping_mapped_ranges": 0, "unconfirmed_tool_replies": 0, "source_lines": 155, "tool_calls": { "grep": 4, "read": 3 }, "sdk_turns": 8, "metrics": { "recall": 0.42738589211618255, "precision": 0.6645161290322581, "f1": 0.5202020202020202 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.21437430000000002, "map_views": [], "later_read_replies_overlapping_mapped_ranges": 0, "unconfirmed_tool_replies": 0, "source_lines": 639, "tool_calls": { "grep": 9, "glob": 1, "read": 10 }, "sdk_turns": 21, "metrics": { "recall": 0.20408163265306123, "precision": 0.06259780907668232, "f1": 0.09580838323353293 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.3606355, "map_views": [], "later_read_replies_overlapping_mapped_ranges": 0, "unconfirmed_tool_replies": 0, "source_lines": 1481, "tool_calls": { "grep": 10, "read": 9, "glob": 1 }, "sdk_turns": 21, "metrics": { "recall": 0.46238130021913804, "precision": 0.4274139095205942, "f1": 0.4442105263157895 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.1035623, "map_views": [], "later_read_replies_overlapping_mapped_ranges": 0, "unconfirmed_tool_replies": 0, "source_lines": 394, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.5362694300518135, "precision": 0.5253807106598984, "f1": 0.5307692307692308 } }, { "arm": "file-map", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.1095194, "map_views": [ { "file": "pylint/checkers/variables.py", "symbols": 75, "omitted_symbols": 0, "source_sha256": "162675ffaef0dcf3c9b302bf2b9c1a6806d18f6e79fa89bfd6a7096bab3f036c" }, { "file": "tests/functional/u/unused/unused_typing_imports.py", "symbols": 10, "omitted_symbols": 0, "source_sha256": "57e6826e6778a8f6d77ba6f93709a8b2af9db94c6a7aa38fb72fa0a283f51a61" } ], "later_read_replies_overlapping_mapped_ranges": 2, "unconfirmed_tool_replies": 0, "source_lines": 231, "tool_calls": { "grep": 4, "read": 4, "glob": 1 }, "sdk_turns": 10, "metrics": { "recall": 0.45643153526970953, "precision": 0.47619047619047616, "f1": 0.46610169491525416 } }, { "arm": "file-map", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.1810315, "map_views": [ { "file": "packages/runtime-dom/src/helpers/useCssVars.ts", "symbols": 3, "omitted_symbols": 0, "source_sha256": "e39d7a58ca27b7633a75550b335e8d481282a77e1e1ebb966b98de825e7d04c8" }, { "file": "packages/runtime-core/src/hydration.ts", "symbols": 9, "omitted_symbols": 0, "source_sha256": "8981f85a31f536747947b2c59dfc1bab43e269b3e121be396e1f128103fe309a" }, { "file": "packages/runtime-core/src/component.ts", "symbols": 20, "omitted_symbols": 0, "source_sha256": "b70e6d8b96e78a8e53b531aeb059d77c18acdb77d70fa366929d68bd3e6a9c2f" }, { "file": "packages/runtime-core/__tests__/hydration.spec.ts", "symbols": 2, "omitted_symbols": 0, "source_sha256": "0da098f8a0aba804635743812a88f3e3168164ebdbbef4eaf8962e1cbcea037d" } ], "later_read_replies_overlapping_mapped_ranges": 2, "unconfirmed_tool_replies": 0, "source_lines": 567, "tool_calls": { "grep": 5, "read": 7 }, "sdk_turns": 13, "metrics": { "recall": 0.29081632653061223, "precision": 0.10052910052910052, "f1": 0.14941022280471822 } }, { "arm": "file-map", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.3291887, "map_views": [ { "file": "src/libponyc/codegen/gencall.c", "symbols": 35, "omitted_symbols": 0, "source_sha256": "b2fbd32dabce1adbbfb0cdbac65533a9f597f1a093f8fdc1d95c5b22a8cfd039" }, { "file": "src/libponyc/verify/call.c", "symbols": 5, "omitted_symbols": 0, "source_sha256": "9cb14a23daa65008418efaed04f8313be5880c152b5dae19e300722806f08e86" }, { "file": "src/libponyc/expr/ffi.c", "symbols": 3, "omitted_symbols": 0, "source_sha256": "b5c862f15dc0c3da372eff47e47e6d0c93097589be0c0bc6b0668a6f5bccd681" }, { "file": "src/libponyc/codegen/codegen.c", "symbols": 39, "omitted_symbols": 0, "source_sha256": "37add58b6ab7057afc7979abf6d5afd581578ea216f09f74c9c5e0cdd9320d03" }, { "file": "src/libponyc/codegen/genreference.c", "symbols": 23, "omitted_symbols": 0, "source_sha256": "1df0dbd7c6b84f578f1ad8fcdc5ae18437faa587757d700b429f0fb1527409d3" }, { "file": "test/libponyc/ffi.cc", "symbols": 21, "omitted_symbols": 0, "source_sha256": "59c6457479f5a64f384a6f85ef66c29960618f011328e3653d50a513383a9673" }, { "file": "test/libponyc/util.cc", "symbols": 25, "omitted_symbols": 0, "source_sha256": "b7ed4b417084f46dc6fc63cc7d013aad94b68147ac55d49ad8d7380343f9df2f" }, { "file": "test/libponyc/bare.cc", "symbols": 30, "omitted_symbols": 0, "source_sha256": "c4306b2ecc8be19e2107ff892beec7469fdc9e30f0238b068e8acfa9c9db7de1" } ], "later_read_replies_overlapping_mapped_ranges": 2, "unconfirmed_tool_replies": 0, "source_lines": 1057, "tool_calls": { "grep": 7, "read": 11, "glob": 1 }, "sdk_turns": 20, "metrics": { "recall": 0.2914536157779401, "precision": 0.37748344370860926, "f1": 0.3289365210222589 } }, { "arm": "file-map", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.1134149, "map_views": [ { "file": "include/fmt/color.h", "symbols": 30, "omitted_symbols": 10, "source_sha256": "6e2a3dbfbbccdfa8758ff75bd155d483e829492b1402defa761c707bc78df1aa" }, { "file": "test/color-test.cc", "symbols": 3, "omitted_symbols": 0, "source_sha256": "e0df02763571419bb90d7945490bc0ab566d5153e91091986b2fa15895c0c793" } ], "later_read_replies_overlapping_mapped_ranges": 0, "unconfirmed_tool_replies": 0, "source_lines": 397, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.5362694300518135, "precision": 0.5214105793450882, "f1": 0.528735632183908 } } ], "primary": { "native": { "recall": 0.40752956376004884, "precision": 0.41997713957235827, "f1": 0.39774754013014335, "official_valid": 4 }, "file-map": { "recall": 0.3937427269075188, "precision": 0.36890339994331856, "f1": 0.3682960177315348, "official_valid": 4 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.4064922193617086, "precision": 0.43246540915492443, "f1": 0.40346007541298246, "official_valid": 4 }, "served_32768": { "recall": 0.4064922193617086, "precision": 0.4194325228987093, "f1": 0.3968109591302712, "official_valid": 4 } }, "file-map": { "served_16384": { "recall": 0.3937427269075188, "precision": 0.36908230204934156, "f1": 0.3683638676739899, "official_valid": 4 }, "served_32768": { "recall": 0.3937427269075188, "precision": 0.36908230204934156, "f1": 0.3683638676739899, "official_valid": 4 } } }, "paired_differences": { "recall": { "mean_difference": -0.013786836852529988, "exploratory_95_interval": [ -0.12093435254251671, 0.06505102040816325 ] }, "precision": { "mean_difference": -0.051073739629039744, "exploratory_95_interval": [ -0.14223677246003902, 0.016980580068803965 ] }, "f1": { "mean_difference": -0.029451522398608546, "exploratory_95_interval": [ -0.08696390361647865, 0.026676298356697454 ] } }, "gates": { "all_eight_sessions_complete": true, "known_usage": true, "one_actual_model_route": true, "all_source_replies_confirmed": true, "map_present_in_three_tasks": true, "metadata_adds_zero_source_credit": true, "greater_recall": false, "nondecreasing_f1": false }, "development_gate_passed": false, "known_new_api_sdk_usd": 1.4965070000000003, "new_sdk_sessions": 8, "repair_calls": 0, "all_frozen_hashes_match": true, "independent_gain_established": false }, "analysis_sha256": "f974bd89b90ae94b8d688721b60e0d1d88ef1f8198ef3c8303b9c60fb5a7037f" }, "grep_neighbors_retrieval_development": { "status": "Complete; initial Recall-first screen passes with wide uncertainty and lower F1. The Recall-first replication gate passes on both the new four and combined eight. This remains development evidence requiring fresh independent retrieval validation.", "runtime_commit": "0a80e3bf3d0ed6e605e651f809161b608996a381", "protocol_sha256": "e3c9d9d51e74967925857d4889823cdfed5ae8d80120bd8511bba4a69fe6d5f8", "model": "claude-opus-5-5", "sdk_reservation_usd": 5, "selected_tasks_per_arm": 4, "arms": [ "native", "neighbors" ], "extra_grep_tokens": 2048, "radius_lines": 20, "repair_calls": 0, "gate": "Prospective user-selected Recall-first primary; report Precision, F1 and cost; require all receipt/reliability audits. Historical gates unchanged.", "offline_opportunity": { "scope": "Fixed-trace opportunity only; no adaptive-agent or new online score. Canonical official gold normalization/denominators retained. No annotation-driven runtime choices or new model calls.", "source_frozen_sha256": "a9a4d5b9a968aafefe8d9e0dfad469491f9b26385fc0899a4777e422a6bc66e1", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "original": { "recall": 0.4896265560165975, "precision": 0.5244444444444445, "f1": 0.5064377682403433, "source_lines": 225 }, "replay": { "recall": 0.5767634854771784, "precision": 0.5205992509363296, "f1": 0.547244094488189, "source_lines": 267 }, "extra_source_lines": 42, "extra_annotated_lines": 21, "extra_tool_reply_tokens": 2038 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "original": { "recall": 0.30612244897959184, "precision": 0.12195121951219512, "f1": 0.17441860465116277, "source_lines": 492 }, "replay": { "recall": 0.30612244897959184, "precision": 0.07444168734491315, "f1": 0.11976047904191617, "source_lines": 806 }, "extra_source_lines": 314, "extra_annotated_lines": 0, "extra_tool_reply_tokens": 8909 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "original": { "recall": 0.5091307523739956, "precision": 0.5349194167306216, "f1": 0.5217065868263472, "source_lines": 1303 }, "replay": { "recall": 0.5193571950328707, "precision": 0.4367321867321867, "f1": 0.47447447447447444, "source_lines": 1628 }, "extra_source_lines": 325, "extra_annotated_lines": 14, "extra_tool_reply_tokens": 11906 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "original": { "recall": 0.5362694300518135, "precision": 0.5149253731343284, "f1": 0.5253807106598986, "source_lines": 402 }, "replay": { "recall": 0.5362694300518135, "precision": 0.3354943273905997, "f1": 0.4127617148554337, "source_lines": 617 }, "extra_source_lines": 215, "extra_annotated_lines": 0, "extra_tool_reply_tokens": 5755 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "original": { "recall": 0.45416666666666666, "precision": 0.8074074074074075, "f1": 0.5813333333333333, "source_lines": 270 }, "replay": { "recall": 0.4666666666666667, "precision": 0.4266666666666667, "f1": 0.44577114427860703, "source_lines": 525 }, "extra_source_lines": 255, "extra_annotated_lines": 6, "extra_tool_reply_tokens": 7259 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "original": { "recall": 0.146779303062302, "precision": 0.25272727272727274, "f1": 0.18570474281897129, "source_lines": 550 }, "replay": { "recall": 0.1921858500527983, "precision": 0.2716417910447761, "f1": 0.2251082251082251, "source_lines": 670 }, "extra_source_lines": 120, "extra_annotated_lines": 43, "extra_tool_reply_tokens": 4096 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "original": { "recall": 0.35456110154905335, "precision": 0.18933823529411764, "f1": 0.24685440383463148, "source_lines": 1088 }, "replay": { "recall": 0.37005163511187605, "precision": 0.14756348661633492, "f1": 0.21099116781157998, "source_lines": 1457 }, "extra_source_lines": 369, "extra_annotated_lines": 9, "extra_tool_reply_tokens": 8850 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "original": { "recall": 0.6666666666666666, "precision": 0.13503649635036497, "f1": 0.22458270106221548, "source_lines": 1644 }, "replay": { "recall": 0.9129129129129129, "precision": 0.15012345679012346, "f1": 0.2578456318914334, "source_lines": 2025 }, "extra_source_lines": 381, "extra_annotated_lines": 82, "extra_tool_reply_tokens": 15051 } ], "macro": { "original": { "recall": 0.4329153656708359, "precision": 0.3850937332000941, "f1": 0.3708023564283629 }, "replay": { "recall": 0.4850412030357135, "precision": 0.29540785669024133, "f1": 0.3367446164937324 } }, "all_original_source_credit_preserved": true, "new_model_calls": 0 }, "independent_gain_established": false, "analysis": { "scope": "Pure ContextBench retrieval-agent development pilot. No repairs, mini-swe-agent or native tests. Cumulative verified source is primary; final served context is secondary. No independent or leaderboard claim.", "protocol_sha256": "e3c9d9d51e74967925857d4889823cdfed5ae8d80120bd8511bba4a69fe6d5f8", "runtime_commit": "0a80e3bf3d0ed6e605e651f809161b608996a381", "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.0875044, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 219, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.38589211618257263, "precision": 0.4246575342465753, "f1": 0.4043478260869566 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.21698679999999998, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 821, "tool_calls": { "grep": 4, "glob": 1, "read": 9 }, "sdk_turns": 15, "metrics": { "recall": 0.015306122448979591, "precision": 0.0036540803897685747, "f1": 0.0058997050147492625 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.3311315, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 1660, "tool_calls": { "grep": 6, "read": 12, "glob": 1 }, "sdk_turns": 20, "metrics": { "recall": 0.38641344046749454, "precision": 0.31867469879518073, "f1": 0.349290194783757 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.1146597, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 406, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.5362694300518135, "precision": 0.5098522167487685, "f1": 0.5227272727272728 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.10484779999999999, "grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 174, "extra_reply_tokens": 3243, "budget_omitted_line_deliveries": 140, "unconfirmed_tool_replies": 0, "source_lines": 268, "tool_calls": { "grep": 2, "read": 2 }, "sdk_turns": 5, "metrics": { "recall": 0.5643153526970954, "precision": 0.5074626865671642, "f1": 0.5343811394891945 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.3047165, "grep_views": 7, "views_with_added_source": 6, "appended_line_deliveries": 537, "extra_reply_tokens": 10148, "budget_omitted_line_deliveries": 1654, "unconfirmed_tool_replies": 0, "source_lines": 1036, "tool_calls": { "grep": 8, "read": 6 }, "sdk_turns": 15, "metrics": { "recall": 0.30612244897959184, "precision": 0.05791505791505792, "f1": 0.09740259740259741 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.4926709, "grep_views": 13, "views_with_added_source": 11, "appended_line_deliveries": 1000, "extra_reply_tokens": 18654, "budget_omitted_line_deliveries": 1378, "unconfirmed_tool_replies": 0, "source_lines": 2036, "tool_calls": { "grep": 13, "read": 7, "glob": 1 }, "sdk_turns": 22, "metrics": { "recall": 0.30094959824689554, "precision": 0.20235756385068762, "f1": 0.24199706314243757 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.1079851, "grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 221, "extra_reply_tokens": 3705, "budget_omitted_line_deliveries": 288, "unconfirmed_tool_replies": 0, "source_lines": 327, "tool_calls": { "grep": 3, "read": 1 }, "sdk_turns": 5, "metrics": { "recall": 0.2616580310880829, "precision": 0.308868501529052, "f1": 0.28330995792426367 } } ], "primary": { "native": { "recall": 0.33097027728771505, "precision": 0.3142096325450733, "f1": 0.3205662496531839, "official_valid": 4 }, "neighbors": { "recall": 0.35826135775291645, "precision": 0.2691509524654904, "f1": 0.2892726894896233, "official_valid": 4 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.33097027728771505, "precision": 0.3315706852480754, "f1": 0.33006067242205267, "official_valid": 4 }, "served_32768": { "recall": 0.33097027728771505, "precision": 0.3142107465939726, "f1": 0.3205677013522525, "official_valid": 4 } }, "neighbors": { "served_16384": { "recall": 0.3572484592641742, "precision": 0.2906276510653765, "f1": 0.30193172332812407, "official_valid": 4 }, "served_32768": { "recall": 0.3572484592641742, "precision": 0.2696918028382232, "f1": 0.28917158093439277, "official_valid": 4 } } }, "paired_differences": { "recall": { "mean_difference": 0.02729108046520136, "exploratory_95_interval": [ -0.1800376205921648, 0.23461978152256752 ] }, "precision": { "mean_difference": -0.04505868007958284, "exploratory_95_interval": [ -0.1586504250821048, 0.06853306492293912 ] }, "f1": { "mean_difference": -0.03129356016356062, "exploratory_95_interval": [ -0.1733552232221643, 0.11076810289504305 ] } }, "gates": { "all_eight_sessions_complete": true, "known_usage": true, "one_actual_model_route": true, "all_source_replies_confirmed": true, "added_source_in_three_tasks": true, "original_hits_preserved_and_source_receipts_exact": true, "greater_recall": true }, "development_gate_passed": true, "known_new_api_sdk_usd": 1.7605027, "new_sdk_sessions": 8, "repair_calls": 0, "all_frozen_hashes_match": true, "independent_gain_established": false, "audit_followup": { "protocol_sha256": "ea0317fbd4d3cfbb15f5204e2566844656dfc27402153e041b1077fedafa07a9", "original_scoring_and_attempts_unchanged": true, "original_analysis_exit_code": 1, "reason": "Auxiliary grep re-execution comparison is invalid for unstable ripgrep order; validate captured originals and delivered replies against frozen source/hash receipts. No gate or selected outcome changed." } }, "analysis_sha256": "dec7aec864b204c5062738f547554389ed47121adcb130c7b34c66dd01f838a1", "audit_followup_protocol_sha256": "ea0317fbd4d3cfbb15f5204e2566844656dfc27402153e041b1077fedafa07a9", "audit_followup_preservation": { "all_preserved_official_reports_and_scores": true, "all_preserved_model_attempts": true, "original_protocol_sha256": "e3c9d9d51e74967925857d4889823cdfed5ae8d80120bd8511bba4a69fe6d5f8", "analysis_sha256": "dec7aec864b204c5062738f547554389ed47121adcb130c7b34c66dd01f838a1" } }, "grep_neighbors_replication_development": { "status": "The Recall-first replication gate passes on both the new four and combined eight. This remains development evidence requiring fresh independent retrieval validation.", "runtime_commit": "0a80e3bf3d0ed6e605e651f809161b608996a381", "protocol_sha256": "d9834073c3f0021cdb71cdc0158b644ccc0247278b2d7c4a3da4480d9ca8ba3f", "selected_tasks_per_arm": 4, "population": [ "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163" ], "sdk_reservation_usd": 5, "repair_calls": 0, "independent_gain_established": false, "gate": "User-approved prospective Recall-first gate: macro cumulative trajectory line Recall must improve versus fresh matched native across all four selected tasks. Report Precision, F1, costs, source lines and repository-paired exploratory 95% intervals regardless of sign; F1 nondecrease is not a veto. Require all eight sessions complete, exact original hit preservation, and valid source/model/route/hash/usage/delivery receipts. Candidate must deliver extra source in at least three tasks; only exact observed returned lines receive source credit. Require positive Recall both on this new four-task cohort and on all eight pilot-plus-replication tasks. Report all eight with failures retained and no independent claim.", "analysis": { "combined_eight_task_primary": { "native": { "recall": 0.35244226555743774, "precision": 0.3350156964981319, "f1": 0.3126216755822505 }, "neighbors": { "recall": 0.3890903451545774, "precision": 0.25494554576527506, "f1": 0.2819557611014275 } }, "pilot_analysis_sha256": "dec7aec864b204c5062738f547554389ed47121adcb130c7b34c66dd01f838a1", "scope": "Pure ContextBench retrieval-agent development pilot. No repairs, mini-swe-agent or native tests. Cumulative verified source is primary; final served context is secondary. No independent or leaderboard claim.", "protocol_sha256": "d9834073c3f0021cdb71cdc0158b644ccc0247278b2d7c4a3da4480d9ca8ba3f", "runtime_commit": "0a80e3bf3d0ed6e605e651f809161b608996a381", "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.0840732, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 205, "tool_calls": { "grep": 4, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 0.3625, "precision": 0.848780487804878, "f1": 0.508029197080292 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.1891437, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 849, "tool_calls": { "read": 7, "grep": 7, "glob": 1 }, "sdk_turns": 16, "metrics": { "recall": 0.20802534318901794, "precision": 0.232037691401649, "f1": 0.21937639198218262 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.2771924, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 1137, "tool_calls": { "read": 12, "grep": 4 }, "sdk_turns": 17, "metrics": { "recall": 0.35456110154905335, "precision": 0.18117854001759015, "f1": 0.23981373690337598 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.24851040000000005, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 1178, "tool_calls": { "grep": 9, "read": 7 }, "sdk_turns": 17, "metrics": { "recall": 0.5705705705705706, "precision": 0.16129032258064516, "f1": 0.2514890800794176 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.12972060000000002, "grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 354, "extra_reply_tokens": 5219, "budget_omitted_line_deliveries": 57, "unconfirmed_tool_replies": 0, "source_lines": 473, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 0.38958333333333334, "precision": 0.3953488372093023, "f1": 0.3924449108079748 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.2522375, "grep_views": 6, "views_with_added_source": 5, "appended_line_deliveries": 311, "extra_reply_tokens": 9328, "budget_omitted_line_deliveries": 933, "unconfirmed_tool_replies": 0, "source_lines": 681, "tool_calls": { "grep": 7, "read": 4, "glob": 1 }, "sdk_turns": 13, "metrics": { "recall": 0.17317845828933473, "precision": 0.24082232011747431, "f1": 0.20147420147420145 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.42864769999999996, "grep_views": 8, "views_with_added_source": 8, "appended_line_deliveries": 831, "extra_reply_tokens": 14227, "budget_omitted_line_deliveries": 1233, "unconfirmed_tool_replies": 0, "source_lines": 1777, "tool_calls": { "read": 8, "grep": 9 }, "sdk_turns": 18, "metrics": { "recall": 0.504302925989673, "precision": 0.16488463702870004, "f1": 0.24851569126378284 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.2757016, "grep_views": 7, "views_with_added_source": 6, "appended_line_deliveries": 472, "extra_reply_tokens": 7721, "budget_omitted_line_deliveries": 249, "unconfirmed_tool_replies": 0, "source_lines": 1260, "tool_calls": { "grep": 7, "read": 6 }, "sdk_turns": 14, "metrics": { "recall": 0.6126126126126126, "precision": 0.1619047619047619, "f1": 0.256120527306968 } } ], "primary": { "native": { "recall": 0.3739142538271605, "precision": 0.35582176045119057, "f1": 0.30467710151131705, "official_valid": 4 }, "neighbors": { "recall": 0.4199193325562384, "precision": 0.24074013906505964, "f1": 0.27463883271323175, "official_valid": 4 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.3739142538271605, "precision": 0.3559015747860001, "f1": 0.3047469773087831, "official_valid": 4 }, "served_32768": { "recall": 0.3739142538271605, "precision": 0.3559015747860001, "f1": 0.3047469773087831, "official_valid": 4 } }, "neighbors": { "served_16384": { "recall": 0.3837747542429854, "precision": 0.24519854354762702, "f1": 0.27298899214338024, "official_valid": 4 }, "served_32768": { "recall": 0.41905874735830384, "precision": 0.24110010837788876, "f1": 0.2747441699178148, "official_valid": 4 } } }, "paired_differences": { "recall": { "mean_difference": 0.046005078729077935, "exploratory_95_interval": [ -0.015624653164251913, 0.11907720166379805 ] }, "precision": { "mean_difference": -0.11508162138613094, "exploratory_95_interval": [ -0.3399201281156526, 0.00469953401997103 ] }, "f1": { "mean_difference": -0.0300382687980853, "exploratory_95_interval": [ -0.08553035289735035, 0.006666700793978617 ] } }, "gates": { "all_eight_sessions_complete": true, "known_usage": true, "one_actual_model_route": true, "all_source_replies_confirmed": true, "added_source_in_three_tasks": true, "original_hits_preserved_and_source_receipts_exact": true, "greater_recall": true, "combined_eight_task_recall_positive": true }, "development_gate_passed": true, "known_new_api_sdk_usd": 1.8852270999999998, "new_sdk_sessions": 8, "repair_calls": 0, "all_frozen_hashes_match": true, "independent_gain_established": false }, "analysis_sha256": "b2f122cf21cbc69ae73eafb7addeb2d50eeaa436c7ebd838cd99582dc4c58c73", "combined_eight": { "scope": "Eight disclosed development tasks; conditional extension after a positive four-task screening point estimate. No independent or leaderboard claim. All selected cases and failures retained. One matched attempt per arm per task.", "pilot_analysis_sha256": "dec7aec864b204c5062738f547554389ed47121adcb130c7b34c66dd01f838a1", "replication_analysis_sha256": "b2f122cf21cbc69ae73eafb7addeb2d50eeaa436c7ebd838cd99582dc4c58c73", "primary": { "native": { "recall": 0.35244226555743774, "precision": 0.3350156964981319, "f1": 0.3126216755822505 }, "neighbors": { "recall": 0.3890903451545774, "precision": 0.25494554576527506, "f1": 0.2819557611014275 } }, "paired_differences": { "recall": { "mean_difference": 0.03664807959713964, "exploratory_95_interval": [ -0.08052110865272841, 0.14662790769465128 ] }, "precision": { "mean_difference": -0.08007015073285689, "exploratory_95_interval": [ -0.2084680797760662, 0.020000702376969836 ] }, "f1": { "mean_difference": -0.03066591448082296, "exploratory_95_interval": [ -0.11403388638519144, 0.04412107724675366 ] } }, "api_sdk_usd": { "native": 1.5492020999999998, "neighbors": 2.0965277 }, "total_api_sdk_usd": 3.6457298, "new_repair_calls": 0, "development_replication_gate_passed": true, "independent_gain_established": false, "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.0875044, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 219, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.38589211618257263, "precision": 0.4246575342465753, "f1": 0.4043478260869566 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.21698679999999998, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 821, "tool_calls": { "grep": 4, "glob": 1, "read": 9 }, "sdk_turns": 15, "metrics": { "recall": 0.015306122448979591, "precision": 0.0036540803897685747, "f1": 0.0058997050147492625 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.3311315, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 1660, "tool_calls": { "grep": 6, "read": 12, "glob": 1 }, "sdk_turns": 20, "metrics": { "recall": 0.38641344046749454, "precision": 0.31867469879518073, "f1": 0.349290194783757 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.1146597, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 406, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.5362694300518135, "precision": 0.5098522167487685, "f1": 0.5227272727272728 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.10484779999999999, "grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 174, "extra_reply_tokens": 3243, "budget_omitted_line_deliveries": 140, "unconfirmed_tool_replies": 0, "source_lines": 268, "tool_calls": { "grep": 2, "read": 2 }, "sdk_turns": 5, "metrics": { "recall": 0.5643153526970954, "precision": 0.5074626865671642, "f1": 0.5343811394891945 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.3047165, "grep_views": 7, "views_with_added_source": 6, "appended_line_deliveries": 537, "extra_reply_tokens": 10148, "budget_omitted_line_deliveries": 1654, "unconfirmed_tool_replies": 0, "source_lines": 1036, "tool_calls": { "grep": 8, "read": 6 }, "sdk_turns": 15, "metrics": { "recall": 0.30612244897959184, "precision": 0.05791505791505792, "f1": 0.09740259740259741 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.4926709, "grep_views": 13, "views_with_added_source": 11, "appended_line_deliveries": 1000, "extra_reply_tokens": 18654, "budget_omitted_line_deliveries": 1378, "unconfirmed_tool_replies": 0, "source_lines": 2036, "tool_calls": { "grep": 13, "read": 7, "glob": 1 }, "sdk_turns": 22, "metrics": { "recall": 0.30094959824689554, "precision": 0.20235756385068762, "f1": 0.24199706314243757 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.1079851, "grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 221, "extra_reply_tokens": 3705, "budget_omitted_line_deliveries": 288, "unconfirmed_tool_replies": 0, "source_lines": 327, "tool_calls": { "grep": 3, "read": 1 }, "sdk_turns": 5, "metrics": { "recall": 0.2616580310880829, "precision": 0.308868501529052, "f1": 0.28330995792426367 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.0840732, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 205, "tool_calls": { "grep": 4, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 0.3625, "precision": 0.848780487804878, "f1": 0.508029197080292 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.1891437, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 849, "tool_calls": { "read": 7, "grep": 7, "glob": 1 }, "sdk_turns": 16, "metrics": { "recall": 0.20802534318901794, "precision": 0.232037691401649, "f1": 0.21937639198218262 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.2771924, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 1137, "tool_calls": { "read": 12, "grep": 4 }, "sdk_turns": 17, "metrics": { "recall": 0.35456110154905335, "precision": 0.18117854001759015, "f1": 0.23981373690337598 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.24851040000000005, "grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "budget_omitted_line_deliveries": 0, "unconfirmed_tool_replies": 0, "source_lines": 1178, "tool_calls": { "grep": 9, "read": 7 }, "sdk_turns": 17, "metrics": { "recall": 0.5705705705705706, "precision": 0.16129032258064516, "f1": 0.2514890800794176 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.12972060000000002, "grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 354, "extra_reply_tokens": 5219, "budget_omitted_line_deliveries": 57, "unconfirmed_tool_replies": 0, "source_lines": 473, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 0.38958333333333334, "precision": 0.3953488372093023, "f1": 0.3924449108079748 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.2522375, "grep_views": 6, "views_with_added_source": 5, "appended_line_deliveries": 311, "extra_reply_tokens": 9328, "budget_omitted_line_deliveries": 933, "unconfirmed_tool_replies": 0, "source_lines": 681, "tool_calls": { "grep": 7, "read": 4, "glob": 1 }, "sdk_turns": 13, "metrics": { "recall": 0.17317845828933473, "precision": 0.24082232011747431, "f1": 0.20147420147420145 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.42864769999999996, "grep_views": 8, "views_with_added_source": 8, "appended_line_deliveries": 831, "extra_reply_tokens": 14227, "budget_omitted_line_deliveries": 1233, "unconfirmed_tool_replies": 0, "source_lines": 1777, "tool_calls": { "read": 8, "grep": 9 }, "sdk_turns": 18, "metrics": { "recall": 0.504302925989673, "precision": 0.16488463702870004, "f1": 0.24851569126378284 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.2757016, "grep_views": 7, "views_with_added_source": 6, "appended_line_deliveries": 472, "extra_reply_tokens": 7721, "budget_omitted_line_deliveries": 249, "unconfirmed_tool_replies": 0, "source_lines": 1260, "tool_calls": { "grep": 7, "read": 6 }, "sdk_turns": 14, "metrics": { "recall": 0.6126126126126126, "precision": 0.1619047619047619, "f1": 0.256120527306968 } } ] }, "combined_analysis_sha256": "a66bc79b34db11286b94b35e7e8a0f3b86a592664be4d1251da11139a9ed589b" }, "fresh_neighbors_independent32": { "status": "The frozen independent Recall gate fails. Do not promote this unchanged policy into repairs, official500 or a product default. Positive development screening is preserved, but has not established a stable independent gain.", "scope": "Fresh independent task-level 32-case retrieval-only validation of the frozen hit-preserving neighborhood policy. Seven languages; no untouched C++ tasks remain. Not a repository-held-out or representative full500 estimate. Old independent32 remains sealed. No repairs/native tests.", "runtime_commit": "0a80e3bf3d0ed6e605e651f809161b608996a381", "protocol_sha256": "b4b92a3e2bd02a7fde3a86b6f86edeb83f23c2492e896ac5f3c6d4e3e0ccf038", "selection_sha256": "956370f5959465b10fe874811847d437fb1c98638b96a3573402874ee0023d7f", "source_manifest_sha256": "9c59597e5e758f2a414bfc5568cfc29462d2b119156d8cd7068085ad45d997ab", "selection_protocol_sha256": "9376989a72ab5e83ba5bf0d57a3baac0d830e0c45b057ce3ad128d196ca02514", "dataset_revision": "c2855792b006af41c67202d33883fb9d46362853", "dataset_sha256": "e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791", "evaluator_revision": "1436c28a8eb95496da4ea69ad458b9f8a8eb7d61", "selected_tasks": 32, "expected_sdk_sessions": 64, "languages": { "c": 4, "python": 8, "javascript": 4, "java": 4, "go": 4, "typescript": 4, "rust": 4 }, "repository_names": 21, "selected_ids": [ "Multi-SWE-Bench__c__maintenance__bugfix__b9b45262", "SWE-Bench-Pro__python__maintenance__bugfix__9c878ad5", "Multi-SWE-Bench__javascript__maintenance__bugfix__b31d2f53", "Multi-SWE-Bench__java__maintenance__bugfix__2bd87230", "SWE-Bench-Verified__python__maintenance__bugfix__b10492be", "Multi-SWE-Bench__go__maintenance__bugfix__3d1b3145", "SWE-Bench-Verified__python__maintenance__bugfix__a14d0e2e", "Multi-SWE-Bench__typescript__maintenance__bugfix__6a53360d", "Multi-SWE-Bench__typescript__maintenance__bugfix__a7aba10f", "Multi-SWE-Bench__rust__maintenance__bugfix__37f525d2", "SWE-PolyBench__python__maintenance__bugfix__8b1080ea", "Multi-SWE-Bench__go__maintenance__bugfix__0498ad7f", "Multi-SWE-Bench__rust__maintenance__bugfix__908342f0", "SWE-Bench-Verified__python__maintenance__bugfix__28ed0b77", "Multi-SWE-Bench__go__maintenance__bugfix__2a889a1d", "Multi-SWE-Bench__go__maintenance__bugfix__1ba303a5", "SWE-Bench-Pro__javascript__maintenance__bugfix__7b6185af", "SWE-PolyBench__typescript__maintenance__bugfix__2487b02a", "Multi-SWE-Bench__c__maintenance__bugfix__8f9042e3", "SWE-PolyBench__javascript__maintenance__bugfix__2d7a3934", "SWE-PolyBench__typescript__maintenance__bugfix__6da325be", "SWE-Bench-Verified__python__maintenance__bugfix__deb49033", "SWE-Bench-Verified__python__maintenance__bugfix__4f130690", "Multi-SWE-Bench__java__maintenance__bugfix__36a08326", "Multi-SWE-Bench__rust__maintenance__bugfix__c9cd2b2c", "Multi-SWE-Bench__c__maintenance__bugfix__1e842312", "Multi-SWE-Bench__rust__maintenance__bugfix__3c69099b", "Multi-SWE-Bench__java__maintenance__bugfix__f0db9a27", "Multi-SWE-Bench__java__maintenance__bugfix__5eea2978", "Multi-SWE-Bench__c__maintenance__bugfix__6b844996", "SWE-Bench-Pro__javascript__maintenance__bugfix__593287ea", "SWE-Bench-Verified__python__maintenance__bugfix__e5236b5f" ], "selection_rule": "Fixed quota: eight Python, four each of C/Go/Java/JavaScript/Rust/TypeScript; seeded SHA ordering and repository round-robin on metadata only. Exclude all105 development and sealed prior32. No C++ available; no full500 distribution claim.", "source_preparations_ready": 32, "runtime_gold_context_empty": true, "gold_loaded_during_inference": false, "model": "claude-opus-5-5", "max_turns": 20, "case_timeout_s": 600, "max_output_tokens": 4096, "sdk_reservation_usd": 22, "forecast_sdk_usd": 14.5829192, "billing_route": "anthropic-api", "primary_gate": "Paired all-32 macro cumulative confirmed-source line Recall difference. Require positive lower endpoint of a two-sided repository-cluster 95% bootstrap interval (5000 resamples, seed20261017), complete known usage/model/route/source/delivery receipts, all 64 terminal sessions with expected budget statuses permitted, and 32 valid official scores per arm. F1 nondecrease is not a veto under explicit user preference. All selected failures remain in denominators; report raw counts, failures, Precision, macro F1 and complete costs.", "bootstrap": "Canonical repository URL clusters, stripping trailing slash and .git and lowercasing. Sample whole repositories with replacement, retaining every task in each sampled cluster; compute task-macro paired differences. Report selected tasks, repository count, languages and official-valid counts. This balanced seven-language cohort does not estimate the full500 distribution.", "annotation_isolation": "Runtime selected.jsonl contains only metadata, issue text and explicit empty gold_context. Do not load annotation columns, patches or native test labels during inference. After terminal execution and immutable output receipts, materialize only gold_context in a separate evaluator cohort; keep original source-only runtime configs and attempts unchanged.", "repair_calls": 0, "official500_started": false, "independent_gain_established": false, "pre_scoring_projection_correction": { "protocol_sha256": "0d2edff293b6fc63cf54657f22a1fa0c72512b6ea2d0fa7d72f284bcd0b3e41f", "reason": "Recompute cached trajectory and served predictions from confirmed messages; message filtering alone does not update the stored predictions consumed by the evaluator.", "registered_before_annotation_access": true, "original_analyzer_preserved": true, "paid_runtime_policy_and_gates_unchanged": true, "new_model_calls": 0, "regression": { "raw_source_lines": 2, "confirmed_source_lines": 1, "unconfirmed_line_omitted": true } }, "analysis": { "scope": "Fresh independent task-level 32-case retrieval-only validation of the frozen hit-preserving neighborhood policy. Seven languages; no untouched C++ tasks remain. Not a repository-held-out or representative full500 estimate. Old independent32 remains sealed. No repairs/native tests.", "protocol_sha256": "b4b92a3e2bd02a7fde3a86b6f86edeb83f23c2492e896ac5f3c6d4e3e0ccf038", "runtime_commit": "0a80e3bf3d0ed6e605e651f809161b608996a381", "selection_sha256": "956370f5959465b10fe874811847d437fb1c98638b96a3573402874ee0023d7f", "tasks_per_arm": 32, "repositories": 21, "languages": { "c": 4, "python": 8, "javascript": 4, "java": 4, "go": 4, "typescript": 4, "rust": 4 }, "rows": [ { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b9b45262", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "status": "ok", "sdk_usd": 0.5465633999999999, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1665, "tool_calls": { "glob": 4, "grep": 6, "read": 10 }, "sdk_turns": 21, "metrics": { "recall": 0.21875, "precision": 0.008408408408408409, "f1": 0.016194331983805668 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__9c878ad5", "repo": "ansible/ansible", "repo_url": "https://github.com/ansible/ansible.git", "language": "python", "status": "ok", "sdk_usd": 0.16450030000000002, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 502, "tool_calls": { "read": 4, "grep": 3 }, "sdk_turns": 8, "metrics": { "recall": 0.9216867469879518, "precision": 0.3047808764940239, "f1": 0.45808383233532934 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__b31d2f53", "repo": "iamkun/dayjs", "repo_url": "https://github.com/iamkun/dayjs.git", "language": "javascript", "status": "ok", "sdk_usd": 0.0807602, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 244, "tool_calls": { "read": 4, "grep": 2, "glob": 1 }, "sdk_turns": 8, "metrics": { "recall": 0.14527027027027026, "precision": 0.3524590163934426, "f1": 0.20574162679425836 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__2bd87230", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "status": "ok", "sdk_usd": 0.1389317, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 331, "tool_calls": { "grep": 3, "read": 5 }, "sdk_turns": 9, "metrics": { "recall": 0.14285714285714285, "precision": 0.0030211480362537764, "f1": 0.005917159763313609 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__b10492be", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "status": "ok", "sdk_usd": 0.1019803, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 254, "tool_calls": { "grep": 3, "read": 5 }, "sdk_turns": 9, "metrics": { "recall": 0.2830188679245283, "precision": 0.23622047244094488, "f1": 0.2575107296137339 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3d1b3145", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.11487610000000001, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 690, "tool_calls": { "grep": 1, "read": 3 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.02753623188405797, "f1": 0.0535966149506347 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__a14d0e2e", "repo": "matplotlib/matplotlib", "repo_url": "https://github.com/matplotlib/matplotlib.git", "language": "python", "status": "ok", "sdk_usd": 0.0699112, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 145, "tool_calls": { "grep": 4, "read": 2 }, "sdk_turns": 7, "metrics": { "recall": 0.5079365079365079, "precision": 0.8827586206896552, "f1": 0.6448362720403022 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__6a53360d", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "status": "ok", "sdk_usd": 0.1069804, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 265, "tool_calls": { "read": 3, "grep": 5 }, "sdk_turns": 9, "metrics": { "recall": 0.6842105263157895, "precision": 0.04905660377358491, "f1": 0.09154929577464789 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__a7aba10f", "repo": "mui/material", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "sdk_usd": 0.2165217, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 765, "tool_calls": { "glob": 6, "grep": 5, "read": 6 }, "sdk_turns": 18, "metrics": { "recall": 0.5776614310645725, "precision": 0.4326797385620915, "f1": 0.4947683109118087 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__37f525d2", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.1384774, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 710, "tool_calls": { "grep": 3, "read": 5 }, "sdk_turns": 9, "metrics": { "recall": 1.0, "precision": 0.023943661971830985, "f1": 0.04676753782668501 } }, { "arm": "native", "instance_id": "SWE-PolyBench__python__maintenance__bugfix__8b1080ea", "repo": "keras-team/keras", "repo_url": "https://github.com/keras-team/keras.git", "language": "python", "status": "ok", "sdk_usd": 0.21250110000000003, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 857, "tool_calls": { "grep": 5, "read": 6 }, "sdk_turns": 12, "metrics": { "recall": 0.208, "precision": 0.030338389731621937, "f1": 0.0529531568228106 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__0498ad7f", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.3437835, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1188, "tool_calls": { "grep": 8, "glob": 5, "read": 7 }, "sdk_turns": 21, "metrics": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__908342f0", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.1620258, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 732, "tool_calls": { "grep": 5, "read": 5 }, "sdk_turns": 11, "metrics": { "recall": 1.0, "precision": 0.023224043715846996, "f1": 0.045393858477970624 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__28ed0b77", "repo": "sphinx-doc/sphinx", "repo_url": "https://github.com/sphinx-doc/sphinx.git", "language": "python", "status": "ok", "sdk_usd": 0.1155273, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 207, "tool_calls": { "grep": 4, "read": 2 }, "sdk_turns": 7, "metrics": { "recall": 0.4892703862660944, "precision": 0.5507246376811594, "f1": 0.5181818181818182 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__2a889a1d", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.35007849999999996, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 2089, "tool_calls": { "grep": 6, "glob": 2, "read": 11 }, "sdk_turns": 20, "metrics": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__1ba303a5", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.17099530000000002, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1113, "tool_calls": { "grep": 1, "read": 4 }, "sdk_turns": 6, "metrics": { "recall": 0.7170454545454545, "precision": 0.5669362084456424, "f1": 0.6332162568991471 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__7b6185af", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "status": "ok", "sdk_usd": 0.2110507, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1157, "tool_calls": { "grep": 5, "read": 10 }, "sdk_turns": 16, "metrics": { "recall": 0.5364891518737672, "precision": 0.23509075194468454, "f1": 0.32692307692307687 } }, { "arm": "native", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__2487b02a", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "status": "ok", "sdk_usd": 0.1991387, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 927, "tool_calls": { "glob": 5, "grep": 2, "read": 3 }, "sdk_turns": 11, "metrics": { "recall": 0.8679245283018868, "precision": 0.04962243797195254, "f1": 0.09387755102040816 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__8f9042e3", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "sdk_usd": 0.42183820000000005, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 2030, "tool_calls": { "grep": 10, "read": 12, "glob": 1 }, "sdk_turns": 24, "metrics": { "recall": 0.9895397489539749, "precision": 0.2330049261083744, "f1": 0.3771929824561404 } }, { "arm": "native", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__2d7a3934", "repo": "prettier/prettier", "repo_url": "https://github.com/prettier/prettier.git", "language": "javascript", "status": "ok", "sdk_usd": 0.114116, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 470, "tool_calls": { "grep": 4, "read": 6 }, "sdk_turns": 11, "metrics": { "recall": 0.6426966292134831, "precision": 0.6085106382978723, "f1": 0.625136612021858 } }, { "arm": "native", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__6da325be", "repo": "mui/material-ui", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "sdk_usd": 0.1383723, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 680, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 1.0, "precision": 0.07647058823529412, "f1": 0.14207650273224046 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__deb49033", "repo": "astropy/astropy", "repo_url": "https://github.com/astropy/astropy.git", "language": "python", "status": "ok", "sdk_usd": 0.2801569, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1095, "tool_calls": { "glob": 1, "read": 11, "grep": 3 }, "sdk_turns": 16, "metrics": { "recall": 0.6567656765676567, "precision": 0.1817351598173516, "f1": 0.28469241773962806 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__4f130690", "repo": "scikit-learn/scikit-learn", "repo_url": "https://github.com/scikit-learn/scikit-learn.git", "language": "python", "status": "ok", "sdk_usd": 0.0879306, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 254, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 0.813953488372093, "precision": 0.41338582677165353, "f1": 0.5483028720626631 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__36a08326", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "status": "ok", "sdk_usd": 0.1583332, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 508, "tool_calls": { "grep": 5, "read": 6 }, "sdk_turns": 12, "metrics": { "recall": 0.43243243243243246, "precision": 0.4094488188976378, "f1": 0.42062689585439844 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__c9cd2b2c", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.0652803, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 205, "tool_calls": { "grep": 4, "read": 3 }, "sdk_turns": 8, "metrics": { "recall": 0.9473684210526315, "precision": 0.08780487804878048, "f1": 0.1607142857142857 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__1e842312", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "sdk_usd": 0.2037289, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1079, "tool_calls": { "grep": 4, "read": 9 }, "sdk_turns": 14, "metrics": { "recall": 0.9795918367346939, "precision": 0.2669138090824838, "f1": 0.4195193008011653 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3c69099b", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.20715050000000002, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1036, "tool_calls": { "grep": 2, "read": 12 }, "sdk_turns": 15, "metrics": { "recall": 0.9243697478991597, "precision": 0.21235521235521235, "f1": 0.3453689167974882 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__f0db9a27", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-core.git", "language": "java", "status": "ok", "sdk_usd": 0.1462275, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 541, "tool_calls": { "grep": 6, "glob": 4, "read": 3 }, "sdk_turns": 14, "metrics": { "recall": 1.0, "precision": 0.031423290203327174, "f1": 0.06093189964157707 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__5eea2978", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-core.git", "language": "java", "status": "ok", "sdk_usd": 0.32082560000000004, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1146, "tool_calls": { "grep": 7, "read": 9 }, "sdk_turns": 17, "metrics": { "recall": 1.0, "precision": 0.06631762652705062, "f1": 0.12438625204582653 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__6b844996", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "status": "ok", "sdk_usd": 0.18868559999999998, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 633, "tool_calls": { "grep": 6, "read": 5 }, "sdk_turns": 12, "metrics": { "recall": 0.7410071942446043, "precision": 0.1627172195892575, "f1": 0.2668393782383419 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__593287ea", "repo": "element-hq/element-web", "repo_url": "https://github.com/element-hq/element-web.git", "language": "javascript", "status": "ok", "sdk_usd": 0.2424598, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 892, "tool_calls": { "glob": 2, "grep": 7, "read": 13 }, "sdk_turns": 23, "metrics": { "recall": 0.5372340425531915, "precision": 0.226457399103139, "f1": 0.31861198738170343 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__e5236b5f", "repo": "pytest-dev/pytest", "repo_url": "https://github.com/pytest-dev/pytest.git", "language": "python", "status": "ok", "sdk_usd": 0.08886540000000001, "grep_views": 0, "confirmed_grep_views": 0, "views_with_added_source": 0, "appended_line_deliveries": 0, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 212, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 1.0, "precision": 0.18867924528301888, "f1": 0.3174603174603175 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b9b45262", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "status": "ok", "sdk_usd": 0.23760240000000002, "grep_views": 2, "confirmed_grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 155, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 981, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.21875, "precision": 0.014271151885830785, "f1": 0.026794258373205742 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__9c878ad5", "repo": "ansible/ansible", "repo_url": "https://github.com/ansible/ansible.git", "language": "python", "status": "ok", "sdk_usd": 0.2249871, "grep_views": 2, "confirmed_grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 205, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 794, "tool_calls": { "read": 3, "grep": 3 }, "sdk_turns": 7, "metrics": { "recall": 0.9518072289156626, "precision": 0.19899244332493704, "f1": 0.3291666666666666 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__b31d2f53", "repo": "iamkun/dayjs", "repo_url": "https://github.com/iamkun/dayjs.git", "language": "javascript", "status": "ok", "sdk_usd": 0.10231589999999999, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 2, "appended_line_deliveries": 237, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 319, "tool_calls": { "glob": 2, "grep": 4, "read": 1 }, "sdk_turns": 8, "metrics": { "recall": 0.10810810810810811, "precision": 0.2006269592476489, "f1": 0.14050493962678376 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__2bd87230", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "status": "ok", "sdk_usd": 0.2652103, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 4, "appended_line_deliveries": 279, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 722, "tool_calls": { "grep": 6, "read": 5 }, "sdk_turns": 12, "metrics": { "recall": 0.42857142857142855, "precision": 0.004155124653739612, "f1": 0.00823045267489712 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__b10492be", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "status": "ok", "sdk_usd": 0.1180727, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 230, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 390, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.2830188679245283, "precision": 0.15384615384615385, "f1": 0.19933554817275748 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3d1b3145", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.09138349999999999, "grep_views": 1, "confirmed_grep_views": 1, "views_with_added_source": 1, "appended_line_deliveries": 120, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 435, "tool_calls": { "read": 2, "grep": 1 }, "sdk_turns": 4, "metrics": { "recall": 1.0, "precision": 0.04367816091954023, "f1": 0.08370044052863436 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__a14d0e2e", "repo": "matplotlib/matplotlib", "repo_url": "https://github.com/matplotlib/matplotlib.git", "language": "python", "status": "ok", "sdk_usd": 0.0621638, "grep_views": 2, "confirmed_grep_views": 2, "views_with_added_source": 1, "appended_line_deliveries": 110, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 161, "tool_calls": { "grep": 2, "read": 1 }, "sdk_turns": 4, "metrics": { "recall": 0.5476190476190477, "precision": 0.8571428571428571, "f1": 0.6682808716707023 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__6a53360d", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "status": "ok", "sdk_usd": 0.1594738, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 5, "appended_line_deliveries": 334, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 502, "tool_calls": { "grep": 5, "read": 2 }, "sdk_turns": 8, "metrics": { "recall": 0.6842105263157895, "precision": 0.025896414342629483, "f1": 0.04990403071017275 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__a7aba10f", "repo": "mui/material", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "sdk_usd": 0.23219109999999998, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 294, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 933, "tool_calls": { "glob": 1, "grep": 4, "read": 6 }, "sdk_turns": 12, "metrics": { "recall": 0.6457242582897034, "precision": 0.3965702036441586, "f1": 0.49136786188579024 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__37f525d2", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.1295828, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 294, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 453, "tool_calls": { "grep": 4, "read": 3 }, "sdk_turns": 8, "metrics": { "recall": 1.0, "precision": 0.037527593818984545, "f1": 0.07234042553191489 } }, { "arm": "neighbors", "instance_id": "SWE-PolyBench__python__maintenance__bugfix__8b1080ea", "repo": "keras-team/keras", "repo_url": "https://github.com/keras-team/keras.git", "language": "python", "status": "ok", "sdk_usd": 0.2504847, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 4, "appended_line_deliveries": 374, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1130, "tool_calls": { "grep": 4, "read": 5 }, "sdk_turns": 10, "metrics": { "recall": 0.36, "precision": 0.03982300884955752, "f1": 0.07171314741035856 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__0498ad7f", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.13459369999999998, "grep_views": 1, "confirmed_grep_views": 1, "views_with_added_source": 1, "appended_line_deliveries": 92, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 747, "tool_calls": { "grep": 2, "read": 3 }, "sdk_turns": 6, "metrics": { "recall": 1.0, "precision": 0.02677376171352075, "f1": 0.05215123859191656 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__908342f0", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.21060700000000002, "grep_views": 6, "confirmed_grep_views": 6, "views_with_added_source": 6, "appended_line_deliveries": 676, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 927, "tool_calls": { "grep": 6, "read": 3 }, "sdk_turns": 10, "metrics": { "recall": 1.0, "precision": 0.018338727076591153, "f1": 0.036016949152542374 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__28ed0b77", "repo": "sphinx-doc/sphinx", "repo_url": "https://github.com/sphinx-doc/sphinx.git", "language": "python", "status": "ok", "sdk_usd": 0.18718070000000003, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 4, "appended_line_deliveries": 291, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 359, "tool_calls": { "grep": 4, "read": 2, "glob": 3 }, "sdk_turns": 10, "metrics": { "recall": 0.6051502145922747, "precision": 0.39275766016713093, "f1": 0.4763513513513513 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__2a889a1d", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.295301, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 4, "appended_line_deliveries": 516, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1313, "tool_calls": { "grep": 4, "read": 13 }, "sdk_turns": 18, "metrics": { "recall": 0.14705882352941177, "precision": 0.003808073115003808, "f1": 0.007423904974016333 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__1ba303a5", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "sdk_usd": 0.2054453, "grep_views": 2, "confirmed_grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 201, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 955, "tool_calls": { "grep": 2, "read": 4 }, "sdk_turns": 7, "metrics": { "recall": 0.6318181818181818, "precision": 0.5821989528795811, "f1": 0.6059945504087194 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__7b6185af", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "status": "ok", "sdk_usd": 0.2573728, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 5, "appended_line_deliveries": 594, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1205, "tool_calls": { "grep": 5, "read": 7 }, "sdk_turns": 13, "metrics": { "recall": 0.5187376725838264, "precision": 0.21825726141078838, "f1": 0.3072429906542056 } }, { "arm": "neighbors", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__2487b02a", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "status": "ok", "sdk_usd": 0.2622527, "grep_views": 2, "confirmed_grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 135, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1071, "tool_calls": { "glob": 1, "grep": 3, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 0.9056603773584906, "precision": 0.04481792717086835, "f1": 0.08540925266903915 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__8f9042e3", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "sdk_usd": 0.4199812, "grep_views": 6, "confirmed_grep_views": 6, "views_with_added_source": 6, "appended_line_deliveries": 618, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1970, "tool_calls": { "grep": 7, "read": 7, "glob": 1 }, "sdk_turns": 16, "metrics": { "recall": 0.997907949790795, "precision": 0.24213197969543146, "f1": 0.3897058823529411 } }, { "arm": "neighbors", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__2d7a3934", "repo": "prettier/prettier", "repo_url": "https://github.com/prettier/prettier.git", "language": "javascript", "status": "ok", "sdk_usd": 0.1527612, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 403, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 606, "tool_calls": { "grep": 4, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 0.6494382022471911, "precision": 0.4768976897689769, "f1": 0.5499524262607042 } }, { "arm": "neighbors", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__6da325be", "repo": "mui/material-ui", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "sdk_usd": 0.18583729999999998, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 4, "appended_line_deliveries": 380, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 704, "tool_calls": { "grep": 4, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 1.0, "precision": 0.07386363636363637, "f1": 0.13756613756613756 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__deb49033", "repo": "astropy/astropy", "repo_url": "https://github.com/astropy/astropy.git", "language": "python", "status": "ok", "sdk_usd": 0.2590545, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 5, "appended_line_deliveries": 383, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 811, "tool_calls": { "glob": 1, "read": 6, "grep": 5 }, "sdk_turns": 13, "metrics": { "recall": 0.5544554455445545, "precision": 0.20715166461159062, "f1": 0.3016157989228007 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__4f130690", "repo": "scikit-learn/scikit-learn", "repo_url": "https://github.com/scikit-learn/scikit-learn.git", "language": "python", "status": "ok", "sdk_usd": 0.09288569999999999, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 160, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 240, "tool_calls": { "grep": 3, "read": 1 }, "sdk_turns": 5, "metrics": { "recall": 0.6589147286821705, "precision": 0.3541666666666667, "f1": 0.4607046070460705 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__36a08326", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "status": "ok", "sdk_usd": 0.258035, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 4, "appended_line_deliveries": 253, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 811, "tool_calls": { "grep": 6, "read": 5 }, "sdk_turns": 12, "metrics": { "recall": 0.4261954261954262, "precision": 0.2527743526510481, "f1": 0.3173374613003096 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__c9cd2b2c", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.0943443, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 4, "appended_line_deliveries": 267, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 283, "tool_calls": { "grep": 5 }, "sdk_turns": 6, "metrics": { "recall": 0.9473684210526315, "precision": 0.0636042402826855, "f1": 0.11920529801324503 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__1e842312", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "sdk_usd": 0.2750004, "grep_views": 6, "confirmed_grep_views": 6, "views_with_added_source": 5, "appended_line_deliveries": 591, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1376, "tool_calls": { "grep": 6, "read": 5 }, "sdk_turns": 12, "metrics": { "recall": 0.9693877551020408, "precision": 0.20712209302325582, "f1": 0.34131736526946105 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3c69099b", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "sdk_usd": 0.18674300000000002, "grep_views": 2, "confirmed_grep_views": 2, "views_with_added_source": 2, "appended_line_deliveries": 257, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 884, "tool_calls": { "grep": 2, "read": 7 }, "sdk_turns": 10, "metrics": { "recall": 0.6428571428571429, "precision": 0.17307692307692307, "f1": 0.27272727272727276 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__f0db9a27", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-core.git", "language": "java", "status": "ok", "sdk_usd": 0.2297221, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 4, "appended_line_deliveries": 324, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 847, "tool_calls": { "grep": 5, "read": 2, "glob": 2 }, "sdk_turns": 10, "metrics": { "recall": 1.0, "precision": 0.020070838252656435, "f1": 0.03935185185185185 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__5eea2978", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-core.git", "language": "java", "status": "ok", "sdk_usd": 0.35817030000000005, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 3, "appended_line_deliveries": 217, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1312, "tool_calls": { "grep": 4, "read": 9, "glob": 1 }, "sdk_turns": 15, "metrics": { "recall": 1.0, "precision": 0.057926829268292686, "f1": 0.10951008645533142 } }, { "arm": "neighbors", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__6b844996", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "status": "ok", "sdk_usd": 0.17916880000000002, "grep_views": 4, "confirmed_grep_views": 4, "views_with_added_source": 3, "appended_line_deliveries": 306, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 600, "tool_calls": { "grep": 4, "read": 2 }, "sdk_turns": 7, "metrics": { "recall": 0.6258992805755396, "precision": 0.145, "f1": 0.23545331529093366 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__593287ea", "repo": "element-hq/element-web", "repo_url": "https://github.com/element-hq/element-web.git", "language": "javascript", "status": "ok", "sdk_usd": 0.27155640000000003, "grep_views": 5, "confirmed_grep_views": 5, "views_with_added_source": 5, "appended_line_deliveries": 474, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 1098, "tool_calls": { "glob": 1, "grep": 7, "read": 9 }, "sdk_turns": 18, "metrics": { "recall": 0.550531914893617, "precision": 0.1885245901639344, "f1": 0.2808683853459973 } }, { "arm": "neighbors", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__e5236b5f", "repo": "pytest-dev/pytest", "repo_url": "https://github.com/pytest-dev/pytest.git", "language": "python", "status": "ok", "sdk_usd": 0.1325127, "grep_views": 3, "confirmed_grep_views": 3, "views_with_added_source": 3, "appended_line_deliveries": 298, "unconfirmed_tool_replies_excluded": 0, "confirmed_source_lines": 387, "tool_calls": { "grep": 4, "read": 2 }, "sdk_turns": 7, "metrics": { "recall": 1.0, "precision": 0.10335917312661498, "f1": 0.18735362997658078 } } ], "primary": { "native": { "recall": 0.6551587572614965, "precision": 0.21693830895205174, "f1": 0.2611678766021058, "official_valid": 32 }, "neighbors": { "recall": 0.6893497188302363, "precision": 0.1820360347550386, "f1": 0.232956199982291, "official_valid": 32 } }, "paired_differences": { "recall": { "mean_difference": 0.034190961568739846, "repository_cluster_95_interval": [ -0.022676075522721578, 0.10084965455592514 ] }, "precision": { "mean_difference": -0.03490227419701314, "repository_cluster_95_interval": [ -0.05856450518712836, -0.01659572590896617 ] }, "f1": { "mean_difference": -0.02821167661981478, "repository_cluster_95_interval": [ -0.04632130897774394, -0.013158127328984442 ] } }, "language_macro_secondary": { "native": { "recall": 0.6615987284050168, "precision": 0.1981326525718193, "f1": 0.24344147654068155 }, "neighbors": { "recall": 0.6992395798545873, "precision": 0.16684047493905288, "f1": 0.21811919960088103 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.6551059700993344, "precision": 0.22018444053376854, "f1": 0.26502656827032667, "official_valid": 32 }, "served_32768": { "recall": 0.6551059700993344, "precision": 0.21720527233450732, "f1": 0.2612855125896857, "official_valid": 32 } }, "neighbors": { "served_16384": { "recall": 0.6885077767265095, "precision": 0.1852481082068965, "f1": 0.23669002261916033, "official_valid": 32 }, "served_32768": { "recall": 0.6885077767265095, "precision": 0.18239849192351398, "f1": 0.2331624612283883, "official_valid": 32 } } }, "gates": { "all_64_sessions_terminal": true, "all_usage_known": true, "one_actual_model_route": true, "all_source_config_and_delivery_receipts_verified": true, "unconfirmed_source_not_scored": true, "gold_only_loaded_after_terminal_inference": true, "all_official_scores_valid": true, "positive_lower_recall_interval": false, "usage_within_reservation": true }, "independent_gate_passed": false, "known_new_api_sdk_usd": 12.630568599999995, "new_sdk_sessions": 64, "repair_calls": 0, "runtime_inputs_contained_gold": false, "annotation_materialization_sha256": "6af8314e7afa9d2a6068920ad0d356c1cff6afaad7b0cf7fec1146b9e7033ed9", "all_frozen_hashes_match": true, "official500_started": false, "scoring_correction_protocol_sha256": "0d2edff293b6fc63cf54657f22a1fa0c72512b6ea2d0fa7d72f284bcd0b3e41f" }, "analysis_sha256": "b5ec503532f036920582f212b649db89d513c3c06d45cda93bed20dfa559c6f6", "annotation_materialization_sha256": "6af8314e7afa9d2a6068920ad0d356c1cff6afaad7b0cf7fec1146b9e7033ed9", "analysis_output_receipts_sha256": "5bd115d63518e484a571a87f00f38372c570632b881faba70b2f1d666ad01150", "source_confirmed_scoring_exclusions": { "native": 0, "neighbors": 0 }, "api_sdk_estimate_usd": 12.630568599999995, "annotation_source_audit": { "scope": "Evaluation-only official canonical annotation/source issues; same gold paths and denominators retained in both arms. No remapping, filtering or runtime use.", "cases_with_source_issues": 7, "audit_sha256": "eb221be3d9025426d4b5390becbbd426e4487648a699b9a7c13f186582eed036", "same_gold_paths_and_denominators_in_both_arms": true, "all_selected_tasks_retained": 32 } }, "post_neighbors_validation_untouched_pool": { "scope": "Historical metadata inventory immediately after sealing the second independent32 and before the separately recorded documentation-preview exposure. Initial post-quarantine counts appear in broader_validation_candidate_inventory; current broader source-identity counts appear in known_source_exposure_inventory.", "excluded_development": 105, "excluded_validation_cohorts": 64, "remaining": 331, "remaining_per_language": { "c": 7, "cpp": 0, "go": 22, "java": 0, "javascript": 40, "python": 212, "rust": 5, "typescript": 45 }, "consequence": "No untouched C++ or Java remains. Do not recycle either validation cohort or imply that future samples preserve the current language coverage." }, "latest_context_strategy_status": "Independent32 sourcecoverage unchanged:Recall74.94to79.09,F130.13to22.32,repository95Recall delta[+1.99,+6.85]points. Canonical500 ongoing;oneSDK-rejected misspelledtool action stoppedstrictgate despite normalterminal/knownusage/validsource. Preservefailedstrictgate;newprelabel sourcequalification givesrejectedaction0credit,retainsotherprovenreads withoutretry/model/compilerchanges. No newquality/repair/freshindependent500claim.", "preserved_source_cached_ranking": { "scope": "Fixed-query actual-source union opportunity only, using shared historical real observations. No new agent behavior, paid calls or independent/leaderboard claim. No retrospective change to earlier failed online or independent gates.", "protocol_sha256": "707b197e50de1591e9ca6abfbb0213b7682440d74043e51ab944161f6e5a1428", "source_replay_sha256": "fb43453185c9e7e526594c978d8722c509420fa3c928ac48cad3318cb7df5a53", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "original": { "recall": 0.5020746887966805, "precision": 0.7423312883435583, "f1": 0.5990099009900989, "source_lines": 163, "intersection": 121 }, "unranked": { "recall": 0.6016597510373444, "precision": 0.6712962962962963, "f1": 0.6345733041575492, "source_lines": 216, "intersection": 145 }, "jev": { "recall": 0.5684647302904564, "precision": 0.6715686274509803, "f1": 0.6157303370786517, "source_lines": 204, "intersection": 137 } }, "source_selection_differs": true, "views": 2, "extra_reply_tokens": { "unranked": 2411, "jev": 2410 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "original": { "recall": 0.25510204081632654, "precision": 0.07363770250368189, "f1": 0.11428571428571428, "source_lines": 679, "intersection": 50 }, "unranked": { "recall": 0.3877551020408163, "precision": 0.06501283147989735, "f1": 0.11135531135531136, "source_lines": 1169, "intersection": 76 }, "jev": { "recall": 0.32142857142857145, "precision": 0.05977229601518026, "f1": 0.10080000000000001, "source_lines": 1054, "intersection": 63 } }, "source_selection_differs": true, "views": 6, "extra_reply_tokens": { "unranked": 9652, "jev": 9650 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "original": { "recall": 0.2914536157779401, "precision": 0.24767225325884543, "f1": 0.2677852348993289, "source_lines": 1611, "intersection": 399 }, "unranked": { "recall": 0.2914536157779401, "precision": 0.19491939423546653, "f1": 0.2336065573770492, "source_lines": 2047, "intersection": 399 }, "jev": { "recall": 0.38641344046749454, "precision": 0.2562984496124031, "f1": 0.3081852607049228, "source_lines": 2064, "intersection": 529 } }, "source_selection_differs": true, "views": 7, "extra_reply_tokens": { "unranked": 10697, "jev": 10516 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "original": { "recall": 0.5362694300518135, "precision": 0.5798319327731093, "f1": 0.5572005383580081, "source_lines": 357, "intersection": 207 }, "unranked": { "recall": 0.5362694300518135, "precision": 0.5686813186813187, "f1": 0.552, "source_lines": 364, "intersection": 207 }, "jev": { "recall": 0.5362694300518135, "precision": 0.48936170212765956, "f1": 0.511742892459827, "source_lines": 423, "intersection": 207 } }, "source_selection_differs": true, "views": 3, "extra_reply_tokens": { "unranked": 2902, "jev": 2896 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "original": { "recall": 0.41458333333333336, "precision": 0.7007042253521126, "f1": 0.5209424083769634, "source_lines": 284, "intersection": 199 }, "unranked": { "recall": 0.4166666666666667, "precision": 0.3284072249589491, "f1": 0.36730945821854916, "source_lines": 609, "intersection": 200 }, "jev": { "recall": 0.41458333333333336, "precision": 0.30709876543209874, "f1": 0.3528368794326241, "source_lines": 648, "intersection": 199 } }, "source_selection_differs": true, "views": 5, "extra_reply_tokens": { "unranked": 7385, "jev": 7381 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "original": { "recall": 0.146779303062302, "precision": 0.2202852614896989, "f1": 0.17617237008871992, "source_lines": 631, "intersection": 139 }, "unranked": { "recall": 0.17106652587117213, "precision": 0.23142857142857143, "f1": 0.19672131147540986, "source_lines": 700, "intersection": 162 }, "jev": { "recall": 0.17423442449841606, "precision": 0.23076923076923078, "f1": 0.19855595667870038, "source_lines": 715, "intersection": 165 } }, "source_selection_differs": true, "views": 4, "extra_reply_tokens": { "unranked": 4698, "jev": 4681 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "original": { "recall": 0.35456110154905335, "precision": 0.1794425087108014, "f1": 0.23828802776171196, "source_lines": 1148, "intersection": 206 }, "unranked": { "recall": 0.3648881239242685, "precision": 0.1444141689373297, "f1": 0.20693020985846755, "source_lines": 1468, "intersection": 212 }, "jev": { "recall": 0.37521514629948366, "precision": 0.15843023255813954, "f1": 0.22278998467041392, "source_lines": 1376, "intersection": 218 } }, "source_selection_differs": true, "views": 6, "extra_reply_tokens": { "unranked": 6135, "jev": 6113 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "original": { "recall": 0.5975975975975976, "precision": 0.16597164303586323, "f1": 0.2597911227154047, "source_lines": 1199, "intersection": 199 }, "unranked": { "recall": 0.5975975975975976, "precision": 0.14014084507042254, "f1": 0.22703936109526524, "source_lines": 1420, "intersection": 199 }, "jev": { "recall": 0.5975975975975976, "precision": 0.14917541229385306, "f1": 0.23875224955008995, "source_lines": 1334, "intersection": 199 } }, "source_selection_differs": true, "views": 8, "extra_reply_tokens": { "unranked": 10887, "jev": 10881 } } ], "macro": { "original": { "recall": 0.3873026388731309, "precision": 0.3637346019334589, "f1": 0.34168441468449373 }, "unranked": { "recall": 0.4209196016209524, "precision": 0.2930375813860314, "f1": 0.31619193919220023 }, "jev": { "recall": 0.4217758342458958, "precision": 0.2903093395324432, "f1": 0.31867419507190375 } }, "jev_vs_unranked": { "recall": { "mean_difference": 0.0008562326249434218, "exploratory_95_interval": [ -0.025719346922912074, 0.03239067487906382 ] }, "precision": { "mean_difference": -0.0027282418535882723, "exploratory_95_interval": [ -0.030029389249312685, 0.02187599655832033 ] }, "f1": { "mean_difference": 0.002482255879703521, "exploratory_95_interval": [ -0.017445174780243572, 0.026322060125732887 ] } }, "gates": { "all_original_source_preserved": true, "all_source_and_cached_score_order_receipts_valid": true, "eight_tasks_retained": true, "changed_selection_in_three_tasks": true, "jev_recall_higher_than_unranked": true }, "development_replay_gate_passed": true, "cached_decision_requests": 75, "cached_historical_api_cost_usd": 0.009593262000000002, "new_api_cost_usd": 0, "new_model_calls": 0, "independent_validation_tasks_used": 0 }, "novel_source_ranking_development": { "scope": "Paid auxiliary scoring, fixed historical source delivery replay only. No new main-agent sessions, repair calls, independent tasks or leaderboard claim. Selection between two prespecified scores is development tuning only.", "protocol_sha256": "2b059f9cf610dbf20848e2e9d66dc4e2ddcab3e9efe1b8c69a8a7d4c1c13d350", "source_replay_sha256": "6013998a7cf35b21e3477ce2e05f8594051c027b48aceb3f95f7dbea764ebfda", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "original": { "recall": 0.5020746887966805, "precision": 0.7423312883435583, "f1": 0.5990099009900989, "source_lines": 163 }, "unranked": { "recall": 0.6016597510373444, "precision": 0.6712962962962963, "f1": 0.6345733041575492, "source_lines": 216 }, "cached-whole": { "recall": 0.5684647302904564, "precision": 0.6715686274509803, "f1": 0.6157303370786517, "source_lines": 204 }, "novel-relevance": { "recall": 0.6016597510373444, "precision": 0.7142857142857143, "f1": 0.6531531531531531, "source_lines": 203 }, "novel-information": { "recall": 0.5684647302904564, "precision": 0.6748768472906403, "f1": 0.6171171171171171, "source_lines": 203 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "original": { "recall": 0.25510204081632654, "precision": 0.07363770250368189, "f1": 0.11428571428571428, "source_lines": 679 }, "unranked": { "recall": 0.3877551020408163, "precision": 0.06501283147989735, "f1": 0.11135531135531136, "source_lines": 1169 }, "cached-whole": { "recall": 0.32142857142857145, "precision": 0.05977229601518026, "f1": 0.10080000000000001, "source_lines": 1054 }, "novel-relevance": { "recall": 0.3877551020408163, "precision": 0.07011070110701106, "f1": 0.11875, "source_lines": 1084 }, "novel-information": { "recall": 0.3877551020408163, "precision": 0.0695970695970696, "f1": 0.11801242236024846, "source_lines": 1092 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "original": { "recall": 0.2914536157779401, "precision": 0.24767225325884543, "f1": 0.2677852348993289, "source_lines": 1611 }, "unranked": { "recall": 0.2914536157779401, "precision": 0.19491939423546653, "f1": 0.2336065573770492, "source_lines": 2047 }, "cached-whole": { "recall": 0.38641344046749454, "precision": 0.2562984496124031, "f1": 0.3081852607049228, "source_lines": 2064 }, "novel-relevance": { "recall": 0.33674214755295834, "precision": 0.24275934702474986, "f1": 0.2821297429620563, "source_lines": 1899 }, "novel-information": { "recall": 0.33966398831263694, "precision": 0.23088381330685204, "f1": 0.27490393142181496, "source_lines": 2014 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "original": { "recall": 0.5362694300518135, "precision": 0.5798319327731093, "f1": 0.5572005383580081, "source_lines": 357 }, "unranked": { "recall": 0.5362694300518135, "precision": 0.5686813186813187, "f1": 0.552, "source_lines": 364 }, "cached-whole": { "recall": 0.5362694300518135, "precision": 0.48936170212765956, "f1": 0.511742892459827, "source_lines": 423 }, "novel-relevance": { "recall": 0.5362694300518135, "precision": 0.48936170212765956, "f1": 0.511742892459827, "source_lines": 423 }, "novel-information": { "recall": 0.5362694300518135, "precision": 0.48936170212765956, "f1": 0.511742892459827, "source_lines": 423 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "original": { "recall": 0.41458333333333336, "precision": 0.7007042253521126, "f1": 0.5209424083769634, "source_lines": 284 }, "unranked": { "recall": 0.4166666666666667, "precision": 0.3284072249589491, "f1": 0.36730945821854916, "source_lines": 609 }, "cached-whole": { "recall": 0.41458333333333336, "precision": 0.30709876543209874, "f1": 0.3528368794326241, "source_lines": 648 }, "novel-relevance": { "recall": 0.41458333333333336, "precision": 0.3085271317829457, "f1": 0.35377777777777775, "source_lines": 645 }, "novel-information": { "recall": 0.41458333333333336, "precision": 0.3085271317829457, "f1": 0.35377777777777775, "source_lines": 645 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "original": { "recall": 0.146779303062302, "precision": 0.2202852614896989, "f1": 0.17617237008871992, "source_lines": 631 }, "unranked": { "recall": 0.17106652587117213, "precision": 0.23142857142857143, "f1": 0.19672131147540986, "source_lines": 700 }, "cached-whole": { "recall": 0.17423442449841606, "precision": 0.23076923076923078, "f1": 0.19855595667870038, "source_lines": 715 }, "novel-relevance": { "recall": 0.146779303062302, "precision": 0.20174165457184326, "f1": 0.16992665036674814, "source_lines": 689 }, "novel-information": { "recall": 0.146779303062302, "precision": 0.21786833855799373, "f1": 0.1753943217665615, "source_lines": 638 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "original": { "recall": 0.35456110154905335, "precision": 0.1794425087108014, "f1": 0.23828802776171196, "source_lines": 1148 }, "unranked": { "recall": 0.3648881239242685, "precision": 0.1444141689373297, "f1": 0.20693020985846755, "source_lines": 1468 }, "cached-whole": { "recall": 0.37521514629948366, "precision": 0.15843023255813954, "f1": 0.22278998467041392, "source_lines": 1376 }, "novel-relevance": { "recall": 0.3648881239242685, "precision": 0.14929577464788732, "f1": 0.21189405297351324, "source_lines": 1420 }, "novel-information": { "recall": 0.3648881239242685, "precision": 0.14929577464788732, "f1": 0.21189405297351324, "source_lines": 1420 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "original": { "recall": 0.5975975975975976, "precision": 0.16597164303586323, "f1": 0.2597911227154047, "source_lines": 1199 }, "unranked": { "recall": 0.5975975975975976, "precision": 0.14014084507042254, "f1": 0.22703936109526524, "source_lines": 1420 }, "cached-whole": { "recall": 0.5975975975975976, "precision": 0.14917541229385306, "f1": 0.23875224955008995, "source_lines": 1334 }, "novel-relevance": { "recall": 0.5975975975975976, "precision": 0.15438324282389448, "f1": 0.24537607891491983, "source_lines": 1289 }, "novel-information": { "recall": 0.5975975975975976, "precision": 0.1479553903345725, "f1": 0.23718712753277713, "source_lines": 1345 } } } ], "macro": { "original": { "recall": 0.3873026388731309, "precision": 0.3637346019334589, "f1": 0.34168441468449373 }, "unranked": { "recall": 0.4209196016209524, "precision": 0.2930375813860314, "f1": 0.31619193919220023 }, "cached-whole": { "recall": 0.4217758342458958, "precision": 0.2903093395324432, "f1": 0.31867419507190375 }, "novel-relevance": { "recall": 0.42328434857505426, "precision": 0.2913081585464632, "f1": 0.3183437935759994 }, "novel-information": { "recall": 0.4195002010766531, "precision": 0.28604575845570257, "f1": 0.31250370542620465 } }, "selected_mode": "novel-relevance", "selected_mode_vs_baselines": { "unranked": { "recall": { "mean_difference": 0.002364746954101847, "exploratory_95_interval": [ -0.0091077085533263, 0.016722782748965173 ] }, "precision": { "mean_difference": -0.0017294228395682513, "exploratory_95_interval": [ -0.030153914873103087, 0.024354547202724126 ] }, "f1": { "mean_difference": 0.002151854383799218, "exploratory_95_interval": [ -0.016304591107241473, 0.021233145231957087 ] } }, "cached-whole": { "recall": { "mean_difference": 0.0015085143291584252, "exploratory_95_interval": [ -0.01991761263985297, 0.02516021176252676 ] }, "precision": { "mean_difference": 0.000998819014020021, "exploratory_95_interval": [ -0.011358212095568001, 0.015400175406787099 ] }, "f1": { "mean_difference": -0.000330401495904303, "exploratory_95_interval": [ -0.013934240849730151, 0.014201229494251861 ] } }, "novel-relevance": { "recall": { "mean_difference": 0.0, "exploratory_95_interval": [ 0.0, 0.0 ] }, "precision": { "mean_difference": 0.0, "exploratory_95_interval": [ 0.0, 0.0 ] }, "f1": { "mean_difference": 0.0, "exploratory_95_interval": [ 0.0, 0.0 ] } } }, "gates": { "all_eight_development_tasks_retained": true, "all_source_scores_and_receipts_verified": true, "usage_known": true, "exact_jev_model": true, "selected_recall_gain_at_least_one_point": false, "spend_within_reservation": true }, "development_screen_passed": false, "new_decision_requests": 102, "new_jev_api_cost_usd": 0.020709527999999998, "unknown_usage_calls": 0, "main_agent_sessions": 0, "repair_calls": 0, "independent_gain_established": false }, "first_read_source_window_replay": { "scope": "First-read same-file source-window opportunity only, not new agent behavior, paid inference, independent quality or leaderboard results. Raw queries/history fixed and both validation cohorts unused.", "protocol_sha256": "a57a2f0857692ce6a80971e28c4a328987ace03819dc823ebfbf134a7ccba4cb", "source_replay_sha256": "fada8ad540f72676a0cbaaa4056fda4ae8f2e1656ca91b88916e5e270b71a162", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "original": { "recall": 0.4896265560165975, "precision": 0.5244444444444445, "f1": 0.5064377682403433, "source_lines": 225 }, "full-window-pool": { "recall": 0.8091286307053942, "precision": 0.08764044943820225, "f1": 0.15815085158150852, "source_lines": 2225 }, "near-greedy": { "recall": 0.4896265560165975, "precision": 0.30569948186528495, "f1": 0.3763955342902711, "source_lines": 386 }, "near-spread": { "recall": 0.4896265560165975, "precision": 0.3072916666666667, "f1": 0.3776, "source_lines": 384 } }, "eligible_files": 1, "rendered_extra_tokens": { "near-greedy": 2045, "near-spread": 1996 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "original": { "recall": 0.30612244897959184, "precision": 0.12195121951219512, "f1": 0.17441860465116277, "source_lines": 492 }, "full-window-pool": { "recall": 0.5153061224489796, "precision": 0.02750544662309368, "f1": 0.052223371251292644, "source_lines": 3672 }, "near-greedy": { "recall": 0.30612244897959184, "precision": 0.06048387096774194, "f1": 0.10101010101010101, "source_lines": 992 }, "near-spread": { "recall": 0.30612244897959184, "precision": 0.0625, "f1": 0.10380622837370242, "source_lines": 960 } }, "eligible_files": 3, "rendered_extra_tokens": { "near-greedy": 6140, "near-spread": 5407 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "original": { "recall": 0.5091307523739956, "precision": 0.5349194167306216, "f1": 0.5217065868263472, "source_lines": 1303 }, "full-window-pool": { "recall": 0.997808619430241, "precision": 0.2579792256846081, "f1": 0.4099639855942377, "source_lines": 5295 }, "near-greedy": { "recall": 0.8100803506208911, "precision": 0.4219939117199391, "f1": 0.5549161871403552, "source_lines": 2628 }, "near-spread": { "recall": 0.7888970051132214, "precision": 0.425531914893617, "f1": 0.5528538520604044, "source_lines": 2538 } }, "eligible_files": 8, "rendered_extra_tokens": { "near-greedy": 14818, "near-spread": 13800 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "original": { "recall": 0.5362694300518135, "precision": 0.5149253731343284, "f1": 0.5253807106598986, "source_lines": 402 }, "full-window-pool": { "recall": 1.0, "precision": 0.5475177304964539, "f1": 0.7076076993583867, "source_lines": 705 }, "near-greedy": { "recall": 0.7046632124352331, "precision": 0.512241054613936, "f1": 0.5932388222464559, "source_lines": 531 }, "near-spread": { "recall": 0.6062176165803109, "precision": 0.4491362763915547, "f1": 0.515986769570011, "source_lines": 521 } }, "eligible_files": 2, "rendered_extra_tokens": { "near-greedy": 2259, "near-spread": 1798 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "original": { "recall": 0.45416666666666666, "precision": 0.8074074074074075, "f1": 0.5813333333333333, "source_lines": 270 }, "full-window-pool": { "recall": 0.6625, "precision": 0.21662125340599456, "f1": 0.32648870636550303, "source_lines": 1468 }, "near-greedy": { "recall": 0.6625, "precision": 0.37902264600715135, "f1": 0.48218347232752073, "source_lines": 839 }, "near-spread": { "recall": 0.63125, "precision": 0.3935064935064935, "f1": 0.4848, "source_lines": 770 } }, "eligible_files": 4, "rendered_extra_tokens": { "near-greedy": 7832, "near-spread": 6885 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "original": { "recall": 0.146779303062302, "precision": 0.25272727272727274, "f1": 0.18570474281897129, "source_lines": 550 }, "full-window-pool": { "recall": 0.6610348468848997, "precision": 0.24811732065001982, "f1": 0.36080691642651297, "source_lines": 2523 }, "near-greedy": { "recall": 0.4118268215417107, "precision": 0.35071942446043164, "f1": 0.3788246721709568, "source_lines": 1112 }, "near-spread": { "recall": 0.36430834213305174, "precision": 0.345, "f1": 0.3543913713405239, "source_lines": 1000 } }, "eligible_files": 5, "rendered_extra_tokens": { "near-greedy": 6844, "near-spread": 5571 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "original": { "recall": 0.35456110154905335, "precision": 0.18933823529411764, "f1": 0.24685440383463148, "source_lines": 1088 }, "full-window-pool": { "recall": 0.35456110154905335, "precision": 0.07142857142857142, "f1": 0.11890331890331889, "source_lines": 2884 }, "near-greedy": { "recall": 0.35456110154905335, "precision": 0.10629514963880289, "f1": 0.16355696705041683, "source_lines": 1938 }, "near-spread": { "recall": 0.35456110154905335, "precision": 0.11547085201793722, "f1": 0.1742071881606765, "source_lines": 1784 } }, "eligible_files": 8, "rendered_extra_tokens": { "near-greedy": 11483, "near-spread": 9331 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "original": { "recall": 0.6666666666666666, "precision": 0.13503649635036497, "f1": 0.22458270106221548, "source_lines": 1644 }, "full-window-pool": { "recall": 0.8798798798798799, "precision": 0.020149920913279693, "f1": 0.03939760656178567, "source_lines": 14541 }, "near-greedy": { "recall": 0.7537537537537538, "precision": 0.08120349401488192, "f1": 0.1466121495327103, "source_lines": 3091 }, "near-spread": { "recall": 0.7057057057057057, "precision": 0.08265916285613788, "f1": 0.14798488664987405, "source_lines": 2843 } }, "eligible_files": 10, "rendered_extra_tokens": { "near-greedy": 19756, "near-spread": 16849 } } ], "macro": { "original": { "recall": 0.4329153656708359, "precision": 0.38509373320009405, "f1": 0.3708023564283629 }, "full-window-pool": { "recall": 0.7350274001123058, "precision": 0.18461998983002792, "f1": 0.2716928070053183 }, "near-greedy": { "recall": 0.5616417806121039, "precision": 0.27720737916102123, "f1": 0.3495922382210985 }, "near-spread": { "recall": 0.5308360970096915, "precision": 0.2726370457915509, "f1": 0.33895378701939904 } }, "pool_recall_headroom": 0.30211203444146995, "gates": { "all_source_originals_preserved": true, "all_eight_cases_retained": true, "eligible_source_in_six_tasks": true, "pool_headroom_at_least_ten_points": true }, "source_pool_opportunity_gate_passed": true, "new_model_calls": 0, "new_api_cost_usd": 0, "runtime_parity": { "runtime_reply_parity": "exact", "files": 46, "tasks": 8, "new_model_calls": 0, "new_api_cost_usd": 0 }, "requested_range_diagnostic": { "scope": "Developer first-read source-only diagnostic; no gold or new calls. Requested-range membership uses native defaults offset1/limit200. Outside-range additions distinguish this policy from raising a native read output-character cap alone.", "views": 46, "extra_line_deliveries_inside_original_requested_range": 164, "extra_line_deliveries_outside_original_requested_range": 5683, "new_model_calls": 0, "new_api_cost_usd": 0 } }, "first_read_source_window_jev_probe": { "scope": "Developer fixed-query first-read file-context opportunity, not new agent behavior or independent/leaderboard performance. Both prior32 cohorts unused. Same physical pool/budget/context, Jev ordering alone versus nearby order.", "protocol_sha256": "d2ad25a5dae22ba4fbb48f2f54e259b7126375bdcc34119ac76b8132c7dcab18", "source_replay_sha256": "eebdb9de5601546fb6cbb1c86ac81d60b04abe4ff31ae6887bc422903837c579", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "original": { "recall": 0.4896265560165975, "precision": 0.5244444444444445, "f1": 0.5064377682403433, "source_lines": 225 }, "near": { "recall": 0.4896265560165975, "precision": 0.30569948186528495, "f1": 0.3763955342902711, "source_lines": 386 }, "jev": { "recall": 0.4979253112033195, "precision": 0.3614457831325301, "f1": 0.418848167539267, "source_lines": 332 } }, "augmented_files": 1 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "original": { "recall": 0.30612244897959184, "precision": 0.12195121951219512, "f1": 0.17441860465116277, "source_lines": 492 }, "near": { "recall": 0.30612244897959184, "precision": 0.06048387096774194, "f1": 0.10101010101010101, "source_lines": 992 }, "jev": { "recall": 0.3673469387755102, "precision": 0.07766990291262135, "f1": 0.12822796081923418, "source_lines": 927 } }, "augmented_files": 3 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "original": { "recall": 0.5091307523739956, "precision": 0.5349194167306216, "f1": 0.5217065868263472, "source_lines": 1303 }, "near": { "recall": 0.8100803506208911, "precision": 0.4219939117199391, "f1": 0.5549161871403552, "source_lines": 2628 }, "jev": { "recall": 0.8392987582176772, "precision": 0.43555724033358606, "f1": 0.5734963813326678, "source_lines": 2638 } }, "augmented_files": 8 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "original": { "recall": 0.5362694300518135, "precision": 0.5149253731343284, "f1": 0.5253807106598986, "source_lines": 402 }, "near": { "recall": 0.7046632124352331, "precision": 0.512241054613936, "f1": 0.5932388222464559, "source_lines": 531 }, "jev": { "recall": 0.655440414507772, "precision": 0.44778761061946903, "f1": 0.5320715036803365, "source_lines": 565 } }, "augmented_files": 2 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "original": { "recall": 0.45416666666666666, "precision": 0.8074074074074075, "f1": 0.5813333333333333, "source_lines": 270 }, "near": { "recall": 0.6625, "precision": 0.37902264600715135, "f1": 0.48218347232752073, "source_lines": 839 }, "jev": { "recall": 0.6625, "precision": 0.38545454545454544, "f1": 0.48735632183908045, "source_lines": 825 } }, "augmented_files": 4 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "original": { "recall": 0.146779303062302, "precision": 0.25272727272727274, "f1": 0.18570474281897129, "source_lines": 550 }, "near": { "recall": 0.4118268215417107, "precision": 0.35071942446043164, "f1": 0.3788246721709568, "source_lines": 1112 }, "jev": { "recall": 0.3611404435058078, "precision": 0.3048128342245989, "f1": 0.33059449009183184, "source_lines": 1122 } }, "augmented_files": 5 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "original": { "recall": 0.35456110154905335, "precision": 0.18933823529411764, "f1": 0.24685440383463148, "source_lines": 1088 }, "near": { "recall": 0.35456110154905335, "precision": 0.10629514963880289, "f1": 0.16355696705041683, "source_lines": 1938 }, "jev": { "recall": 0.35456110154905335, "precision": 0.10499490316004077, "f1": 0.16201337003539126, "source_lines": 1962 } }, "augmented_files": 8 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "original": { "recall": 0.6666666666666666, "precision": 0.13503649635036497, "f1": 0.22458270106221548, "source_lines": 1644 }, "near": { "recall": 0.7537537537537538, "precision": 0.08120349401488192, "f1": 0.1466121495327103, "source_lines": 3091 }, "jev": { "recall": 0.8078078078078078, "precision": 0.08942819148936171, "f1": 0.1610296318467525, "source_lines": 3008 } }, "augmented_files": 10 } ], "macro": { "original": { "recall": 0.4329153656708359, "precision": 0.38509373320009405, "f1": 0.3708023564283629 }, "near": { "recall": 0.5616417806121039, "precision": 0.27720737916102123, "f1": 0.3495922382210985 }, "jev": { "recall": 0.5682525969458685, "precision": 0.2758938764158442, "f1": 0.3492047283980702 } }, "jev_vs_baselines": { "original": { "recall": { "mean_difference": 0.1353372312750326, "exploratory_95_interval": [ 0.06623364513687618, 0.2109846967354788 ] }, "precision": { "mean_difference": -0.10919985678424987, "exploratory_95_interval": [ -0.2085509556501544, -0.03224833002452308 ] }, "f1": { "mean_difference": -0.021597628030292713, "exploratory_95_interval": [ -0.07059662707922292, 0.0371295199625647 ] } }, "near": { "recall": { "mean_difference": 0.00661081633376457, "exploratory_95_interval": [ -0.020472284715342844, 0.03350928131043162 ] }, "precision": { "mean_difference": -0.0013135027451770462, "exploratory_95_interval": [ -0.02608299556203121, 0.022566432099232536 ] }, "f1": { "mean_difference": -0.00038750982302828037, "exploratory_95_interval": [ -0.024887429560438577, 0.022276175988397776 ] } } }, "gates": { "all8_cases_retained": true, "all_source_model_and_score_receipts_verified": true, "known_usage": true, "jev_recall_gain_over_near_at_least_one_point": false, "spend_within_reservation": true }, "development_ranking_gate_passed": false, "new_jev_api_cost_usd": 0.024354119999999993, "new_decision_requests": 71, "main_agent_sessions": 0, "repair_calls": 0, "independent_gain_established": false }, "first_read_source_window_wider_replay": { "scope": "All24 disclosed earlier native development trajectories, fixed-query opportunity replay only. No new agent behavior, inference, repair, independent results or leaderboard claim.", "protocol_sha256": "cd5d688c0e17fa443f85dd0f100f761814f72129618ff4e4dbd11ebe3e6ca47e", "source_replay_sha256": "3f38a7a7f2444fb90c5a4cc09b2ca236f495719ff7ebf8b8216241a53ba158c2", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "original": { "recall": 0.3651452282157676, "precision": 0.704, "f1": 0.48087431693989063 }, "window": { "recall": 0.3651452282157676, "precision": 0.3087719298245614, "f1": 0.33460076045627374 }, "files_with_added_source": 1 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "original": { "recall": 0.40816326530612246, "precision": 0.10854816824966079, "f1": 0.17148981779206862 }, "window": { "recall": 0.40816326530612246, "precision": 0.056100981767180924, "f1": 0.09864364981504316 }, "files_with_added_source": 4 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "original": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "window": { "recall": 1.0, "precision": 0.042263610315186245, "f1": 0.0810996563573883 }, "files_with_added_source": 3 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "original": { "recall": 1.0, "precision": 0.3169642857142857, "f1": 0.48135593220338985 }, "window": { "recall": 1.0, "precision": 0.1710843373493976, "f1": 0.29218106995884774 }, "files_with_added_source": 1 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "original": { "recall": 0.36303871439006574, "precision": 0.3066008636644047, "f1": 0.3324414715719064 }, "window": { "recall": 0.6486486486486487, "precision": 0.30484037075180226, "f1": 0.41475945819710414 }, "files_with_added_source": 9 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "original": { "recall": 0.8706140350877193, "precision": 0.5251322751322751, "f1": 0.6551155115511551 }, "window": { "recall": 0.8706140350877193, "precision": 0.3861867704280156, "f1": 0.5350404312668463 }, "files_with_added_source": 3 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "original": { "recall": 0.5362694300518135, "precision": 0.5036496350364964, "f1": 0.519447929736512 }, "window": { "recall": 0.7046632124352331, "precision": 0.39080459770114945, "f1": 0.5027726432532347 }, "files_with_added_source": 3 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "original": { "recall": 0.425, "precision": 0.8326530612244898, "f1": 0.5627586206896551 }, "window": { "recall": 0.6625, "precision": 0.38686131386861317, "f1": 0.4884792626728111 }, "files_with_added_source": 4 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "original": { "recall": 0.14889123548046462, "precision": 0.2256, "f1": 0.17938931297709926 }, "window": { "recall": 0.4118268215417107, "precision": 0.3382480485689506, "f1": 0.37142857142857144 }, "files_with_added_source": 4 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "original": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 }, "window": { "recall": 1.0, "precision": 0.027510316368638238, "f1": 0.05354752342704149 }, "files_with_added_source": 2 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "original": { "recall": 0.29678638941398866, "precision": 0.48606811145510836, "f1": 0.3685446009389672 }, "window": { "recall": 0.5973534971644613, "precision": 0.39012345679012345, "f1": 0.4719940253920837 }, "files_with_added_source": 3 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "original": { "recall": 0.5, "precision": 0.23148148148148148, "f1": 0.3164556962025316 }, "window": { "recall": 0.5, "precision": 0.09652509652509653, "f1": 0.16181229773462782 }, "files_with_added_source": 2 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "original": { "recall": 0.2047244094488189, "precision": 0.19030192131747484, "f1": 0.19724988146040778 }, "window": { "recall": 0.39665354330708663, "precision": 0.20119820269595606, "f1": 0.26697581980788343 }, "files_with_added_source": 7 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "original": { "recall": 0.3528399311531842, "precision": 0.2074898785425101, "f1": 0.26131293817718293 }, "window": { "recall": 0.3528399311531842, "precision": 0.10939167556029883, "f1": 0.16700610997963342 }, "files_with_added_source": 9 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "original": { "recall": 0.5795795795795796, "precision": 0.1643952299829642, "f1": 0.25613802256138024 }, "window": { "recall": 0.6756756756756757, "precision": 0.0959079283887468, "f1": 0.16797312430011196 }, "files_with_added_source": 8 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "original": { "recall": 0.5652173913043478, "precision": 0.03117505995203837, "f1": 0.05909090909090909 }, "window": { "recall": 0.5652173913043478, "precision": 0.01478953356086462, "f1": 0.028824833702882482 }, "files_with_added_source": 3 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "original": { "recall": 0.8996282527881041, "precision": 0.36890243902439024, "f1": 0.5232432432432432 }, "window": { "recall": 0.8996282527881041, "precision": 0.22916666666666666, "f1": 0.36528301886792447 }, "files_with_added_source": 4 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "original": { "recall": 0.7195767195767195, "precision": 0.1348314606741573, "f1": 0.22710826607291956 }, "window": { "recall": 0.7389770723104057, "precision": 0.08834071262913767, "f1": 0.15781544256120528 }, "files_with_added_source": 11 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "original": { "recall": 0.46819787985865724, "precision": 0.828125, "f1": 0.5981941309255079 }, "window": { "recall": 0.6183745583038869, "precision": 0.5131964809384164, "f1": 0.5608974358974359 }, "files_with_added_source": 3 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "original": { "recall": 1.0, "precision": 0.04105571847507331, "f1": 0.07887323943661972 }, "window": { "recall": 1.0, "precision": 0.026923076923076925, "f1": 0.05243445692883896 }, "files_with_added_source": 1 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "original": { "recall": 0.0, "precision": 0.0, "f1": 0 }, "window": { "recall": 0.0, "precision": 0.0, "f1": 0 }, "files_with_added_source": 2 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "original": { "recall": 0.2745664739884393, "precision": 0.1317614424410541, "f1": 0.17806935332708526 }, "window": { "recall": 0.5520231213872833, "precision": 0.21364653243847875, "f1": 0.3080645161290323 }, "files_with_added_source": 1 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "original": { "recall": 0.23621323529411764, "precision": 0.7427745664739884, "f1": 0.3584379358437936 }, "window": { "recall": 0.39889705882352944, "precision": 0.6770670826833073, "f1": 0.5020242914979757 }, "files_with_added_source": 3 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "original": { "recall": 0.5901639344262295, "precision": 0.4044943820224719, "f1": 0.48 }, "window": { "recall": 0.5901639344262295, "precision": 0.1489655172413793, "f1": 0.23788546255506607 }, "files_with_added_source": 3 } ], "macro": { "original": { "recall": 0.5335256710568391, "precision": 0.31624050372742957, "f1": 0.3117640515791511 }, "window": { "recall": 0.6232235519949749, "precision": 0.21741309333271022, "f1": 0.2758976609244943 } }, "recall_difference": 0.0896978809381358, "gates": { "all24_retained": true, "exact_source_and_original_preservation": true, "recall_opportunity_five_points": true, "added_source_in18_tasks": true }, "opportunity_gate_passed": true, "new_model_calls": 0, "new_api_cost_usd": 0, "analysis_correction": { "scope": "Evaluation-only bridge: historical24 reports predate canonical gold_line_spans export. Preserve original failed driver and completed source choices; use pinned official Gold.line_spans_init on already disclosed evaluation input. No policy/gate/task change and no model calls.", "original_driver_sha256": "ec0b9a63020c1e6644ea51f7ebe673b3bc552b58e2ecfefc18d395752b8af90c", "frozen_source_replay_sha256": "3f38a7a7f2444fb90c5a4cc09b2ca236f495719ff7ebf8b8216241a53ba158c2", "drivers": { "canonical_gold.py": "da9db85694d46db23623734181564e10d0e72b592612d46f8479f7612d19a387", "finish_analysis.py": "b20efdf4555110ad922894b6492729ebc4426c68995126d4b80e85c33c39fec9" }, "preflight_correction": "Use prediction.instance_id from the historical evaluation input schema; the zero-call KeyError and original bridge logs are retained. Source choices, selected tasks and gates are unchanged." } }, "first_read_source_window_online_development": { "scope": "Pure ContextBench retrieval-agent development pilot. No repairs, mini-swe-agent or native tests. Cumulative verified source is primary; final served context is secondary. No independent or leaderboard claim.", "protocol_sha256": "90acdd335dc7215c8909f029a5a5fe7f24cf4f1a05e779c864191c5a431e5b99", "runtime_commit": "4a6992cb13977f682d1e8317a41f5df677720aba", "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.0882124, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 222, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.38589211618257263, "precision": 0.4189189189189189, "f1": 0.4017278617710583 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.1886935, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 568, "tool_calls": { "grep": 6, "glob": 1, "read": 8 }, "sdk_turns": 16, "metrics": { "recall": 0.2653061224489796, "precision": 0.09154929577464789, "f1": 0.13612565445026178 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.38395170000000006, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1785, "tool_calls": { "grep": 8, "read": 12, "glob": 2 }, "sdk_turns": 23, "metrics": { "recall": 0.5273922571219869, "precision": 0.40448179271708684, "f1": 0.4578313253012048 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.11077899999999999, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 367, "tool_calls": { "grep": 4, "read": 2 }, "sdk_turns": 7, "metrics": { "recall": 0.5362694300518135, "precision": 0.5640326975476839, "f1": 0.5498007968127491 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.1094498, "read_views": 3, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 161, "extra_reply_tokens": 2045, "unconfirmed_tool_replies_excluded": 0, "source_lines": 373, "tool_calls": { "grep": 2, "read": 5, "glob": 1 }, "sdk_turns": 9, "metrics": { "recall": 0.4896265560165975, "precision": 0.3163538873994638, "f1": 0.3843648208469055 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.2364218, "read_views": 5, "confirmed_views_with_added_source": 3, "appended_line_deliveries": 536, "extra_reply_tokens": 6138, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1289, "tool_calls": { "grep": 5, "read": 7, "glob": 1 }, "sdk_turns": 14, "metrics": { "recall": 0.27040816326530615, "precision": 0.041117145073700546, "f1": 0.07138047138047138 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.39622889999999994, "read_views": 9, "confirmed_views_with_added_source": 7, "appended_line_deliveries": 1171, "extra_reply_tokens": 13395, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2391, "tool_calls": { "grep": 5, "read": 10, "glob": 1 }, "sdk_turns": 17, "metrics": { "recall": 0.5076698319941563, "precision": 0.29067335842743625, "f1": 0.3696808510638298 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.129472, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 139, "extra_reply_tokens": 2429, "unconfirmed_tool_replies_excluded": 0, "source_lines": 531, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.7046632124352331, "precision": 0.512241054613936, "f1": 0.5932388222464559 } } ], "primary": { "native": { "recall": 0.42871498145133813, "precision": 0.3697456762395844, "f1": 0.3863714095838185, "official_valid": 4 }, "file-source": { "recall": 0.49309194092782327, "precision": 0.29009636137863415, "f1": 0.3546662413844156, "official_valid": 4 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.41775807860254344, "precision": 0.38551813851283556, "f1": 0.39077895272510194, "official_valid": 4 }, "served_32768": { "recall": 0.42871498145133813, "precision": 0.37022799199187983, "f1": 0.38663509246571426, "official_valid": 4 } }, "file-source": { "served_16384": { "recall": 0.49309194092782327, "precision": 0.3397010689216695, "f1": 0.38679799994462305, "official_valid": 4 }, "served_32768": { "recall": 0.49309194092782327, "precision": 0.29018765326256235, "f1": 0.35474003979759045, "official_valid": 4 } } }, "paired_differences": { "recall": { "mean_difference": 0.0643769594764851, "exploratory_95_interval": [ -0.007310192155752043, 0.13606411110872224 ] }, "precision": { "mean_difference": -0.07964931486095025, "exploratory_95_interval": [ -0.10818673290455286, -0.05111189681734764 ] }, "f1": { "mean_difference": -0.031705168199402854, "exploratory_95_interval": [ -0.0764478286535827, 0.016392223307832486 ] } }, "gates": { "all_eight_sessions_complete": true, "known_usage": true, "one_actual_model_route": true, "confirmed_source_projection_recomputed": true, "added_source_in_three_tasks": true, "original_reads_preserved_and_source_receipts_exact": true, "greater_recall": true }, "development_gate_passed": true, "known_new_api_sdk_usd": 1.6432091, "new_sdk_sessions": 8, "repair_calls": 0, "all_frozen_hashes_match": true, "independent_gain_established": false }, "broader_validation_candidate_inventory": { "scope": "Metadata-only additional validation-candidate inventory, no new selection or inference. Additional default-config tasks are not the official verified500 population; require a separately frozen protocol, broader exposure audit and source/format readiness before use.", "full_dataset_sha256": "2f56535bdc73eb8a68bf4ebb49789d8e9cd4f219ea60df6290b85278aee61ca8", "verified_dataset_sha256": "e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791", "columns_read": [ "instance_id", "original_inst_id", "language", "repo", "repo_url", "base_commit" ], "full_rows": 1136, "verified_rows": 500, "additional_by_id_original_and_repo_commit": 636, "preview_quarantine_sha256": "86fbaf8df71d9518ad024f518309d2506801e61c9ee3e1898c5aea25702c2a09", "preview_ids_not_eligible": 10, "additional_after_preview_quarantine": 630, "additional_per_language_after_preview": { "python": 240, "go": 64, "javascript": 93, "typescript": 53, "c": 45, "cpp": 50, "java": 42, "rust": 43 }, "remaining_verified_after_preview": 329, "remaining_verified_per_language": { "python": 210, "javascript": 40, "typescript": 45, "c": 7, "go": 22, "rust": 5 }, "new_task_selection": false, "annotation_columns_loaded": false, "new_api_cost_usd": 0 }, "documentation_preview_quarantine": { "scope": "Prospective data quarantine only; completed validation/model inputs predate this documentation browse and are unchanged.", "date": "2026-10-17", "source": "https://huggingface.co/datasets/Contextbench/ContextBench", "reason": "Dataset card HTML automatically rendered gold_context and reference-patch previews. Exclude every shown task from future unseen validation; prefer raw README/schema for documentation. No preview values were sent to experiment models.", "instance_ids": [ "SWE-Bench-Verified__python__maintenance__bugfix__1d90db61", "SWE-Bench-Verified__python__maintenance__bugfix__3b78a79f", "SWE-Bench-Verified__python__maintenance__bugfix__6ed8d38b", "SWE-Bench-Verified__python__maintenance__bugfix__71f348da", "SWE-Bench-Verified__python__maintenance__bugfix__88e3486b", "SWE-Bench-Verified__python__maintenance__bugfix__8adb3e00", "SWE-Bench-Verified__python__maintenance__bugfix__958ed1fd", "SWE-Bench-Verified__python__maintenance__bugfix__bfcf7192", "SWE-Bench-Verified__python__maintenance__bugfix__deb49033", "SWE-Bench-Verified__python__maintenance__bugfix__e5a3bf2a" ] }, "first_read_source_window_online_replication": { "scope": "Pure ContextBench retrieval-agent development pilot. No repairs, mini-swe-agent or native tests. Cumulative verified source is primary; final served context is secondary. No independent or leaderboard claim.", "protocol_sha256": "53a71235ce999d42fbc2ede073f63e082ff694814b26fdac686eae92759b3dda", "runtime_commit": "4a6992cb13977f682d1e8317a41f5df677720aba", "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.10579999999999998, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 259, "tool_calls": { "grep": 5, "read": 5 }, "sdk_turns": 11, "metrics": { "recall": 0.375, "precision": 0.694980694980695, "f1": 0.4871447902571043 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.166744, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 610, "tool_calls": { "read": 9, "grep": 4 }, "sdk_turns": 14, "metrics": { "recall": 0.15100316789862725, "precision": 0.23442622950819672, "f1": 0.18368657675016056 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.2933673, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1074, "tool_calls": { "read": 9, "grep": 9 }, "sdk_turns": 19, "metrics": { "recall": 0.3648881239242685, "precision": 0.1973929236499069, "f1": 0.25619335347432026 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.3170146, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1504, "tool_calls": { "grep": 8, "read": 11 }, "sdk_turns": 20, "metrics": { "recall": 0.6576576576576577, "precision": 0.1456117021276596, "f1": 0.23843222645617856 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.17972660000000001, "read_views": 5, "confirmed_views_with_added_source": 5, "appended_line_deliveries": 803, "extra_reply_tokens": 9717, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1027, "tool_calls": { "grep": 5, "read": 5 }, "sdk_turns": 11, "metrics": { "recall": 0.6625, "precision": 0.30963972736124634, "f1": 0.42203052422030524 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.2115655, "read_views": 6, "confirmed_views_with_added_source": 5, "appended_line_deliveries": 645, "extra_reply_tokens": 7449, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1155, "tool_calls": { "grep": 6, "read": 6, "glob": 1 }, "sdk_turns": 14, "metrics": { "recall": 0.4118268215417107, "precision": 0.33766233766233766, "f1": 0.37107516650808753 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.3632396, "read_views": 10, "confirmed_views_with_added_source": 9, "appended_line_deliveries": 1103, "extra_reply_tokens": 14505, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2169, "tool_calls": { "read": 10, "grep": 3 }, "sdk_turns": 14, "metrics": { "recall": 0.35456110154905335, "precision": 0.09497464269248501, "f1": 0.1498181818181818 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.5267852000000001, "read_views": 9, "confirmed_views_with_added_source": 9, "appended_line_deliveries": 1462, "extra_reply_tokens": 17697, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2725, "tool_calls": { "grep": 14, "read": 11 }, "sdk_turns": 26, "metrics": { "recall": 0.7537537537537538, "precision": 0.09211009174311927, "f1": 0.16415958142576847 } } ], "primary": { "native": { "recall": 0.3871372373701384, "precision": 0.31810288756661453, "f1": 0.2913642367344409, "official_valid": 4 }, "file-source": { "recall": 0.5456604192111294, "precision": 0.20859669986479706, "f1": 0.2767708634930858, "official_valid": 4 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.3871372373701384, "precision": 0.32080084367140466, "f1": 0.2947942206803057, "official_valid": 4 }, "served_32768": { "recall": 0.3871372373701384, "precision": 0.3183396618498675, "f1": 0.2915362107207078, "official_valid": 4 } }, "file-source": { "served_16384": { "recall": 0.4911760725581081, "precision": 0.2403705858723615, "f1": 0.3104427450281027, "official_valid": 4 }, "served_32768": { "recall": 0.5321469056976159, "precision": 0.20761956208580354, "f1": 0.27474474246728303, "official_valid": 4 } } }, "paired_differences": { "recall": { "mean_difference": 0.1585231818409911, "exploratory_95_interval": [ 0.04288453686044044, 0.2741618268215417 ] }, "precision": { "mean_difference": -0.10950618770181748, "exploratory_95_interval": [ -0.3023811283107216, 0.051822510876250234 ] }, "f1": { "mean_difference": -0.014593373241355158, "exploratory_95_interval": [ -0.09605994525130361, 0.12197328106084271 ] } }, "gates": { "all_eight_sessions_complete": true, "known_usage": true, "one_actual_model_route": true, "confirmed_source_projection_recomputed": true, "added_source_in_three_tasks": true, "original_reads_preserved_and_source_receipts_exact": true, "greater_recall": true }, "development_gate_passed": true, "known_new_api_sdk_usd": 2.1642428000000002, "new_sdk_sessions": 8, "repair_calls": 0, "all_frozen_hashes_match": true, "independent_gain_established": false }, "first_read_source_window_combined8": { "scope": "Eight disclosed development tasks, fixed original order, unchanged first-read source policy. Exploratory development uncertainty, not independent or leaderboard evidence.", "population": [ "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163" ], "macro": { "native": { "recall": 0.40792610941073826, "precision": 0.3439242819030995, "f1": 0.33886782315912967 }, "file-source": { "recall": 0.5193761800694763, "precision": 0.2493465306217156, "f1": 0.3157185524387507 } }, "paired_differences": { "recall": { "mean_difference": 0.11145007065873808, "exploratory_repository_cluster_95_interval": [ 0.03610270633875995, 0.1932384558362046 ] }, "precision": { "mean_difference": -0.09457775128138388, "exploratory_repository_cluster_95_interval": [ -0.19084528505673587, -0.014528715608895983 ] }, "f1": { "mean_difference": -0.023149270720379006, "exploratory_repository_cluster_95_interval": [ -0.0764935375796234, 0.04335434579324511 ] } }, "canonical_repository_clusters": 8, "component_analysis_sha256": { "contextbench-file-source-online-4-20261017": "e49b51cf6733246883620a27c4b2fa5aa8608dd4b410b8edffdbdb365274456b", "contextbench-file-source-replication-4-20261017": "c8542f558c772e897a2b88f9571eb841dfb67747aa1af29c290efcefb52b7779" }, "all_component_screens_passed": true, "api_sdk_estimate_usd": 3.8074519000000002, "sdk_sessions": 16, "repair_calls": 0, "independent_gain_established": false }, "known_source_exposure_inventory": { "scope": "Metadata-only known-exposure/source-identity inventory, not task selection or annotation access. Covers original105 ContextBench development tasks, both sealed32 cohorts, ten preview tasks and the complete100-issue CodeNib Base test split used by research/product Jev comparisons. Exclude any exact previously seen commit regardless of repo URL/fork alias.", "full_dataset_sha256": "2f56535bdc73eb8a68bf4ebb49789d8e9cd4f219ea60df6290b85278aee61ca8", "verified_dataset_sha256": "e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791", "columns_read": [ "instance_id", "original_inst_id", "repo", "repo_url", "base_commit", "language" ], "sources_sha256": { "verification/context-agent-holdout-32/selection.json": "8139a993504a6caec2e2b9ecaa85897ff6321053a2fccded438b122af53594f8", "verification/context-agent-holdout-32/selected.jsonl": "49e0370ff0f384aed1691cb8dd3f071cf900c9020fadd3608d3f1955912be64a", "verification/contextbench-neighbors-independent-32-20261017/prepared/selected.jsonl": "be63992acdb658789f5c87c0f63a1652defdd0a3811559a7ebe7ca373e3f935b", "verification/contextbench-dataset-preview-exposure-20261017.json": "86fbaf8df71d9518ad024f518309d2506801e61c9ee3e1898c5aea25702c2a09", "docs/assets/grep_jev_product_live_v2.json": "69cb3e5ae5da83f1d12325157ec7cbb65530e89c793ee2392f669d150d550580" }, "contextbench_development": 105, "sealed_validation": 64, "preview_quarantine": 10, "codenib_base_exposed": 100, "additional_outside_verified500": 636, "additional_after_known_exposure_audit": 623, "additional_available_per_language": { "c": 44, "cpp": 48, "go": 64, "java": 42, "javascript": 93, "python": 237, "rust": 42, "typescript": 53 }, "additional_new_known_exclusions_beyond_preview": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__f3191113", "reasons": [ "original_identity", "same_source_commit" ] }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__6eb2aa41", "reasons": [ "original_identity", "same_source_commit" ] }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__5642a8e0", "reasons": [ "original_identity", "same_source_commit" ] }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__9bd79689", "reasons": [ "original_identity" ] }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__13ef1e10", "reasons": [ "original_identity" ] }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__2a3f0297", "reasons": [ "original_identity", "same_source_commit" ] }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__0cdc1a09", "reasons": [ "original_identity" ] } ], "remaining_verified_after_known_exposure_audit": 324, "remaining_verified_per_language": { "c": 7, "go": 22, "javascript": 40, "python": 206, "rust": 5, "typescript": 44 }, "new_task_selection": false, "annotation_columns_loaded": false, "new_api_cost_usd": 0, "open_readiness_gates": [ "Freeze separate default-only sampling scope if needed; it is not verified500.", "Prepare exact selected source commits with empty gold before inference." ] }, "default_dataset_annotation_format_readiness": { "scope": "Schema/encoding readiness only on ten already quarantined preview rows, no new unseen-task annotation access or selection. Do not print ranges, code or reference patches.", "protocol_sha256": "0f69c78474e19a9d51703e5fcf53cc6958f7ce24a7cd41ba4a2d298f4f5c5202", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__8adb3e00", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 3 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__6ed8d38b", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 4 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__deb49033", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 18 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__958ed1fd", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 2 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__71f348da", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 5 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__bfcf7192", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 4 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1d90db61", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 5 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__3b78a79f", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 4 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__88e3486b", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 4 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__e5a3bf2a", "gold_encoding_types": [ "str", "list" ], "json_decode_depth": 1, "direct_existing_loader": "ok", "explicit_normalized_schema": "ok", "span_count": 6 } ], "existing_loader_status_counts": { "ok": 10 }, "encoding_depth_counts": { "1": 10 }, "all_quarantined_rows_normalize_and_validate": true, "unseen_annotation_rows_loaded": 0, "reference_patch_columns_loaded": false, "new_task_selection": false, "new_model_calls": 0, "new_api_cost_usd": 0 }, "default_dataset_pinned_provenance": { "scope": "Read-only Hugging Face file metadata verified via huggingface_hub.get_hf_file_metadata; no dataset preview page or new annotation access.", "repo_id": "Contextbench/ContextBench", "filename": "data/full.parquet", "dataset_revision": "c2855792b006af41c67202d33883fb9d46362853", "metadata_commit_hash": "c2855792b006af41c67202d33883fb9d46362853", "etag": "2f56535bdc73eb8a68bf4ebb49789d8e9cd4f219ea60df6290b85278aee61ca8", "local_sha256": "2f56535bdc73eb8a68bf4ebb49789d8e9cd4f219ea60df6290b85278aee61ca8", "size": 26102607, "local_size": 26102607, "matches": true, "new_model_calls": 0, "new_api_cost_usd": 0 }, "first_read_source_window_wider_online": { "scope": "Unchanged first-read source policy on all16 remaining disclosed first-source development tasks. Pure context retrieval; no repairs, independent or leaderboard claim.", "protocol_sha256": "6a71770c44645adb172fff2d437dcd53db146c7a5bb4efb0367667a691fda8c7", "runtime_commit": "4a6992cb13977f682d1e8317a41f5df677720aba", "rows": [ { "arm": "native", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "status": "ok", "sdk_usd": 0.2329812, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 914, "tool_calls": { "grep": 5, "read": 7 }, "sdk_turns": 13, "metrics": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "status": "ok", "sdk_usd": 0.08864910000000001, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 330, "tool_calls": { "grep": 2, "read": 2 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.21515151515151515, "f1": 0.35411471321695764 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "status": "ok", "sdk_usd": 0.1509037, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 733, "tool_calls": { "grep": 2, "read": 5 }, "sdk_turns": 8, "metrics": { "recall": 0.8706140350877193, "precision": 0.5416098226466576, "f1": 0.6677880571909168 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "status": "ok", "sdk_usd": 0.1315079, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 471, "tool_calls": { "grep": 6, "glob": 2, "read": 3 }, "sdk_turns": 12, "metrics": { "recall": 1.0, "precision": 0.04246284501061571, "f1": 0.0814663951120163 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "status": "ok", "sdk_usd": 0.1176269, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 334, "tool_calls": { "grep": 4, "read": 6, "glob": 1 }, "sdk_turns": 12, "metrics": { "recall": 0.2608695652173913, "precision": 0.41317365269461076, "f1": 0.3198146002317497 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "status": "ok", "sdk_usd": 0.0842546, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 319, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.5, "precision": 0.15673981191222572, "f1": 0.23866348448687352 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "status": "ok", "sdk_usd": 0.4211373, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1500, "tool_calls": { "grep": 6, "read": 21 }, "sdk_turns": 28, "metrics": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "status": "ok", "sdk_usd": 0.1245299, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 357, "tool_calls": { "grep": 4, "read": 5 }, "sdk_turns": 10, "metrics": { "recall": 0.782608695652174, "precision": 0.05042016806722689, "f1": 0.09473684210526315 } }, { "arm": "native", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "status": "ok", "sdk_usd": 0.1445584, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 604, "tool_calls": { "grep": 4, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "status": "ok", "sdk_usd": 0.5918497000000001, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2887, "tool_calls": { "grep": 10, "read": 19 }, "sdk_turns": 30, "metrics": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "status": "ok", "sdk_usd": 0.159743, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 564, "tool_calls": { "grep": 3, "read": 3, "glob": 1 }, "sdk_turns": 8, "metrics": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "status": "ok", "sdk_usd": 0.08079030000000001, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 321, "tool_calls": { "glob": 1, "read": 2, "grep": 1 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "status": "ok", "sdk_usd": 0.0967964, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 332, "tool_calls": { "grep": 4, "read": 2, "glob": 1 }, "sdk_turns": 8, "metrics": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "status": "ok", "sdk_usd": 0.13847730000000003, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 758, "tool_calls": { "glob": 3, "grep": 3, "read": 5 }, "sdk_turns": 12, "metrics": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "status": "ok", "sdk_usd": 0.0839265, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 341, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "status": "ok", "sdk_usd": 0.1287801, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 314, "tool_calls": { "grep": 4, "read": 7 }, "sdk_turns": 12, "metrics": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 } }, { "arm": "file-source", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "status": "ok", "sdk_usd": 0.24850150000000001, "read_views": 4, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 543, "extra_reply_tokens": 7112, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1082, "tool_calls": { "grep": 7, "read": 5 }, "sdk_turns": 13, "metrics": { "recall": 1.0, "precision": 0.054528650646950096, "f1": 0.10341805433829976 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "status": "ok", "sdk_usd": 0.10421459999999999, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 221, "extra_reply_tokens": 2421, "unconfirmed_tool_replies_excluded": 0, "source_lines": 548, "tool_calls": { "grep": 2, "read": 2 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.12956204379562045, "f1": 0.22940226171243944 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "status": "ok", "sdk_usd": 0.16442550000000003, "read_views": 3, "confirmed_views_with_added_source": 3, "appended_line_deliveries": 419, "extra_reply_tokens": 6133, "unconfirmed_tool_replies_excluded": 0, "source_lines": 872, "tool_calls": { "grep": 5, "read": 4 }, "sdk_turns": 10, "metrics": { "recall": 0.5482456140350878, "precision": 0.286697247706422, "f1": 0.37650602409638556 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "status": "ok", "sdk_usd": 0.14360350000000002, "read_views": 3, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 200, "extra_reply_tokens": 1950, "unconfirmed_tool_replies_excluded": 0, "source_lines": 764, "tool_calls": { "grep": 4, "glob": 3, "read": 3 }, "sdk_turns": 11, "metrics": { "recall": 1.0, "precision": 0.02617801047120419, "f1": 0.05102040816326531 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "status": "ok", "sdk_usd": 0.24683710000000003, "read_views": 5, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 643, "extra_reply_tokens": 8189, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1238, "tool_calls": { "grep": 7, "read": 7, "glob": 1 }, "sdk_turns": 16, "metrics": { "recall": 0.6616257088846881, "precision": 0.2827140549273021, "f1": 0.39615166949632147 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "status": "ok", "sdk_usd": 0.1104165, "read_views": 3, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 227, "extra_reply_tokens": 2765, "unconfirmed_tool_replies_excluded": 0, "source_lines": 503, "tool_calls": { "grep": 4, "read": 3 }, "sdk_turns": 8, "metrics": { "recall": 0.53, "precision": 0.10536779324055666, "f1": 0.175787728026534 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "status": "ok", "sdk_usd": 0.49294229999999994, "read_views": 9, "confirmed_views_with_added_source": 9, "appended_line_deliveries": 1183, "extra_reply_tokens": 17682, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2301, "tool_calls": { "grep": 4, "read": 19 }, "sdk_turns": 24, "metrics": { "recall": 0.37893700787401574, "precision": 0.16731855714906563, "f1": 0.23213747362074164 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "status": "ok", "sdk_usd": 0.1890532, "read_views": 4, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 650, "extra_reply_tokens": 8182, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1031, "tool_calls": { "grep": 4, "read": 6 }, "sdk_turns": 11, "metrics": { "recall": 0.7391304347826086, "precision": 0.016488845780795344, "f1": 0.03225806451612903 } }, { "arm": "file-source", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "status": "ok", "sdk_usd": 0.1882015, "read_views": 4, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 462, "extra_reply_tokens": 5397, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1021, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.9033457249070632, "precision": 0.23800195886385897, "f1": 0.3767441860465116 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "status": "ok", "sdk_usd": 0.5858146999999999, "read_views": 12, "confirmed_views_with_added_source": 10, "appended_line_deliveries": 1574, "extra_reply_tokens": 18552, "unconfirmed_tool_replies_excluded": 0, "source_lines": 3919, "tool_calls": { "grep": 6, "read": 13 }, "sdk_turns": 20, "metrics": { "recall": 0.7266313932980599, "precision": 0.1051288594029089, "f1": 0.18368256798930002 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "status": "ok", "sdk_usd": 0.16880620000000002, "read_views": 2, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 179, "extra_reply_tokens": 2044, "unconfirmed_tool_replies_excluded": 0, "source_lines": 693, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.9982332155477032, "precision": 0.8152958152958153, "f1": 0.897537728355838 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "status": "ok", "sdk_usd": 0.1025027, "read_views": 2, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 183, "extra_reply_tokens": 2045, "unconfirmed_tool_replies_excluded": 0, "source_lines": 504, "tool_calls": { "glob": 1, "read": 2, "grep": 1 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.027777777777777776, "f1": 0.05405405405405406 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "status": "ok", "sdk_usd": 0.1142163, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 331, "extra_reply_tokens": 4091, "unconfirmed_tool_replies_excluded": 0, "source_lines": 548, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "status": "ok", "sdk_usd": 0.1699741, "read_views": 5, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 181, "extra_reply_tokens": 2044, "unconfirmed_tool_replies_excluded": 0, "source_lines": 988, "tool_calls": { "glob": 4, "grep": 4, "read": 5 }, "sdk_turns": 14, "metrics": { "recall": 0.5491329479768786, "precision": 0.19230769230769232, "f1": 0.28485757121439287 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "status": "ok", "sdk_usd": 0.10206570000000001, "read_views": 3, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 280, "extra_reply_tokens": 3201, "unconfirmed_tool_replies_excluded": 0, "source_lines": 613, "tool_calls": { "grep": 2, "read": 3 }, "sdk_turns": 6, "metrics": { "recall": 0.37316176470588236, "precision": 0.6623164763458401, "f1": 0.4773662551440329 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "status": "ok", "sdk_usd": 0.1455667, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 299, "extra_reply_tokens": 4091, "unconfirmed_tool_replies_excluded": 0, "source_lines": 734, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.7759562841530054, "precision": 0.19346049046321526, "f1": 0.30970556161395857 } } ], "primary": { "native": { "recall": 0.6718878434887245, "precision": 0.2766106953368631, "f1": 0.3100519933706072, "official_valid": 16 }, "file-source": { "recall": 0.699025006010312, "precision": 0.20644651713593906, "f1": 0.2612893505242628, "official_valid": 16 } }, "served_secondary": { "native": { "served_16384": { "recall": 0.6588868866798313, "precision": 0.27868812857076236, "f1": 0.31175073821996546, "official_valid": 16 }, "served_32768": { "recall": 0.671777419460456, "precision": 0.2774227181297024, "f1": 0.31098447388483885, "official_valid": 16 } }, "file-source": { "served_16384": { "recall": 0.6832045798098784, "precision": 0.22280937953631152, "f1": 0.2745295675427645, "official_valid": 16 }, "served_32768": { "recall": 0.6980855321908307, "precision": 0.20890162945635202, "f1": 0.26450053663863227, "official_valid": 16 } } }, "paired_differences": { "recall": { "mean_difference": 0.027137162521587523, "exploratory_95_interval": [ -0.0841004296089015, 0.12825143950036355 ] }, "precision": { "mean_difference": -0.07016417820092405, "exploratory_95_interval": [ -0.11108405585571073, -0.03125912931003419 ] }, "f1": { "mean_difference": -0.048762642846344406, "exploratory_95_interval": [ -0.10534721824308144, 0.006897323973596482 ] } }, "gates": { "all32_sessions_complete": true, "known_usage": true, "one_actual_model_route": true, "confirmed_source_projection_payloads_validated": true, "added_source_in12_tasks": true, "original_reads_preserved_and_source_receipts_exact": true, "greater_recall": true }, "development_gate_passed": true, "known_new_api_sdk_usd": 6.053654399999998, "new_sdk_sessions": 32, "canonical_repository_clusters": 16, "repair_calls": 0, "all_frozen_hashes_match": true, "independent_gain_established": false }, "first_read_source_window_combined24": { "scope": "All24 previously disclosed development tasks, unchanged runtime/policy, matched actual source agents. Development uncertainty only, not independent or full500 quality. No repair or native test batch.", "protocol_sha256": "6a71770c44645adb172fff2d437dcd53db146c7a5bb4efb0367667a691fda8c7", "runtime_commit": "4a6992cb13977f682d1e8317a41f5df677720aba", "population": [ "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "SWE-Bench-Verified__python__maintenance__bugfix__42c77239" ], "rows": [ { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.0882124, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 222, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.38589211618257263, "precision": 0.4189189189189189, "f1": 0.4017278617710583 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.1886935, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 568, "tool_calls": { "grep": 6, "glob": 1, "read": 8 }, "sdk_turns": 16, "metrics": { "recall": 0.2653061224489796, "precision": 0.09154929577464789, "f1": 0.13612565445026178 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.38395170000000006, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1785, "tool_calls": { "grep": 8, "read": 12, "glob": 2 }, "sdk_turns": 23, "metrics": { "recall": 0.5273922571219869, "precision": 0.40448179271708684, "f1": 0.4578313253012048 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.11077899999999999, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 367, "tool_calls": { "grep": 4, "read": 2 }, "sdk_turns": 7, "metrics": { "recall": 0.5362694300518135, "precision": 0.5640326975476839, "f1": 0.5498007968127491 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "language": "python", "status": "ok", "sdk_usd": 0.1094498, "read_views": 3, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 161, "extra_reply_tokens": 2045, "unconfirmed_tool_replies_excluded": 0, "source_lines": 373, "tool_calls": { "grep": 2, "read": 5, "glob": 1 }, "sdk_turns": 9, "metrics": { "recall": 0.4896265560165975, "precision": 0.3163538873994638, "f1": 0.3843648208469055 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "language": "typescript", "status": "ok", "sdk_usd": 0.2364218, "read_views": 5, "confirmed_views_with_added_source": 3, "appended_line_deliveries": 536, "extra_reply_tokens": 6138, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1289, "tool_calls": { "grep": 5, "read": 7, "glob": 1 }, "sdk_turns": 14, "metrics": { "recall": 0.27040816326530615, "precision": 0.041117145073700546, "f1": 0.07138047138047138 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "language": "c", "status": "ok", "sdk_usd": 0.39622889999999994, "read_views": 9, "confirmed_views_with_added_source": 7, "appended_line_deliveries": 1171, "extra_reply_tokens": 13395, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2391, "tool_calls": { "grep": 5, "read": 10, "glob": 1 }, "sdk_turns": 17, "metrics": { "recall": 0.5076698319941563, "precision": 0.29067335842743625, "f1": 0.3696808510638298 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "language": "cpp", "status": "ok", "sdk_usd": 0.129472, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 139, "extra_reply_tokens": 2429, "unconfirmed_tool_replies_excluded": 0, "source_lines": 531, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.7046632124352331, "precision": 0.512241054613936, "f1": 0.5932388222464559 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.10579999999999998, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 259, "tool_calls": { "grep": 5, "read": 5 }, "sdk_turns": 11, "metrics": { "recall": 0.375, "precision": 0.694980694980695, "f1": 0.4871447902571043 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.166744, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 610, "tool_calls": { "read": 9, "grep": 4 }, "sdk_turns": 14, "metrics": { "recall": 0.15100316789862725, "precision": 0.23442622950819672, "f1": 0.18368657675016056 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.2933673, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1074, "tool_calls": { "read": 9, "grep": 9 }, "sdk_turns": 19, "metrics": { "recall": 0.3648881239242685, "precision": 0.1973929236499069, "f1": 0.25619335347432026 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.3170146, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1504, "tool_calls": { "grep": 8, "read": 11 }, "sdk_turns": 20, "metrics": { "recall": 0.6576576576576577, "precision": 0.1456117021276596, "f1": 0.23843222645617856 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "language": "go", "status": "ok", "sdk_usd": 0.17972660000000001, "read_views": 5, "confirmed_views_with_added_source": 5, "appended_line_deliveries": 803, "extra_reply_tokens": 9717, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1027, "tool_calls": { "grep": 5, "read": 5 }, "sdk_turns": 11, "metrics": { "recall": 0.6625, "precision": 0.30963972736124634, "f1": 0.42203052422030524 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "language": "java", "status": "ok", "sdk_usd": 0.2115655, "read_views": 6, "confirmed_views_with_added_source": 5, "appended_line_deliveries": 645, "extra_reply_tokens": 7449, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1155, "tool_calls": { "grep": 6, "read": 6, "glob": 1 }, "sdk_turns": 14, "metrics": { "recall": 0.4118268215417107, "precision": 0.33766233766233766, "f1": 0.37107516650808753 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "language": "javascript", "status": "ok", "sdk_usd": 0.3632396, "read_views": 10, "confirmed_views_with_added_source": 9, "appended_line_deliveries": 1103, "extra_reply_tokens": 14505, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2169, "tool_calls": { "read": 10, "grep": 3 }, "sdk_turns": 14, "metrics": { "recall": 0.35456110154905335, "precision": 0.09497464269248501, "f1": 0.1498181818181818 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "language": "rust", "status": "ok", "sdk_usd": 0.5267852000000001, "read_views": 9, "confirmed_views_with_added_source": 9, "appended_line_deliveries": 1462, "extra_reply_tokens": 17697, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2725, "tool_calls": { "grep": 14, "read": 11 }, "sdk_turns": 26, "metrics": { "recall": 0.7537537537537538, "precision": 0.09211009174311927, "f1": 0.16415958142576847 } }, { "arm": "native", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "status": "ok", "sdk_usd": 0.2329812, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 914, "tool_calls": { "grep": 5, "read": 7 }, "sdk_turns": 13, "metrics": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "status": "ok", "sdk_usd": 0.08864910000000001, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 330, "tool_calls": { "grep": 2, "read": 2 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.21515151515151515, "f1": 0.35411471321695764 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "status": "ok", "sdk_usd": 0.1509037, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 733, "tool_calls": { "grep": 2, "read": 5 }, "sdk_turns": 8, "metrics": { "recall": 0.8706140350877193, "precision": 0.5416098226466576, "f1": 0.6677880571909168 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "status": "ok", "sdk_usd": 0.1315079, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 471, "tool_calls": { "grep": 6, "glob": 2, "read": 3 }, "sdk_turns": 12, "metrics": { "recall": 1.0, "precision": 0.04246284501061571, "f1": 0.0814663951120163 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "status": "ok", "sdk_usd": 0.1176269, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 334, "tool_calls": { "grep": 4, "read": 6, "glob": 1 }, "sdk_turns": 12, "metrics": { "recall": 0.2608695652173913, "precision": 0.41317365269461076, "f1": 0.3198146002317497 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "status": "ok", "sdk_usd": 0.0842546, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 319, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.5, "precision": 0.15673981191222572, "f1": 0.23866348448687352 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "status": "ok", "sdk_usd": 0.4211373, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1500, "tool_calls": { "grep": 6, "read": 21 }, "sdk_turns": 28, "metrics": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "status": "ok", "sdk_usd": 0.1245299, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 357, "tool_calls": { "grep": 4, "read": 5 }, "sdk_turns": 10, "metrics": { "recall": 0.782608695652174, "precision": 0.05042016806722689, "f1": 0.09473684210526315 } }, { "arm": "native", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "status": "ok", "sdk_usd": 0.1445584, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 604, "tool_calls": { "grep": 4, "read": 4 }, "sdk_turns": 9, "metrics": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "status": "ok", "sdk_usd": 0.5918497000000001, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2887, "tool_calls": { "grep": 10, "read": 19 }, "sdk_turns": 30, "metrics": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "status": "ok", "sdk_usd": 0.159743, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 564, "tool_calls": { "grep": 3, "read": 3, "glob": 1 }, "sdk_turns": 8, "metrics": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "status": "ok", "sdk_usd": 0.08079030000000001, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 321, "tool_calls": { "glob": 1, "read": 2, "grep": 1 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "status": "ok", "sdk_usd": 0.0967964, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 332, "tool_calls": { "grep": 4, "read": 2, "glob": 1 }, "sdk_turns": 8, "metrics": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 } }, { "arm": "native", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "status": "ok", "sdk_usd": 0.13847730000000003, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 758, "tool_calls": { "glob": 3, "grep": 3, "read": 5 }, "sdk_turns": 12, "metrics": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 } }, { "arm": "native", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "status": "ok", "sdk_usd": 0.0839265, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 341, "tool_calls": { "grep": 3, "read": 3 }, "sdk_turns": 7, "metrics": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 } }, { "arm": "native", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "status": "ok", "sdk_usd": 0.1287801, "read_views": 0, "confirmed_views_with_added_source": 0, "appended_line_deliveries": 0, "extra_reply_tokens": 0, "unconfirmed_tool_replies_excluded": 0, "source_lines": 314, "tool_calls": { "grep": 4, "read": 7 }, "sdk_turns": 12, "metrics": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 } }, { "arm": "file-source", "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "language": "typescript", "status": "ok", "sdk_usd": 0.24850150000000001, "read_views": 4, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 543, "extra_reply_tokens": 7112, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1082, "tool_calls": { "grep": 7, "read": 5 }, "sdk_turns": 13, "metrics": { "recall": 1.0, "precision": 0.054528650646950096, "f1": 0.10341805433829976 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "language": "python", "status": "ok", "sdk_usd": 0.10421459999999999, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 221, "extra_reply_tokens": 2421, "unconfirmed_tool_replies_excluded": 0, "source_lines": 548, "tool_calls": { "grep": 2, "read": 2 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.12956204379562045, "f1": 0.22940226171243944 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "language": "c", "status": "ok", "sdk_usd": 0.16442550000000003, "read_views": 3, "confirmed_views_with_added_source": 3, "appended_line_deliveries": 419, "extra_reply_tokens": 6133, "unconfirmed_tool_replies_excluded": 0, "source_lines": 872, "tool_calls": { "grep": 5, "read": 4 }, "sdk_turns": 10, "metrics": { "recall": 0.5482456140350878, "precision": 0.286697247706422, "f1": 0.37650602409638556 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "language": "typescript", "status": "ok", "sdk_usd": 0.14360350000000002, "read_views": 3, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 200, "extra_reply_tokens": 1950, "unconfirmed_tool_replies_excluded": 0, "source_lines": 764, "tool_calls": { "grep": 4, "glob": 3, "read": 3 }, "sdk_turns": 11, "metrics": { "recall": 1.0, "precision": 0.02617801047120419, "f1": 0.05102040816326531 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "language": "cpp", "status": "ok", "sdk_usd": 0.24683710000000003, "read_views": 5, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 643, "extra_reply_tokens": 8189, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1238, "tool_calls": { "grep": 7, "read": 7, "glob": 1 }, "sdk_turns": 16, "metrics": { "recall": 0.6616257088846881, "precision": 0.2827140549273021, "f1": 0.39615166949632147 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "language": "cpp", "status": "ok", "sdk_usd": 0.1104165, "read_views": 3, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 227, "extra_reply_tokens": 2765, "unconfirmed_tool_replies_excluded": 0, "source_lines": 503, "tool_calls": { "grep": 4, "read": 3 }, "sdk_turns": 8, "metrics": { "recall": 0.53, "precision": 0.10536779324055666, "f1": 0.175787728026534 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "language": "c", "status": "ok", "sdk_usd": 0.49294229999999994, "read_views": 9, "confirmed_views_with_added_source": 9, "appended_line_deliveries": 1183, "extra_reply_tokens": 17682, "unconfirmed_tool_replies_excluded": 0, "source_lines": 2301, "tool_calls": { "grep": 4, "read": 19 }, "sdk_turns": 24, "metrics": { "recall": 0.37893700787401574, "precision": 0.16731855714906563, "f1": 0.23213747362074164 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "language": "rust", "status": "ok", "sdk_usd": 0.1890532, "read_views": 4, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 650, "extra_reply_tokens": 8182, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1031, "tool_calls": { "grep": 4, "read": 6 }, "sdk_turns": 11, "metrics": { "recall": 0.7391304347826086, "precision": 0.016488845780795344, "f1": 0.03225806451612903 } }, { "arm": "file-source", "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "language": "javascript", "status": "ok", "sdk_usd": 0.1882015, "read_views": 4, "confirmed_views_with_added_source": 4, "appended_line_deliveries": 462, "extra_reply_tokens": 5397, "unconfirmed_tool_replies_excluded": 0, "source_lines": 1021, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.9033457249070632, "precision": 0.23800195886385897, "f1": 0.3767441860465116 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "language": "rust", "status": "ok", "sdk_usd": 0.5858146999999999, "read_views": 12, "confirmed_views_with_added_source": 10, "appended_line_deliveries": 1574, "extra_reply_tokens": 18552, "unconfirmed_tool_replies_excluded": 0, "source_lines": 3919, "tool_calls": { "grep": 6, "read": 13 }, "sdk_turns": 20, "metrics": { "recall": 0.7266313932980599, "precision": 0.1051288594029089, "f1": 0.18368256798930002 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "language": "java", "status": "ok", "sdk_usd": 0.16880620000000002, "read_views": 2, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 179, "extra_reply_tokens": 2044, "unconfirmed_tool_replies_excluded": 0, "source_lines": 693, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.9982332155477032, "precision": 0.8152958152958153, "f1": 0.897537728355838 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "language": "go", "status": "ok", "sdk_usd": 0.1025027, "read_views": 2, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 183, "extra_reply_tokens": 2045, "unconfirmed_tool_replies_excluded": 0, "source_lines": 504, "tool_calls": { "glob": 1, "read": 2, "grep": 1 }, "sdk_turns": 5, "metrics": { "recall": 1.0, "precision": 0.027777777777777776, "f1": 0.05405405405405406 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "language": "java", "status": "ok", "sdk_usd": 0.1142163, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 331, "extra_reply_tokens": 4091, "unconfirmed_tool_replies_excluded": 0, "source_lines": 548, "tool_calls": { "grep": 3, "read": 2 }, "sdk_turns": 6, "metrics": { "recall": 0.0, "precision": 0.0, "f1": 0 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "language": "go", "status": "ok", "sdk_usd": 0.1699741, "read_views": 5, "confirmed_views_with_added_source": 1, "appended_line_deliveries": 181, "extra_reply_tokens": 2044, "unconfirmed_tool_replies_excluded": 0, "source_lines": 988, "tool_calls": { "glob": 4, "grep": 4, "read": 5 }, "sdk_turns": 14, "metrics": { "recall": 0.5491329479768786, "precision": 0.19230769230769232, "f1": 0.28485757121439287 } }, { "arm": "file-source", "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "language": "javascript", "status": "ok", "sdk_usd": 0.10206570000000001, "read_views": 3, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 280, "extra_reply_tokens": 3201, "unconfirmed_tool_replies_excluded": 0, "source_lines": 613, "tool_calls": { "grep": 2, "read": 3 }, "sdk_turns": 6, "metrics": { "recall": 0.37316176470588236, "precision": 0.6623164763458401, "f1": 0.4773662551440329 } }, { "arm": "file-source", "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "language": "python", "status": "ok", "sdk_usd": 0.1455667, "read_views": 2, "confirmed_views_with_added_source": 2, "appended_line_deliveries": 299, "extra_reply_tokens": 4091, "unconfirmed_tool_replies_excluded": 0, "source_lines": 734, "tool_calls": { "grep": 3, "read": 4 }, "sdk_turns": 8, "metrics": { "recall": 0.7759562841530054, "precision": 0.19346049046321526, "f1": 0.30970556161395857 } } ], "macro": { "native": { "recall": 0.5839005987960625, "precision": 0.2990485575256086, "f1": 0.31965726996678134 }, "file-source": { "recall": 0.6391420640300335, "precision": 0.2207465216311979, "f1": 0.27943241782909206 } }, "paired_differences": { "recall": { "mean_difference": 0.05524146523397105, "paired_repository_cluster_95_interval": [ -0.027556323449777605, 0.12518045998254798 ] }, "precision": { "mean_difference": -0.07830203589441066, "paired_repository_cluster_95_interval": [ -0.12083346709306969, -0.04120163211797684 ] }, "f1": { "mean_difference": -0.040224852137689275, "paired_repository_cluster_95_interval": [ -0.08301724525530074, 0.00150039182102356 ] } }, "canonical_repository_clusters": 24, "component_analysis_sha256": { "contextbench-file-source-online-4-20261017": "e49b51cf6733246883620a27c4b2fa5aa8608dd4b410b8edffdbdb365274456b", "contextbench-file-source-replication-4-20261017": "c8542f558c772e897a2b88f9571eb841dfb67747aa1af29c290efcefb52b7779", "contextbench-file-source-wider-online-16-20261017": "e94afc0320d98f21b64fb4d17c097fa6a414d509a9812cc85f6cf1c94bb7c034" }, "gates": { "all_three_development_screens_passed": true, "all48_sessions_complete": true, "all24_retained_in_both_arms": true, "matched_runtime_source_usage_route_delivery_audits": true, "recall_lower_95_bound_positive": false }, "development_stability_gate_passed": false, "api_sdk_estimate_usd": 9.8611063, "sdk_sessions": 48, "repair_calls": 0, "independent_gain_established": false }, "first_read_source_window_runtime_environment": { "scope": "Completed three-stage24-task development environment; matching configuration values, not inferred underlying HTTP request counts. max_turns is a configured SDK guard; separately reported SDK num_turns is not an exact model-request count.", "configs": 48, "versions": { "sdk_version": [ "0.2.165" ], "cli_version": [ "2.1.301 (Claude Code)" ], "model_requested": [ "claude-opus-5-5" ], "billing_route": [ "anthropic-api" ], "max_turns": [ "20" ], "case_meter_usd": [ "0.75" ], "case_timeout": [ "600.0" ] }, "all_matched": true, "new_model_calls": 0 }, "first_read_source_window_behavior_diagnostic": { "scope": "Descriptive fixed-call source decomposition, not causal attribution or a new quality trial. All24 tasks and failed Recall stability gate retained.", "protocol_sha256": "d1a97264c6bc7cf78f22233e5d606df9fcc9c4ab3ed4ba007f64a88e291d6a5c", "source_projection_sha256": "e1fb1e250b7dd800be2b33731bf18de0be05692236800c40647dbd6acd396ddd", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "recall": { "native": 0.38589211618257263, "candidate_calls_without_additions": 0.4896265560165975, "candidate_actual": 0.4896265560165975 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.10373443983402486, "total_difference": 0.10373443983402486, "tool_calls": { "native": { "grep": 3, "read": 4 }, "file-source": { "grep": 2, "read": 5, "glob": 1 } }, "source_files": { "native": 2, "candidate_calls_without_additions": 3, "candidate_actual": 3 }, "reported_sdk_turns": { "native": 8, "file-source": 9 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "recall": { "native": 0.2653061224489796, "candidate_calls_without_additions": 0.27040816326530615, "candidate_actual": 0.27040816326530615 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.005102040816326536, "total_difference": 0.005102040816326536, "tool_calls": { "native": { "grep": 6, "glob": 1, "read": 8 }, "file-source": { "grep": 5, "read": 7, "glob": 1 } }, "source_files": { "native": 25, "candidate_calls_without_additions": 26, "candidate_actual": 26 }, "reported_sdk_turns": { "native": 16, "file-source": 14 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "recall": { "native": 0.5273922571219869, "candidate_calls_without_additions": 0.36888239590942296, "candidate_actual": 0.5076698319941563 }, "mechanical_added_source_recall": 0.13878743608473332, "unaugmented_candidate_call_difference": -0.15850986121256394, "total_difference": -0.019722425127830623, "tool_calls": { "native": { "grep": 8, "read": 12, "glob": 2 }, "file-source": { "grep": 5, "read": 10, "glob": 1 } }, "source_files": { "native": 28, "candidate_calls_without_additions": 25, "candidate_actual": 25 }, "reported_sdk_turns": { "native": 23, "file-source": 17 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "recall": { "native": 0.5362694300518135, "candidate_calls_without_additions": 0.5362694300518135, "candidate_actual": 0.7046632124352331 }, "mechanical_added_source_recall": 0.16839378238341962, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.16839378238341962, "tool_calls": { "native": { "grep": 4, "read": 2 }, "file-source": { "grep": 3, "read": 2 } }, "source_files": { "native": 6, "candidate_calls_without_additions": 8, "candidate_actual": 8 }, "reported_sdk_turns": { "native": 7, "file-source": 6 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "recall": { "native": 0.375, "candidate_calls_without_additions": 0.3541666666666667, "candidate_actual": 0.6625 }, "mechanical_added_source_recall": 0.3083333333333333, "unaugmented_candidate_call_difference": -0.020833333333333315, "total_difference": 0.2875, "tool_calls": { "native": { "grep": 5, "read": 5 }, "file-source": { "grep": 5, "read": 5 } }, "source_files": { "native": 8, "candidate_calls_without_additions": 8, "candidate_actual": 8 }, "reported_sdk_turns": { "native": 11, "file-source": 11 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "recall": { "native": 0.15100316789862725, "candidate_calls_without_additions": 0.1246040126715945, "candidate_actual": 0.4118268215417107 }, "mechanical_added_source_recall": 0.2872228088701162, "unaugmented_candidate_call_difference": -0.026399155227032747, "total_difference": 0.26082365364308346, "tool_calls": { "native": { "read": 9, "grep": 4 }, "file-source": { "grep": 6, "read": 6, "glob": 1 } }, "source_files": { "native": 15, "candidate_calls_without_additions": 6, "candidate_actual": 6 }, "reported_sdk_turns": { "native": 14, "file-source": 14 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "recall": { "native": 0.3648881239242685, "candidate_calls_without_additions": 0.35456110154905335, "candidate_actual": 0.35456110154905335 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": -0.010327022375215156, "total_difference": -0.010327022375215156, "tool_calls": { "native": { "read": 9, "grep": 9 }, "file-source": { "read": 10, "grep": 3 } }, "source_files": { "native": 33, "candidate_calls_without_additions": 37, "candidate_actual": 37 }, "reported_sdk_turns": { "native": 19, "file-source": 14 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "recall": { "native": 0.6576576576576577, "candidate_calls_without_additions": 0.5525525525525525, "candidate_actual": 0.7537537537537538 }, "mechanical_added_source_recall": 0.20120120120120122, "unaugmented_candidate_call_difference": -0.10510510510510518, "total_difference": 0.09609609609609604, "tool_calls": { "native": { "grep": 8, "read": 11 }, "file-source": { "grep": 14, "read": 11 } }, "source_files": { "native": 12, "candidate_calls_without_additions": 14, "candidate_actual": 14 }, "reported_sdk_turns": { "native": 20, "file-source": 26 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "recall": { "native": 1.0, "candidate_calls_without_additions": 1.0, "candidate_actual": 1.0 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.0, "tool_calls": { "native": { "grep": 5, "read": 7 }, "file-source": { "grep": 7, "read": 5 } }, "source_files": { "native": 6, "candidate_calls_without_additions": 7, "candidate_actual": 7 }, "reported_sdk_turns": { "native": 13, "file-source": 13 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "recall": { "native": 1.0, "candidate_calls_without_additions": 1.0, "candidate_actual": 1.0 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.0, "tool_calls": { "native": { "grep": 2, "read": 2 }, "file-source": { "grep": 2, "read": 2 } }, "source_files": { "native": 2, "candidate_calls_without_additions": 2, "candidate_actual": 2 }, "reported_sdk_turns": { "native": 5, "file-source": 5 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "recall": { "native": 0.8706140350877193, "candidate_calls_without_additions": 0.3706140350877193, "candidate_actual": 0.5482456140350878 }, "mechanical_added_source_recall": 0.17763157894736847, "unaugmented_candidate_call_difference": -0.5, "total_difference": -0.3223684210526315, "tool_calls": { "native": { "grep": 2, "read": 5 }, "file-source": { "grep": 5, "read": 4 } }, "source_files": { "native": 4, "candidate_calls_without_additions": 4, "candidate_actual": 4 }, "reported_sdk_turns": { "native": 8, "file-source": 10 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "recall": { "native": 1.0, "candidate_calls_without_additions": 1.0, "candidate_actual": 1.0 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.0, "tool_calls": { "native": { "grep": 6, "glob": 2, "read": 3 }, "file-source": { "grep": 4, "glob": 3, "read": 3 } }, "source_files": { "native": 7, "candidate_calls_without_additions": 14, "candidate_actual": 14 }, "reported_sdk_turns": { "native": 12, "file-source": 11 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "recall": { "native": 0.2608695652173913, "candidate_calls_without_additions": 0.3440453686200378, "candidate_actual": 0.6616257088846881 }, "mechanical_added_source_recall": 0.31758034026465026, "unaugmented_candidate_call_difference": 0.08317580340264652, "total_difference": 0.4007561436672968, "tool_calls": { "native": { "grep": 4, "read": 6, "glob": 1 }, "file-source": { "grep": 7, "read": 7, "glob": 1 } }, "source_files": { "native": 7, "candidate_calls_without_additions": 11, "candidate_actual": 11 }, "reported_sdk_turns": { "native": 12, "file-source": 16 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "recall": { "native": 0.5, "candidate_calls_without_additions": 0.53, "candidate_actual": 0.53 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.030000000000000027, "total_difference": 0.030000000000000027, "tool_calls": { "native": { "grep": 3, "read": 4 }, "file-source": { "grep": 4, "read": 3 } }, "source_files": { "native": 12, "candidate_calls_without_additions": 15, "candidate_actual": 15 }, "reported_sdk_turns": { "native": 8, "file-source": 8 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "recall": { "native": 0.18700787401574803, "candidate_calls_without_additions": 0.18700787401574803, "candidate_actual": 0.37893700787401574 }, "mechanical_added_source_recall": 0.1919291338582677, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.1919291338582677, "tool_calls": { "native": { "grep": 6, "read": 21 }, "file-source": { "grep": 4, "read": 19 } }, "source_files": { "native": 17, "candidate_calls_without_additions": 21, "candidate_actual": 21 }, "reported_sdk_turns": { "native": 28, "file-source": 24 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "recall": { "native": 0.782608695652174, "candidate_calls_without_additions": 0.7391304347826086, "candidate_actual": 0.7391304347826086 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": -0.0434782608695653, "total_difference": -0.0434782608695653, "tool_calls": { "native": { "grep": 4, "read": 5 }, "file-source": { "grep": 4, "read": 6 } }, "source_files": { "native": 5, "candidate_calls_without_additions": 6, "candidate_actual": 6 }, "reported_sdk_turns": { "native": 10, "file-source": 11 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "recall": { "native": 0.8661710037174721, "candidate_calls_without_additions": 0.8661710037174721, "candidate_actual": 0.9033457249070632 }, "mechanical_added_source_recall": 0.03717472118959109, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.03717472118959109, "tool_calls": { "native": { "grep": 4, "read": 4 }, "file-source": { "grep": 3, "read": 4 } }, "source_files": { "native": 12, "candidate_calls_without_additions": 9, "candidate_actual": 9 }, "reported_sdk_turns": { "native": 9, "file-source": 8 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "recall": { "native": 0.6807760141093474, "candidate_calls_without_additions": 0.7054673721340388, "candidate_actual": 0.7266313932980599 }, "mechanical_added_source_recall": 0.021164021164021163, "unaugmented_candidate_call_difference": 0.024691358024691357, "total_difference": 0.04585537918871252, "tool_calls": { "native": { "grep": 10, "read": 19 }, "file-source": { "grep": 6, "read": 13 } }, "source_files": { "native": 35, "candidate_calls_without_additions": 23, "candidate_actual": 23 }, "reported_sdk_turns": { "native": 30, "file-source": 20 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "recall": { "native": 0.9699646643109541, "candidate_calls_without_additions": 0.8303886925795053, "candidate_actual": 0.9982332155477032 }, "mechanical_added_source_recall": 0.16784452296819785, "unaugmented_candidate_call_difference": -0.13957597173144876, "total_difference": 0.028268551236749095, "tool_calls": { "native": { "grep": 3, "read": 3, "glob": 1 }, "file-source": { "grep": 3, "read": 2 } }, "source_files": { "native": 4, "candidate_calls_without_additions": 7, "candidate_actual": 7 }, "reported_sdk_turns": { "native": 8, "file-source": 6 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "recall": { "native": 1.0, "candidate_calls_without_additions": 1.0, "candidate_actual": 1.0 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.0, "tool_calls": { "native": { "glob": 1, "read": 2, "grep": 1 }, "file-source": { "glob": 1, "read": 2, "grep": 1 } }, "source_files": { "native": 2, "candidate_calls_without_additions": 2, "candidate_actual": 2 }, "reported_sdk_turns": { "native": 5, "file-source": 5 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "recall": { "native": 0.546875, "candidate_calls_without_additions": 0.0, "candidate_actual": 0.0 }, "mechanical_added_source_recall": 0.0, "unaugmented_candidate_call_difference": -0.546875, "total_difference": -0.546875, "tool_calls": { "native": { "grep": 4, "read": 2, "glob": 1 }, "file-source": { "grep": 3, "read": 2 } }, "source_files": { "native": 2, "candidate_calls_without_additions": 2, "candidate_actual": 2 }, "reported_sdk_turns": { "native": 8, "file-source": 6 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "recall": { "native": 0.2745664739884393, "candidate_calls_without_additions": 0.3583815028901734, "candidate_actual": 0.5491329479768786 }, "mechanical_added_source_recall": 0.19075144508670522, "unaugmented_candidate_call_difference": 0.08381502890173409, "total_difference": 0.2745664739884393, "tool_calls": { "native": { "glob": 3, "grep": 3, "read": 5 }, "file-source": { "glob": 4, "grep": 4, "read": 5 } }, "source_files": { "native": 16, "candidate_calls_without_additions": 6, "candidate_actual": 6 }, "reported_sdk_turns": { "native": 12, "file-source": 14 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "recall": { "native": 0.22058823529411764, "candidate_calls_without_additions": 0.21323529411764705, "candidate_actual": 0.37316176470588236 }, "mechanical_added_source_recall": 0.1599264705882353, "unaugmented_candidate_call_difference": -0.00735294117647059, "total_difference": 0.15257352941176472, "tool_calls": { "native": { "grep": 3, "read": 3 }, "file-source": { "grep": 2, "read": 3 } }, "source_files": { "native": 9, "candidate_calls_without_additions": 8, "candidate_actual": 8 }, "reported_sdk_turns": { "native": 7, "file-source": 6 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "recall": { "native": 0.5901639344262295, "candidate_calls_without_additions": 0.5901639344262295, "candidate_actual": 0.7759562841530054 }, "mechanical_added_source_recall": 0.18579234972677594, "unaugmented_candidate_call_difference": 0.0, "total_difference": 0.18579234972677594, "tool_calls": { "native": { "grep": 4, "read": 7 }, "file-source": { "grep": 3, "read": 4 } }, "source_files": { "native": 4, "candidate_calls_without_additions": 4, "candidate_actual": 4 }, "reported_sdk_turns": { "native": 12, "file-source": 8 } } ], "macro_recall": { "native": 0.5839005987960625, "candidate_calls_without_additions": 0.5327365162939245, "candidate_actual": 0.6391420640300335 }, "mean_mechanical_added_source_recall": 0.10640554773610904, "mean_unaugmented_candidate_call_difference": -0.05116408250213798, "wins_ties_losses": { "wins": 15, "ties": 4, "losses": 5 }, "new_model_calls": 0, "new_api_cost_usd": 0, "turn_count_scope": "Reported SDK num_turns from each terminal receipt; these are not individual underlying HTTP/model request counts." }, "late_source_pool_opportunity": { "scope": "New late-delivery opportunity only on all24 disclosed development cases. Keep their completed native trajectory fixed and preserve the first-read policy failed stability gate. No new main-agent, ranking, repair or independent calls.", "protocol_sha256": "91b1efb85b5088336688de89fcc8326d19a13973d7ed4eb732246fba6263cfc8", "source_replay_sha256": "d1e3b2991035df0b04ea98588ddf40164d69dec9b44e8b843eb471f16e78f5ab", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "native": { "recall": 0.38589211618257263, "precision": 0.4189189189189189, "f1": 0.4017278617710583 }, "near": { "recall": 0.5020746887966805, "precision": 0.3002481389578164, "f1": 0.3757763975155279 }, "full_capped_pool": { "recall": 0.8091286307053942, "precision": 0.08717031738936075, "f1": 0.15738498789346247 } }, "candidate_windows": 34, "eligible": true, "late_context_tokens": 2038 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "native": { "recall": 0.2653061224489796, "precision": 0.09154929577464789, "f1": 0.13612565445026178 }, "near": { "recall": 0.33163265306122447, "precision": 0.08701472556894244, "f1": 0.1378579003181336 }, "full_capped_pool": { "recall": 0.5153061224489796, "precision": 0.010169150221506242, "f1": 0.019944707740916272 } }, "candidate_windows": 161, "eligible": true, "late_context_tokens": 2044 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "native": { "recall": 0.5273922571219869, "precision": 0.40448179271708684, "f1": 0.4578313253012048 }, "near": { "recall": 0.5434623813002192, "precision": 0.3803680981595092, "f1": 0.44751879699248126 }, "full_capped_pool": { "recall": 0.9240321402483564, "precision": 0.09326845093268452, "f1": 0.16943477096169302 } }, "candidate_windows": 200, "eligible": true, "late_context_tokens": 2045 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "native": { "recall": 0.5362694300518135, "precision": 0.5640326975476839, "f1": 0.5498007968127491 }, "near": { "recall": 0.5544041450777202, "precision": 0.40150093808630394, "f1": 0.46572361262241563 }, "full_capped_pool": { "recall": 1.0, "precision": 0.2410993129294191, "f1": 0.3885254151987922 } }, "candidate_windows": 25, "eligible": true, "late_context_tokens": 2035 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "native": { "recall": 0.375, "precision": 0.694980694980695, "f1": 0.4871447902571043 }, "near": { "recall": 0.4270833333333333, "precision": 0.46485260770975056, "f1": 0.4451682953311618 }, "full_capped_pool": { "recall": 0.6895833333333333, "precision": 0.11152291105121294, "f1": 0.19199535962877032 } }, "candidate_windows": 48, "eligible": true, "late_context_tokens": 2040 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "native": { "recall": 0.15100316789862725, "precision": 0.23442622950819672, "f1": 0.18368657675016056 }, "near": { "recall": 0.15100316789862725, "precision": 0.17987421383647798, "f1": 0.16417910447761194 }, "full_capped_pool": { "recall": 0.6610348468848997, "precision": 0.1051747311827957, "f1": 0.1814755761704595 } }, "candidate_windows": 93, "eligible": true, "late_context_tokens": 2043 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "native": { "recall": 0.3648881239242685, "precision": 0.1973929236499069, "f1": 0.25619335347432026 }, "near": { "recall": 0.3648881239242685, "precision": 0.17462932454695224, "f1": 0.23621169916434542 }, "full_capped_pool": { "recall": 0.5370051635111877, "precision": 0.025419586116995276, "f1": 0.048541423570595106 } }, "candidate_windows": 200, "eligible": true, "late_context_tokens": 2044 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "native": { "recall": 0.6576576576576577, "precision": 0.1456117021276596, "f1": 0.23843222645617856 }, "near": { "recall": 0.6756756756756757, "precision": 0.13529765484064943, "f1": 0.22545090180360722 }, "full_capped_pool": { "recall": 0.6966966966966966, "precision": 0.023594020136275807, "f1": 0.04564233720243951 } }, "candidate_windows": 142, "eligible": true, "late_context_tokens": 2038 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "metrics": { "native": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "near": { "recall": 1.0, "precision": 0.0560303893637227, "f1": 0.10611510791366907 }, "full_capped_pool": { "recall": 1.0, "precision": 0.00717848886725879, "f1": 0.01425465088185552 } }, "candidate_windows": 120, "eligible": true, "late_context_tokens": 2041 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "metrics": { "native": { "recall": 1.0, "precision": 0.21515151515151515, "f1": 0.35411471321695764 }, "near": { "recall": 1.0, "precision": 0.1362763915547025, "f1": 0.23986486486486489 }, "full_capped_pool": { "recall": 1.0, "precision": 0.06299911268855368, "f1": 0.11853088480801337 } }, "candidate_windows": 14, "eligible": true, "late_context_tokens": 2045 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "metrics": { "native": { "recall": 0.8706140350877193, "precision": 0.5416098226466576, "f1": 0.6677880571909168 }, "near": { "recall": 1.0, "precision": 0.5129358830146231, "f1": 0.6780669144981413 }, "full_capped_pool": { "recall": 1.0, "precision": 0.04952215464813206, "f1": 0.09437086092715231 } }, "candidate_windows": 137, "eligible": true, "late_context_tokens": 2044 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "metrics": { "native": { "recall": 1.0, "precision": 0.04246284501061571, "f1": 0.0814663951120163 }, "near": { "recall": 1.0, "precision": 0.031007751937984496, "f1": 0.06015037593984962 }, "full_capped_pool": { "recall": 1.0, "precision": 0.007936507936507936, "f1": 0.015748031496062992 } }, "candidate_windows": 36, "eligible": true, "late_context_tokens": 2048 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "metrics": { "native": { "recall": 0.2608695652173913, "precision": 0.41317365269461076, "f1": 0.3198146002317497 }, "near": { "recall": 0.3686200378071834, "precision": 0.38922155688622756, "f1": 0.3786407766990292 }, "full_capped_pool": { "recall": 0.9262759924385633, "precision": 0.038784233022004115, "f1": 0.07445111296816835 } }, "candidate_windows": 200, "eligible": true, "late_context_tokens": 2040 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "metrics": { "native": { "recall": 0.5, "precision": 0.15673981191222572, "f1": 0.23866348448687352 }, "near": { "recall": 0.5, "precision": 0.10395010395010396, "f1": 0.17211703958691912 }, "full_capped_pool": { "recall": 0.5, "precision": 0.004005126561999359, "f1": 0.007946598855689765 } }, "candidate_windows": 200, "eligible": true, "late_context_tokens": 2031 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "metrics": { "native": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 }, "near": { "recall": 0.18700787401574803, "precision": 0.11728395061728394, "f1": 0.1441578148710167 }, "full_capped_pool": { "recall": 0.656496062992126, "precision": 0.05381636275617234, "f1": 0.09947800149142431 } }, "candidate_windows": 200, "eligible": true, "late_context_tokens": 2045 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "metrics": { "native": { "recall": 0.782608695652174, "precision": 0.05042016806722689, "f1": 0.09473684210526315 }, "near": { "recall": 0.782608695652174, "precision": 0.03495145631067961, "f1": 0.06691449814126393 }, "full_capped_pool": { "recall": 1.0, "precision": 0.004493942946463462, "f1": 0.008947675549503988 } }, "candidate_windows": 81, "eligible": true, "late_context_tokens": 2036 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "metrics": { "native": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 }, "near": { "recall": 0.8773234200743495, "precision": 0.30569948186528495, "f1": 0.45341018251681076 }, "full_capped_pool": { "recall": 0.9888475836431226, "precision": 0.07916666666666666, "f1": 0.14659685863874344 } }, "candidate_windows": 51, "eligible": true, "late_context_tokens": 2047 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "metrics": { "native": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 }, "near": { "recall": 0.6807760141093474, "precision": 0.12659888488028861, "f1": 0.2134955752212389 }, "full_capped_pool": { "recall": 0.8201058201058201, "precision": 0.03282044042913608, "f1": 0.06311503223617237 } }, "candidate_windows": 200, "eligible": true, "late_context_tokens": 2045 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "metrics": { "native": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 }, "near": { "recall": 0.9982332155477032, "precision": 0.773972602739726, "f1": 0.8719135802469136 }, "full_capped_pool": { "recall": 0.9982332155477032, "precision": 0.5231481481481481, "f1": 0.6865127582017011 } }, "candidate_windows": 10, "eligible": true, "late_context_tokens": 2044 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "metrics": { "native": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 }, "near": { "recall": 1.0, "precision": 0.029411764705882353, "f1": 0.05714285714285715 }, "full_capped_pool": { "recall": 1.0, "precision": 0.026923076923076925, "f1": 0.05243445692883896 } }, "candidate_windows": 4, "eligible": true, "late_context_tokens": 2043 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "metrics": { "native": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 }, "near": { "recall": 0.6875, "precision": 0.08782435129740519, "f1": 0.15575221238938053 }, "full_capped_pool": { "recall": 1.0, "precision": 0.006967123884171565, "f1": 0.013837837837837838 } }, "candidate_windows": 141, "eligible": true, "late_context_tokens": 2047 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "metrics": { "native": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 }, "near": { "recall": 0.4479768786127168, "precision": 0.16630901287553648, "f1": 0.24256651017214398 }, "full_capped_pool": { "recall": 0.5664739884393064, "precision": 0.017752015216013044, "f1": 0.034425221744094146 } }, "candidate_windows": 181, "eligible": true, "late_context_tokens": 2035 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "metrics": { "native": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 }, "near": { "recall": 0.2545955882352941, "precision": 0.5326923076923077, "f1": 0.3445273631840796 }, "full_capped_pool": { "recall": 0.7941176470588235, "precision": 0.3737024221453287, "f1": 0.508235294117647 } }, "candidate_windows": 37, "eligible": true, "late_context_tokens": 2048 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "metrics": { "native": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 }, "near": { "recall": 0.5901639344262295, "precision": 0.21686746987951808, "f1": 0.31718061674008813 }, "full_capped_pool": { "recall": 0.7759562841530054, "precision": 0.012627834593152512, "f1": 0.024851242562128107 } }, "candidate_windows": 178, "eligible": true, "late_context_tokens": 2032 } ], "macro": { "native": { "recall": 0.5839005987960625, "precision": 0.2990485575256086, "f1": 0.31965726996678134 }, "near": { "recall": 0.6218762428145207, "precision": 0.23936746088657, "f1": 0.291662624931398 }, "full_capped_pool": { "recall": 0.8274705636753049, "precision": 0.08330258905804315, "f1": 0.13194504573384008 } }, "gates": { "all24_retained": true, "exact_source_preserved": true, "eligible_in18_tasks": true, "capped_pool_headroom_ten_points": true }, "opportunity_gate_passed": true, "new_model_calls": 0, "new_api_cost_usd": 0, "independent_gain_established": false }, "late_source_jev_ranking_development": { "scope": "Developer-only late source ordering on all24 completed native traces. Native prefix remains fixed; one2K extra envelope per case, no adaptation or new main-agent behavior. Both old32 independent cohorts sealed. No repair/full500.", "protocol_sha256": "9ac29fab97f8923a6106c42be55b443479a8bf82a8e7c2704bef1c7f26ea043d", "source_delivery_replay_sha256": "34a143712b8056eb15e9de037a36084d29ba379221cdb384bd9c75eac5d6196b", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "native": { "recall": 0.38589211618257263, "precision": 0.4189189189189189, "f1": 0.4017278617710583 }, "near": { "recall": 0.5020746887966805, "precision": 0.3002481389578164, "f1": 0.3757763975155279 }, "novel-relevance": { "recall": 0.4979253112033195, "precision": 0.30927835051546393, "f1": 0.3815580286168522 }, "novel-information": { "recall": 0.4979253112033195, "precision": 0.31007751937984496, "f1": 0.3821656050955414 } }, "new_source_lines": { "novel-relevance": 166, "novel-information": 165 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "native": { "recall": 0.2653061224489796, "precision": 0.09154929577464789, "f1": 0.13612565445026178 }, "near": { "recall": 0.33163265306122447, "precision": 0.08701472556894244, "f1": 0.1378579003181336 }, "novel-relevance": { "recall": 0.4489795918367347, "precision": 0.11655629139072848, "f1": 0.185068349106204 }, "novel-information": { "recall": 0.2653061224489796, "precision": 0.06806282722513089, "f1": 0.10833333333333334 } }, "new_source_lines": { "novel-relevance": 187, "novel-information": 196 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "native": { "recall": 0.5273922571219869, "precision": 0.40448179271708684, "f1": 0.4578313253012048 }, "near": { "recall": 0.5434623813002192, "precision": 0.3803680981595092, "f1": 0.44751879699248126 }, "novel-relevance": { "recall": 0.5273922571219869, "precision": 0.3742871954380508, "f1": 0.4378411158277744 }, "novel-information": { "recall": 0.5273922571219869, "precision": 0.3742871954380508, "f1": 0.4378411158277744 } }, "new_source_lines": { "novel-relevance": 144, "novel-information": 144 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "native": { "recall": 0.5362694300518135, "precision": 0.5640326975476839, "f1": 0.5498007968127491 }, "near": { "recall": 0.5544041450777202, "precision": 0.40150093808630394, "f1": 0.46572361262241563 }, "novel-relevance": { "recall": 0.5362694300518135, "precision": 0.40194174757281553, "f1": 0.45948945615982245 }, "novel-information": { "recall": 0.5544041450777202, "precision": 0.4107485604606526, "f1": 0.4718853362734289 } }, "new_source_lines": { "novel-relevance": 148, "novel-information": 154 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "native": { "recall": 0.375, "precision": 0.694980694980695, "f1": 0.4871447902571043 }, "near": { "recall": 0.4270833333333333, "precision": 0.46485260770975056, "f1": 0.4451682953311618 }, "novel-relevance": { "recall": 0.49375, "precision": 0.5537383177570093, "f1": 0.5220264317180616 }, "novel-information": { "recall": 0.38125, "precision": 0.4357142857142857, "f1": 0.4066666666666667 } }, "new_source_lines": { "novel-relevance": 169, "novel-information": 161 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "native": { "recall": 0.15100316789862725, "precision": 0.23442622950819672, "f1": 0.18368657675016056 }, "near": { "recall": 0.15100316789862725, "precision": 0.17987421383647798, "f1": 0.16417910447761194 }, "novel-relevance": { "recall": 0.3262935586061246, "precision": 0.39819587628865977, "f1": 0.35867672663958206 }, "novel-information": { "recall": 0.2565997888067582, "precision": 0.314359637774903, "f1": 0.28255813953488373 } }, "new_source_lines": { "novel-relevance": 166, "novel-information": 163 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "native": { "recall": 0.3648881239242685, "precision": 0.1973929236499069, "f1": 0.25619335347432026 }, "near": { "recall": 0.3648881239242685, "precision": 0.17462932454695224, "f1": 0.23621169916434542 }, "novel-relevance": { "recall": 0.3648881239242685, "precision": 0.17622610141313383, "f1": 0.23766816143497754 }, "novel-information": { "recall": 0.3648881239242685, "precision": 0.17549668874172186, "f1": 0.2370039128004472 } }, "new_source_lines": { "novel-relevance": 129, "novel-information": 134 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "native": { "recall": 0.6576576576576577, "precision": 0.1456117021276596, "f1": 0.23843222645617856 }, "near": { "recall": 0.6756756756756757, "precision": 0.13529765484064943, "f1": 0.22545090180360722 }, "novel-relevance": { "recall": 0.6756756756756757, "precision": 0.1348112642300779, "f1": 0.2247752247752248 }, "novel-information": { "recall": 0.6756756756756757, "precision": 0.1348112642300779, "f1": 0.2247752247752248 } }, "new_source_lines": { "novel-relevance": 165, "novel-information": 165 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "metrics": { "native": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "near": { "recall": 1.0, "precision": 0.0560303893637227, "f1": 0.10611510791366907 }, "novel-relevance": { "recall": 1.0, "precision": 0.05629770992366412, "f1": 0.1065943992773261 }, "novel-information": { "recall": 1.0, "precision": 0.05640535372848948, "f1": 0.10678733031674209 } }, "new_source_lines": { "novel-relevance": 134, "novel-information": 132 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "metrics": { "native": { "recall": 1.0, "precision": 0.21515151515151515, "f1": 0.35411471321695764 }, "near": { "recall": 1.0, "precision": 0.1362763915547025, "f1": 0.23986486486486489 }, "novel-relevance": { "recall": 1.0, "precision": 0.1362763915547025, "f1": 0.23986486486486489 }, "novel-information": { "recall": 1.0, "precision": 0.14087301587301587, "f1": 0.24695652173913044 } }, "new_source_lines": { "novel-relevance": 191, "novel-information": 174 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "metrics": { "native": { "recall": 0.8706140350877193, "precision": 0.5416098226466576, "f1": 0.6677880571909168 }, "near": { "recall": 1.0, "precision": 0.5129358830146231, "f1": 0.6780669144981413 }, "novel-relevance": { "recall": 1.0, "precision": 0.5049833887043189, "f1": 0.6710816777041942 }, "novel-information": { "recall": 1.0, "precision": 0.5049833887043189, "f1": 0.6710816777041942 } }, "new_source_lines": { "novel-relevance": 170, "novel-information": 170 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "metrics": { "native": { "recall": 1.0, "precision": 0.04246284501061571, "f1": 0.0814663951120163 }, "near": { "recall": 1.0, "precision": 0.031007751937984496, "f1": 0.06015037593984962 }, "novel-relevance": { "recall": 1.0, "precision": 0.029498525073746312, "f1": 0.05730659025787966 }, "novel-information": { "recall": 1.0, "precision": 0.029498525073746312, "f1": 0.05730659025787966 } }, "new_source_lines": { "novel-relevance": 207, "novel-information": 207 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "metrics": { "native": { "recall": 0.2608695652173913, "precision": 0.41317365269461076, "f1": 0.3198146002317497 }, "near": { "recall": 0.3686200378071834, "precision": 0.38922155688622756, "f1": 0.3786407766990292 }, "novel-relevance": { "recall": 0.2608695652173913, "precision": 0.2787878787878788, "f1": 0.26953124999999994 }, "novel-information": { "recall": 0.2608695652173913, "precision": 0.2732673267326733, "f1": 0.26692456479690524 } }, "new_source_lines": { "novel-relevance": 161, "novel-information": 171 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "metrics": { "native": { "recall": 0.5, "precision": 0.15673981191222572, "f1": 0.23866348448687352 }, "near": { "recall": 0.5, "precision": 0.10395010395010396, "f1": 0.17211703958691912 }, "novel-relevance": { "recall": 0.5, "precision": 0.10482180293501048, "f1": 0.17331022530329288 }, "novel-information": { "recall": 0.5, "precision": 0.10183299389002037, "f1": 0.1692047377326565 } }, "new_source_lines": { "novel-relevance": 158, "novel-information": 172 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "metrics": { "native": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 }, "near": { "recall": 0.18700787401574803, "precision": 0.11728395061728394, "f1": 0.1441578148710167 }, "novel-relevance": { "recall": 0.18700787401574803, "precision": 0.1162079510703364, "f1": 0.14334213504337986 }, "novel-information": { "recall": 0.18700787401574803, "precision": 0.11677934849416104, "f1": 0.14377601210745364 } }, "new_source_lines": { "novel-relevance": 135, "novel-information": 127 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "metrics": { "native": { "recall": 0.782608695652174, "precision": 0.05042016806722689, "f1": 0.09473684210526315 }, "near": { "recall": 0.782608695652174, "precision": 0.03495145631067961, "f1": 0.06691449814126393 }, "novel-relevance": { "recall": 0.782608695652174, "precision": 0.03409090909090909, "f1": 0.06533575317604355 }, "novel-information": { "recall": 0.782608695652174, "precision": 0.03495145631067961, "f1": 0.06691449814126393 } }, "new_source_lines": { "novel-relevance": 171, "novel-information": 158 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "metrics": { "native": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 }, "near": { "recall": 0.8773234200743495, "precision": 0.30569948186528495, "f1": 0.45341018251681076 }, "novel-relevance": { "recall": 0.8921933085501859, "precision": 0.31007751937984496, "f1": 0.46021093000958774 }, "novel-information": { "recall": 0.9516728624535316, "precision": 0.33376792698826596, "f1": 0.49420849420849416 } }, "new_source_lines": { "novel-relevance": 170, "novel-information": 163 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "metrics": { "native": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 }, "near": { "recall": 0.6807760141093474, "precision": 0.12659888488028861, "f1": 0.2134955752212389 }, "novel-relevance": { "recall": 0.689594356261023, "precision": 0.12878787878787878, "f1": 0.21704135442686648 }, "novel-information": { "recall": 0.689594356261023, "precision": 0.12878787878787878, "f1": 0.21704135442686648 } }, "new_source_lines": { "novel-relevance": 149, "novel-information": 149 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "metrics": { "native": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 }, "near": { "recall": 0.9982332155477032, "precision": 0.773972602739726, "f1": 0.8719135802469136 }, "novel-relevance": { "recall": 0.9699646643109541, "precision": 0.746938775510204, "f1": 0.8439661798616449 }, "novel-information": { "recall": 0.9699646643109541, "precision": 0.7398921832884097, "f1": 0.8394495412844035 } }, "new_source_lines": { "novel-relevance": 171, "novel-information": 178 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "metrics": { "native": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 }, "near": { "recall": 1.0, "precision": 0.029411764705882353, "f1": 0.05714285714285715 }, "novel-relevance": { "recall": 1.0, "precision": 0.029411764705882353, "f1": 0.05714285714285715 }, "novel-information": { "recall": 1.0, "precision": 0.029411764705882353, "f1": 0.05714285714285715 } }, "new_source_lines": { "novel-relevance": 155, "novel-information": 155 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "metrics": { "native": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 }, "near": { "recall": 0.6875, "precision": 0.08782435129740519, "f1": 0.15575221238938053 }, "novel-relevance": { "recall": 0.546875, "precision": 0.07056451612903226, "f1": 0.12500000000000003 }, "novel-information": { "recall": 0.546875, "precision": 0.0691699604743083, "f1": 0.12280701754385964 } }, "new_source_lines": { "novel-relevance": 164, "novel-information": 174 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "metrics": { "native": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 }, "near": { "recall": 0.4479768786127168, "precision": 0.16630901287553648, "f1": 0.24256651017214398 }, "novel-relevance": { "recall": 0.5433526011560693, "precision": 0.20546448087431693, "f1": 0.29817605075337034 }, "novel-information": { "recall": 0.47398843930635837, "precision": 0.1796276013143483, "f1": 0.26052422557585386 } }, "new_source_lines": { "novel-relevance": 157, "novel-information": 155 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "metrics": { "native": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 }, "near": { "recall": 0.2545955882352941, "precision": 0.5326923076923077, "f1": 0.3445273631840796 }, "novel-relevance": { "recall": 0.27941176470588236, "precision": 0.5714285714285714, "f1": 0.3753086419753086 }, "novel-information": { "recall": 0.27941176470588236, "precision": 0.5714285714285714, "f1": 0.3753086419753086 } }, "new_source_lines": { "novel-relevance": 191, "novel-information": 191 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "metrics": { "native": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 }, "near": { "recall": 0.5901639344262295, "precision": 0.21686746987951808, "f1": 0.31718061674008813 }, "novel-relevance": { "recall": 0.5901639344262295, "precision": 0.2302771855010661, "f1": 0.3312883435582822 }, "novel-information": { "recall": 0.5901639344262295, "precision": 0.22268041237113403, "f1": 0.32335329341317365 } }, "new_source_lines": { "novel-relevance": 155, "novel-information": 171 } } ], "macro": { "native": { "recall": 0.5839005987960625, "precision": 0.2990485575256086, "f1": 0.31965726996678134 }, "near": { "recall": 0.6218762428145207, "precision": 0.23936746088657, "f1": 0.291662624931398 }, "novel-relevance": { "recall": 0.6297173213631493, "precision": 0.25078959975263765, "f1": 0.30173353115139157 }, "novel-information": { "recall": 0.6148166075253334, "precision": 0.2398714869637738, "f1": 0.29041736219476433 } }, "selected_mode": "novel-relevance", "selected_mode_vs_near_recall": { "mean_difference": 0.007841078548628576, "exploratory_repository_paired_95_interval": [ -0.018024078410891833, 0.03325954730178231 ] }, "gates": { "all24_retained": true, "all_responses_source_and_models_verified": true, "known_usage": true, "selected_jev_recall_above_near_by_one_point": false, "within_reservation": true }, "development_ranking_screen_passed": false, "new_jev_api_cost_usd": 0.27171501, "new_decision_requests": 558, "main_agent_sessions": 0, "repair_calls": 0, "independent_gain_established": false }, "late_callable_source_opportunity": { "scope": "New zero-API late-envelope development opportunity, all24 native prefixes fixed. Preserve every failed first-read and late-Jev gate. Neither a real model-input delivery nor independent quality trial.", "protocol_sha256": "04dbd71bd77955513dff867aee7dc6bde184004c9c715e2131f144ef5ddbd108", "source_replay_sha256": "adc1ff0fd0dce5cbd8901721ee40da7e42ffe24ac4b525518fb30b4590720f3a", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "metrics": { "native": { "recall": 0.38589211618257263, "precision": 0.4189189189189189, "f1": 0.4017278617710583 }, "physical2048": { "recall": 0.5020746887966805, "precision": 0.3002481389578164, "f1": 0.3757763975155279 }, "physical4096": { "recall": 0.5518672199170125, "precision": 0.2345679012345679, "f1": 0.3292079207920792 }, "callable2048": { "recall": 0.6016597510373444, "precision": 0.41076487252124644, "f1": 0.4882154882154882 }, "callable4096": { "recall": 0.6016597510373444, "precision": 0.41076487252124644, "f1": 0.4882154882154882 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2038, "physical4096": 4091, "callable2048": 1807, "callable4096": 1807 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "metrics": { "native": { "recall": 0.2653061224489796, "precision": 0.09154929577464789, "f1": 0.13612565445026178 }, "physical2048": { "recall": 0.33163265306122447, "precision": 0.08701472556894244, "f1": 0.1378579003181336 }, "physical4096": { "recall": 0.33163265306122447, "precision": 0.06989247311827956, "f1": 0.11545293072824155 }, "callable2048": { "recall": 0.39285714285714285, "precision": 0.1032171581769437, "f1": 0.16348195329087048 }, "callable4096": { "recall": 0.39285714285714285, "precision": 0.08226495726495726, "f1": 0.13604240282685512 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2044, "physical4096": 4084, "callable2048": 2047, "callable4096": 4096 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "metrics": { "native": { "recall": 0.5273922571219869, "precision": 0.40448179271708684, "f1": 0.4578313253012048 }, "physical2048": { "recall": 0.5434623813002192, "precision": 0.3803680981595092, "f1": 0.44751879699248126 }, "physical4096": { "recall": 0.5434623813002192, "precision": 0.34572490706319703, "f1": 0.42260721385969896 }, "callable2048": { "recall": 0.5310445580715851, "precision": 0.36884830035514965, "f1": 0.4353293413173653 }, "callable4096": { "recall": 0.5310445580715851, "precision": 0.32910819375282935, "f1": 0.4063722750139743 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2045, "physical4096": 4095, "callable2048": 2047, "callable4096": 4083 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "metrics": { "native": { "recall": 0.5362694300518135, "precision": 0.5640326975476839, "f1": 0.5498007968127491 }, "physical2048": { "recall": 0.5544041450777202, "precision": 0.40150093808630394, "f1": 0.46572361262241563 }, "physical4096": { "recall": 0.5544041450777202, "precision": 0.30440967283072545, "f1": 0.3930211202938475 }, "callable2048": { "recall": 0.7487046632124352, "precision": 0.6422222222222222, "f1": 0.6913875598086124 }, "callable4096": { "recall": 0.9430051813471503, "precision": 0.6488413547237076, "f1": 0.7687434002111933 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2035, "physical4096": 4096, "callable2048": 2047, "callable4096": 4095 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "metrics": { "native": { "recall": 0.375, "precision": 0.694980694980695, "f1": 0.4871447902571043 }, "physical2048": { "recall": 0.4270833333333333, "precision": 0.46485260770975056, "f1": 0.4451682953311618 }, "physical4096": { "recall": 0.4479166666666667, "precision": 0.35537190082644626, "f1": 0.39631336405529954 }, "callable2048": { "recall": 0.5270833333333333, "precision": 0.6246913580246913, "f1": 0.5717514124293784 }, "callable4096": { "recall": 0.5333333333333333, "precision": 0.44912280701754387, "f1": 0.4876190476190477 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2040, "physical4096": 4094, "callable2048": 2048, "callable4096": 4093 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "metrics": { "native": { "recall": 0.15100316789862725, "precision": 0.23442622950819672, "f1": 0.18368657675016056 }, "physical2048": { "recall": 0.15100316789862725, "precision": 0.17987421383647798, "f1": 0.16417910447761194 }, "physical4096": { "recall": 0.23231256599788808, "precision": 0.23036649214659685, "f1": 0.23133543638275497 }, "callable2048": { "recall": 0.15945089757127773, "precision": 0.20214190093708165, "f1": 0.1782762691853601 }, "callable4096": { "recall": 0.15945089757127773, "precision": 0.16538882803943045, "f1": 0.16236559139784948 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2043, "physical4096": 4091, "callable2048": 2044, "callable4096": 4077 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "metrics": { "native": { "recall": 0.3648881239242685, "precision": 0.1973929236499069, "f1": 0.25619335347432026 }, "physical2048": { "recall": 0.3648881239242685, "precision": 0.17462932454695224, "f1": 0.23621169916434542 }, "physical4096": { "recall": 0.3648881239242685, "precision": 0.15657311669128507, "f1": 0.2191214470284238 }, "callable2048": { "recall": 0.3648881239242685, "precision": 0.17362817362817362, "f1": 0.23529411764705882 }, "callable4096": { "recall": 0.3648881239242685, "precision": 0.15599705665930833, "f1": 0.21855670103092786 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2044, "physical4096": 4093, "callable2048": 2040, "callable4096": 4088 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "metrics": { "native": { "recall": 0.6576576576576577, "precision": 0.1456117021276596, "f1": 0.23843222645617856 }, "physical2048": { "recall": 0.6756756756756757, "precision": 0.13529765484064943, "f1": 0.22545090180360722 }, "physical4096": { "recall": 0.6756756756756757, "precision": 0.1232876712328767, "f1": 0.20852641334569041 }, "callable2048": { "recall": 0.6576576576576577, "precision": 0.13168971737823212, "f1": 0.21943887775551105 }, "callable4096": { "recall": 0.6576576576576577, "precision": 0.11799568965517242, "f1": 0.20009136592051166 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2038, "physical4096": 4092, "callable2048": 2037, "callable4096": 4092 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "metrics": { "native": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "physical2048": { "recall": 1.0, "precision": 0.0560303893637227, "f1": 0.10611510791366907 }, "physical4096": { "recall": 1.0, "precision": 0.049248747913188645, "f1": 0.09387430389817024 }, "callable2048": { "recall": 1.0, "precision": 0.05624404194470925, "f1": 0.10649819494584839 }, "callable4096": { "recall": 1.0, "precision": 0.053057553956834536, "f1": 0.10076857386848848 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2041, "physical4096": 4089, "callable2048": 2039, "callable4096": 3009 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "metrics": { "native": { "recall": 1.0, "precision": 0.21515151515151515, "f1": 0.35411471321695764 }, "physical2048": { "recall": 1.0, "precision": 0.1362763915547025, "f1": 0.23986486486486489 }, "physical4096": { "recall": 1.0, "precision": 0.09712722298221614, "f1": 0.1770573566084788 }, "callable2048": { "recall": 1.0, "precision": 0.20579710144927535, "f1": 0.3413461538461538 }, "callable4096": { "recall": 1.0, "precision": 0.20579710144927535, "f1": 0.3413461538461538 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2045, "physical4096": 4084, "callable2048": 285, "callable4096": 285 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "metrics": { "native": { "recall": 0.8706140350877193, "precision": 0.5416098226466576, "f1": 0.6677880571909168 }, "physical2048": { "recall": 1.0, "precision": 0.5129358830146231, "f1": 0.6780669144981413 }, "physical4096": { "recall": 1.0, "precision": 0.4351145038167939, "f1": 0.6063829787234043 }, "callable2048": { "recall": 0.8947368421052632, "precision": 0.49877750611246946, "f1": 0.6405023547880692 }, "callable4096": { "recall": 0.8947368421052632, "precision": 0.49877750611246946, "f1": 0.6405023547880692 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2044, "physical4096": 4094, "callable2048": 1041, "callable4096": 1041 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "metrics": { "native": { "recall": 1.0, "precision": 0.04246284501061571, "f1": 0.0814663951120163 }, "physical2048": { "recall": 1.0, "precision": 0.031007751937984496, "f1": 0.06015037593984962 }, "physical4096": { "recall": 1.0, "precision": 0.025157232704402517, "f1": 0.049079754601226995 }, "callable2048": { "recall": 1.0, "precision": 0.04081632653061224, "f1": 0.07843137254901959 }, "callable4096": { "recall": 1.0, "precision": 0.04081632653061224, "f1": 0.07843137254901959 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2048, "physical4096": 4075, "callable2048": 353, "callable4096": 353 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "metrics": { "native": { "recall": 0.2608695652173913, "precision": 0.41317365269461076, "f1": 0.3198146002317497 }, "physical2048": { "recall": 0.3686200378071834, "precision": 0.38922155688622756, "f1": 0.3786407766990292 }, "physical4096": { "recall": 0.387523629489603, "precision": 0.30460624071322434, "f1": 0.3410981697171381 }, "callable2048": { "recall": 0.3383742911153119, "precision": 0.3403041825095057, "f1": 0.33933649289099527 }, "callable4096": { "recall": 0.3383742911153119, "precision": 0.3403041825095057, "f1": 0.33933649289099527 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2040, "physical4096": 4094, "callable2048": 2034, "callable4096": 2034 } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "metrics": { "native": { "recall": 0.5, "precision": 0.15673981191222572, "f1": 0.23866348448687352 }, "physical2048": { "recall": 0.5, "precision": 0.10395010395010396, "f1": 0.17211703958691912 }, "physical4096": { "recall": 0.5, "precision": 0.08210180623973727, "f1": 0.14104372355430184 }, "callable2048": { "recall": 0.5, "precision": 0.13054830287206268, "f1": 0.20703933747412012 }, "callable4096": { "recall": 0.5, "precision": 0.13054830287206268, "f1": 0.20703933747412012 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2031, "physical4096": 4079, "callable2048": 891, "callable4096": 891 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "metrics": { "native": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 }, "physical2048": { "recall": 0.18700787401574803, "precision": 0.11728395061728394, "f1": 0.1441578148710167 }, "physical4096": { "recall": 0.18700787401574803, "precision": 0.1074660633484163, "f1": 0.13649425287356323 }, "callable2048": { "recall": 0.18700787401574803, "precision": 0.11677934849416104, "f1": 0.14377601210745364 }, "callable4096": { "recall": 0.18700787401574803, "precision": 0.10777084515031196, "f1": 0.13673983447283194 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2045, "physical4096": 4095, "callable2048": 2039, "callable4096": 4087 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "metrics": { "native": { "recall": 0.782608695652174, "precision": 0.05042016806722689, "f1": 0.09473684210526315 }, "physical2048": { "recall": 0.782608695652174, "precision": 0.03495145631067961, "f1": 0.06691449814126393 }, "physical4096": { "recall": 1.0, "precision": 0.032670454545454544, "f1": 0.06327372764786794 }, "callable2048": { "recall": 0.782608695652174, "precision": 0.034026465028355386, "f1": 0.06521739130434782 }, "callable4096": { "recall": 0.782608695652174, "precision": 0.026200873362445413, "f1": 0.05070422535211268 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2036, "physical4096": 4091, "callable2048": 2035, "callable4096": 4096 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "metrics": { "native": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 }, "physical2048": { "recall": 0.8773234200743495, "precision": 0.30569948186528495, "f1": 0.45341018251681076 }, "physical4096": { "recall": 0.8773234200743495, "precision": 0.251063829787234, "f1": 0.3904052936311001 }, "callable2048": { "recall": 0.9033457249070632, "precision": 0.3103448275862069, "f1": 0.461977186311787 }, "callable4096": { "recall": 0.9033457249070632, "precision": 0.2793103448275862, "f1": 0.4266900790166814 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2047, "physical4096": 4092, "callable2048": 2048, "callable4096": 3006 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "metrics": { "native": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 }, "physical2048": { "recall": 0.6807760141093474, "precision": 0.12659888488028861, "f1": 0.2134955752212389 }, "physical4096": { "recall": 0.691358024691358, "precision": 0.1206896551724138, "f1": 0.20550458715596331 }, "callable2048": { "recall": 0.6807760141093474, "precision": 0.12610258085592943, "f1": 0.2127894156560088 }, "callable4096": { "recall": 0.6807760141093474, "precision": 0.1188423645320197, "f1": 0.20235910878112712 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2045, "physical4096": 4096, "callable2048": 2048, "callable4096": 4090 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "metrics": { "native": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 }, "physical2048": { "recall": 0.9982332155477032, "precision": 0.773972602739726, "f1": 0.8719135802469136 }, "physical4096": { "recall": 0.9982332155477032, "precision": 0.6075268817204301, "f1": 0.7553475935828877 }, "callable2048": { "recall": 0.9699646643109541, "precision": 0.8941368078175895, "f1": 0.9305084745762711 }, "callable4096": { "recall": 0.9699646643109541, "precision": 0.8941368078175895, "f1": 0.9305084745762711 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2044, "physical4096": 4091, "callable2048": 730, "callable4096": 730 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "metrics": { "native": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 }, "physical2048": { "recall": 1.0, "precision": 0.029411764705882353, "f1": 0.05714285714285715 }, "physical4096": { "recall": 1.0, "precision": 0.026923076923076925, "f1": 0.05243445692883896 }, "callable2048": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 }, "callable4096": { "recall": 1.0, "precision": 0.03636363636363636, "f1": 0.07017543859649122 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2043, "physical4096": 2536, "callable2048": 931, "callable4096": 931 } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "metrics": { "native": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 }, "physical2048": { "recall": 0.6875, "precision": 0.08782435129740519, "f1": 0.15575221238938053 }, "physical4096": { "recall": 1.0, "precision": 0.0935672514619883, "f1": 0.1711229946524064 }, "callable2048": { "recall": 0.546875, "precision": 0.09259259259259259, "f1": 0.1583710407239819 }, "callable4096": { "recall": 0.546875, "precision": 0.09259259259259259, "f1": 0.1583710407239819 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2047, "physical4096": 4094, "callable2048": 682, "callable4096": 682 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "metrics": { "native": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 }, "physical2048": { "recall": 0.4479768786127168, "precision": 0.16630901287553648, "f1": 0.24256651017214398 }, "physical4096": { "recall": 0.4479768786127168, "precision": 0.1388888888888889, "f1": 0.21203830369357046 }, "callable2048": { "recall": 0.2745664739884393, "precision": 0.10237068965517242, "f1": 0.14913657770800628 }, "callable4096": { "recall": 0.37572254335260113, "precision": 0.11937557392102846, "f1": 0.18118466898954702 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2035, "physical4096": 4087, "callable2048": 2040, "callable4096": 4092 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "metrics": { "native": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 }, "physical2048": { "recall": 0.2545955882352941, "precision": 0.5326923076923077, "f1": 0.3445273631840796 }, "physical4096": { "recall": 0.27389705882352944, "precision": 0.4226950354609929, "f1": 0.33240379252649194 }, "callable2048": { "recall": 0.22058823529411764, "precision": 0.5620608899297423, "f1": 0.31683168316831684 }, "callable4096": { "recall": 0.22058823529411764, "precision": 0.5620608899297423, "f1": 0.31683168316831684 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2048, "physical4096": 4085, "callable2048": 905, "callable4096": 905 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "metrics": { "native": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 }, "physical2048": { "recall": 0.5901639344262295, "precision": 0.21686746987951808, "f1": 0.31718061674008813 }, "physical4096": { "recall": 0.5901639344262295, "precision": 0.16071428571428573, "f1": 0.25263157894736843 }, "callable2048": { "recall": 0.5901639344262295, "precision": 0.22736842105263158, "f1": 0.3282674772036474 }, "callable4096": { "recall": 0.6174863387978142, "precision": 0.17465224111282843, "f1": 0.272289156626506 } }, "eligible_callable": true, "eligible_physical": true, "late_context_tokens": { "physical2048": 2032, "physical4096": 4091, "callable2048": 2045, "callable4096": 4087 } } ], "macro": { "native": { "recall": 0.5839005987960625, "precision": 0.2990485575256086, "f1": 0.31965726996678134 }, "physical2048": { "recall": 0.6218762428145207, "precision": 0.23936746088657, "f1": 0.291662624931398 }, "callable2048": { "recall": 0.6196814115662372, "precision": 0.26799322600159975, "f1": 0.31389081764584015 }, "physical4096": { "recall": 0.6523184778042465, "precision": 0.19898980468902994, "f1": 0.26232411313453396 }, "callable4096": { "recall": 0.6333909528941731, "precision": 0.251670454278114, "f1": 0.30505351116485674 } }, "physical4096_gain": 0.06841787900818397, "callable4096_gain": 0.049490354098110645, "callable_minus_physical4096": -0.018927524910073323, "physical_delivery_opportunity_gates": { "all24_retained": true, "exact_source": true, "gain_five_points": true, "positive_in8_tasks": true, "eligible18": true }, "callable_targeting_opportunity_gates": { "all24_retained": true, "exact_source": true, "gain_five_points": false, "one_point_over_matched_physical": false, "eligible18": true }, "selected_candidate_for_possible_input_study": "physical4096", "new_api_cost_usd": 0, "new_model_calls": 0, "independent_gain_established": false, "zero_call_preflight_correction": { "scope": "Zero-call Python representation correction before any source/label comparison. Compare point sets rather than tuple-versus-JSON-list arrays. No change to source policy, budgets, selected tasks or gates.", "old_driver_sha256": "89d82c57722e817ed561901050a9df56a49d2fd88d579e4d3c0789d0ee83d646", "new_driver_sha256": "b5e93ecd83ab9f43539c55779c88a702f075869e125f7eedb8022f4182daa899", "original_protocol_sha256": "9c43ebb95a0db3344834de3e9abd9d880aae25c248df6fff98277d58dc0e1bc0", "new_api_cost_usd": 0, "new_model_calls": 0 } }, "late_packet_input_first_attempt": { "scope": "Prospective input-delivery mechanism check, not fresh adaptive-agent discovery, independent quality, Pass@1 or a leaderboard result. Native prefix stays identical in each paired arm. Late source is chosen by the separately passed physical4K opportunity, not by the failed Jev ranking screen.", "protocol_sha256": "cfe51d9c65987fc13aa1d5613cb3f0d7f118d9821d663471d928d8112d53cce0", "runtime_commit": "539e6a6bb7f70c53afcee9037bf6b43695922075", "primary_cumulative": { "native_packet": { "recall": 0.42871498145133813, "precision": 0.36974567623958443, "f1": 0.38637140958381844, "official_valid": 4 }, "late_physical4096": { "recall": 0.47020875738494816, "precision": 0.32386550403197856, "f1": 0.3684456476149993, "official_valid": 4 } }, "secondary_final32k_packet": { "native_packet": { "recall": 0.09647302904564316, "precision": 0.10472972972972973, "f1": 0.10043196544276457, "official_valid": 1 }, "late_physical4096": { "recall": 0.13796680497925312, "precision": 0.058849557522123896, "f1": 0.08250620347394541, "official_valid": 1 } }, "gates": { "all8_sessions_normal": false, "all_actual_inputs_version_bound_and_confirmed": false, "known_usage": true, "all_official_scores_valid": false, "runtime_source_driver_receipts_exact": true, "all_selected_positions_retained": true, "within_reservation": true }, "mechanism_gate_passed": false, "new_api_sdk_estimate_usd": 0.8501327999999999, "new_sdk_sessions": 4, "reused_prefix_sdk_estimate_usd": 0.7716366, "allocated_pipeline_sdk_usd": { "native_packet": 1.1803831, "late_physical4096": 1.2130229000000001 }, "environments": [ { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" } ], "cost_scope": "Incremental experiment spending excludes reused prefixes. Each pipeline arm allocates its full historical prefix cost; those allocations are not additional API spending. This is a shared-prefix mechanism check, not two fresh adaptive-agent runs.", "repair_calls": 0, "independent_gain_established": false, "prior_failed_gates_unchanged": true, "analysis_sha256": "3d49a09a00018bda308dce66f4054205fcc42d800b1eda41dcf37b6c1dae8688", "implementation_commit": "539e6a6bb7f70c53afcee9037bf6b43695922075", "source_packet_sha256": "ab3b64c05b3ecc670ec1624c36af2d33b2b06e9cd9a9b87f474cc7318c26416b", "selected_tasks": 4, "planned_sessions": 8, "attempted_sessions": 4, "status_counts": { "error": 2, "ok": 2 }, "not_run_sessions": 4, "raw_attempts_preserved": true }, "late_packet_input_followup": { "scope": "Separate prospective packet input mechanism check with explicit thinking disabled and at most120-word review. Same model/source packets and case budgets as closed first attempt. Retain the original two successful/two budget-error/four not-run positions and its failed gate. No repair or fresh adaptive/independent/leaderboard claim.", "protocol_sha256": "4e1b0193accf1d176e2938ad419ba3cd2a8682b3cba96a6d46c55f8a2311e3ef", "runtime_commit": "c141a2b7348b43c3fb39deadc6b3d0d43f65d473", "primary_cumulative": { "native_packet": { "recall": 0.42871498145133813, "precision": 0.36974567623958443, "f1": 0.38637140958381844, "official_valid": 4 }, "late_physical4096": { "recall": 0.4940660896349624, "precision": 0.2391237670550279, "f1": 0.31527428214593783, "official_valid": 4 } }, "secondary_final32k_packet": { "native_packet": { "recall": 0.42871498145133813, "precision": 0.37022799199187983, "f1": 0.38663509246571426, "official_valid": 4 }, "late_physical4096": { "recall": 0.4940660896349624, "precision": 0.23929183871900586, "f1": 0.31542040903571045, "official_valid": 4 } }, "gates": { "all8_sessions_normal": true, "all_actual_inputs_version_bound_and_confirmed": true, "known_usage": true, "all_official_scores_valid": true, "runtime_source_driver_receipts_exact": true, "all_selected_positions_retained": true, "within_reservation": true }, "mechanism_gate_passed": false, "source_delivery_receipt_gate_passed": true, "mode_conformance_gate_passed": false, "new_api_sdk_estimate_usd": 0.9106428999999999, "new_sdk_sessions": 8, "reused_prefix_sdk_estimate_usd": 0.7716366, "allocated_pipeline_sdk_usd": { "native_packet": 1.1588559, "late_physical4096": 1.2950602000000002 }, "environments": [ { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" }, { "sdk_version": "0.2.165", "cli_version": "2.1.301 (Claude Code)", "model_requested": "claude-opus-5-5" } ], "cost_scope": "Incremental experiment spending excludes reused prefixes. Each pipeline arm allocates its full historical prefix cost; those allocations are not additional API spending. This is a shared-prefix mechanism check, not two fresh adaptive-agent runs.", "repair_calls": 0, "independent_gain_established": false, "prior_failed_gates_unchanged": true, "aggregate_mechanism_sdk_usd": 1.7607757, "analysis_sha256": "34c492554cb02e084da2f3a084ee0b41920e3114708a8fa7012bf44fd38629d4", "implementation_commit": "c141a2b7348b43c3fb39deadc6b3d0d43f65d473", "source_packet_sha256": "ab3b64c05b3ecc670ec1624c36af2d33b2b06e9cd9a9b87f474cc7318c26416b", "selected_tasks": 4, "planned_sessions": 8, "attempted_sessions": 8, "status_counts": { "ok": 8 }, "not_run_sessions": 0, "raw_attempts_preserved": true, "actual_mode_summary": { "requested_thinking": "disabled", "sessions_with_thinking_blocks": 7, "sessions": 8, "sdk_reported_turn_range": [ 1, 3 ], "thought_contents_inspected_or_published": false, "thinking_mode_conformance": false }, "delivery_analysis_protocol_sha256": "5bb2654944fa0e707ae72eba06a4d239c840e0490c0c67adc560eccb29808106", "mode_conformance_audit_sha256": "4c1eb6634a34f4884c39b05f0d838ee41d9af91dc13b211c43898b0dc2e3f39c" }, "late_packet_input_next_gate": "Only a passing complete24 engineering gate permits freezing a new independent source-only protocol on untouched IDs. Both old32 cohorts remain sealed. No repair/full500 launch automatically; verified500 remains a separate measured budget decision.", "late_packet_source_compiler_parity": { "scope": "Unpaid construction parity on already disclosed development source, not new source delivery or independent quality. No gold or reference patch fields loaded.", "producer_sha256": "5dbd7fc8578f87e470615079ab1776953057526a5540d0e848d61fb65b5c2424", "drivers_sha256": "aa78abce8011a03eed6bd7ab7eaa88257ddd261d5a334164277769a0cc6bad13", "source_replay_sha256": "adc1ff0fd0dce5cbd8901721ee40da7e42ffe24ac4b525518fb30b4590720f3a", "metadata_replay_sha256": "d1e3b2991035df0b04ea98588ddf40164d69dec9b44e8b843eb471f16e78f5ab", "prior_input_packets_sha256": "ab3b64c05b3ecc670ec1624c36af2d33b2b06e9cd9a9b87f474cc7318c26416b", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": true, "context_tokens": { "native_packet": 2434, "late_physical4096": 6514 }, "extra_tokens": 4091, "candidate_windows": 34 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": true, "context_tokens": { "native_packet": 8012, "late_physical4096": 12085 }, "extra_tokens": 4084, "candidate_windows": 161 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": true, "context_tokens": { "native_packet": 20416, "late_physical4096": 24481 }, "extra_tokens": 4095, "candidate_windows": 200 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": true, "context_tokens": { "native_packet": 5203, "late_physical4096": 9289 }, "extra_tokens": 4096, "candidate_windows": 25 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 3593, "late_physical4096": 7659 }, "extra_tokens": 4094, "candidate_windows": 48 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 7741, "late_physical4096": 11830 }, "extra_tokens": 4091, "candidate_windows": 93 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 15726, "late_physical4096": 19809 }, "extra_tokens": 4093, "candidate_windows": 200 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 17535, "late_physical4096": 21620 }, "extra_tokens": 4092, "candidate_windows": 142 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 13838, "late_physical4096": 17912 }, "extra_tokens": 4089, "candidate_windows": 120 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 4175, "late_physical4096": 8253 }, "extra_tokens": 4084, "candidate_windows": 14 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 10142, "late_physical4096": 14234 }, "extra_tokens": 4094, "candidate_windows": 137 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 5960, "late_physical4096": 10029 }, "extra_tokens": 4075, "candidate_windows": 36 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 4183, "late_physical4096": 8251 }, "extra_tokens": 4094, "candidate_windows": 200 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 3898, "late_physical4096": 7967 }, "extra_tokens": 4079, "candidate_windows": 200 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 22266, "late_physical4096": 26355 }, "extra_tokens": 4095, "candidate_windows": 200 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 4935, "late_physical4096": 9014 }, "extra_tokens": 4091, "candidate_windows": 81 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 8305, "late_physical4096": 12375 }, "extra_tokens": 4092, "candidate_windows": 51 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 32749, "late_physical4096": 32766 }, "extra_tokens": 4096, "candidate_windows": 200 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 6571, "late_physical4096": 10644 }, "extra_tokens": 4091, "candidate_windows": 10 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 4138, "late_physical4096": 6672 }, "extra_tokens": 2536, "candidate_windows": 4 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 4406, "late_physical4096": 8484 }, "extra_tokens": 4094, "candidate_windows": 141 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 8925, "late_physical4096": 12982 }, "extra_tokens": 4087, "candidate_windows": 181 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 3651, "late_physical4096": 7722 }, "extra_tokens": 4085, "candidate_windows": 37 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "extra_packet_byte_identical": true, "native_packet_byte_identical": true, "prior_actual_input_pair_byte_identical": null, "context_tokens": { "native_packet": 4113, "late_physical4096": 8146 }, "extra_tokens": 4091, "candidate_windows": 178 } ], "all24_extra_and_native_packets_byte_identical": true, "all4_prior_actual_packet_pairs_byte_identical": true, "new_model_calls": 0, "new_api_cost_usd": 0, "implementation_commit": "1aa5bac1fc025b1e615fbece6ac8c4b442a4b8e8", "receipt_sha256": "5fbc4a7b2b129eeeaa7a598cbc52f67b7930ef69c66aa611eccea7c38183c24e", "focused_tests": 96 }, "late_packet_input_remaining20_development": { "scope": "Complete actual source-input engineering validation on remaining20 disclosed development tasks, then report all24 together. Native investigation is shared; not fresh paired adaptive agents, independent quality, repair or leaderboard evidence. Existing source opportunity was already scored, so this is not a new confirmatory quality test. Preserve every original failed mechanism/mode/quality gate.", "protocol_sha256": "13c82732a9cc6b3c7ff30e7d2b98fe88c99c997829472c72b771f63244d1f4a3", "runtime_commit": "1aa5bac1fc025b1e615fbece6ac8c4b442a4b8e8", "primary_cumulative": { "native_packet": { "recall": 0.6149377222650073, "precision": 0.28490913378281346, "f1": 0.30631444204337394, "official_valid": 20 }, "late_physical4096": { "recall": 0.6471269751224085, "precision": 0.24177135070181716, "f1": 0.28057417923243533, "official_valid": 20 } }, "secondary_final32k_packet": { "native_packet": { "recall": 0.3574824559726793, "precision": 0.1442215692437978, "f1": 0.15977357584138788, "official_valid": 12 }, "late_physical4096": { "recall": 0.38032131512929296, "precision": 0.09472311995942258, "f1": 0.12646077109797305, "official_valid": 11 } }, "gates": { "all40_sessions_normal": false, "all_actual_inputs_version_bound_and_confirmed": false, "known_usage": true, "all_official_scores_valid": false, "runtime_source_driver_receipts_exact": true, "all_selected_positions_retained": true, "within_reservation": true }, "source_delivery_receipt_gate_passed": false, "new_api_sdk_estimate_usd": 2.8706633000000004, "new_sdk_sessions": 24, "reused_prefix_sdk_estimate_usd": 3.6594382000000008, "allocated_pipeline_sdk_usd": { "native_packet": 4.885899999999999, "late_physical4096": 5.3036397 }, "repair_calls": 0, "independent_gain_established": false, "prior_failed_gates_unchanged": true, "analysis_sha256": "480a741d7a802eec0c2a9350729576b866b80e7830b4fec246e3d5ed913935c6", "actual_thinking_sessions": 21, "sdk_reported_turn_range": [ 1, 3 ], "selected_positions": 20, "normal_sessions": 23, "error_sessions": 1, "not_run_sessions": 16, "stopped_reason": "Declared source/model/route/input/status/usage stop rule", "source_packet_sha256": "f333548ecc223939c0cfc1030b85816bf6cb771d4cd1a512432797477f385f4e" }, "late_packet_input_combined24_development": { "scope": "Complete24 disclosed development cases; shared actual native prefixes plus actually confirmed source packets. Component source delivery, not fresh paired adaptive agents or independent quality. The source opportunity was already scored; intervals are descriptive. Preserve both old32 sealed cohorts and every prior failed gate.", "protocol_sha256": "13c82732a9cc6b3c7ff30e7d2b98fe88c99c997829472c72b771f63244d1f4a3", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "repo": "pylint-dev/pylint", "repo_url": "https://github.com/pylint-dev/pylint.git", "language": "python", "metrics": { "native_packet": { "recall": 0.38589211618257263, "precision": 0.4189189189189189, "f1": 0.4017278617710583 }, "late_physical4096": { "recall": 0.5518672199170125, "precision": 0.23539823008849559, "f1": 0.33002481389578164 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "metrics": { "native_packet": { "recall": 0.2653061224489796, "precision": 0.09154929577464789, "f1": 0.13612565445026178 }, "late_physical4096": { "recall": 0.32653061224489793, "precision": 0.06896551724137931, "f1": 0.11387900355871886 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "metrics": { "native_packet": { "recall": 0.5273922571219869, "precision": 0.40448179271708684, "f1": 0.4578313253012048 }, "late_physical4096": { "recall": 0.5434623813002192, "precision": 0.34685314685314683, "f1": 0.4234490608992601 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "repo": "fmtlib/fmt", "repo_url": "https://github.com/fmtlib/fmt.git", "language": "cpp", "metrics": { "native_packet": { "recall": 0.5362694300518135, "precision": 0.5640326975476839, "f1": 0.5498007968127491 }, "late_physical4096": { "recall": 0.5544041450777202, "precision": 0.30527817403708984, "f1": 0.39374425022999077 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "repo": "future-architect/vuls", "repo_url": "https://github.com/future-architect/vuls.git", "language": "go", "metrics": { "native_packet": { "recall": 0.375, "precision": 0.694980694980695, "f1": 0.4871447902571043 }, "late_physical4096": { "recall": 0.4479166666666667, "precision": 0.35714285714285715, "f1": 0.39741219963031427 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "metrics": { "native_packet": { "recall": 0.15100316789862725, "precision": 0.23442622950819672, "f1": 0.18368657675016056 }, "late_physical4096": { "recall": 0.23231256599788808, "precision": 0.23036649214659685, "f1": 0.23133543638275497 } } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "metrics": { "native_packet": { "recall": 0.3648881239242685, "precision": 0.1973929236499069, "f1": 0.25619335347432026 }, "late_physical4096": { "recall": 0.3648881239242685, "precision": 0.15680473372781065, "f1": 0.21934816347646144 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "metrics": { "native_packet": { "recall": 0.6576576576576577, "precision": 0.1456117021276596, "f1": 0.23843222645617856 }, "late_physical4096": { "recall": 0.6756756756756757, "precision": 0.12335526315789473, "f1": 0.2086230876216968 } } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "metrics": { "native_packet": { "recall": 1.0, "precision": 0.06455142231947483, "f1": 0.12127440904419322 }, "late_physical4096": { "recall": 1.0, "precision": 0.049372384937238493, "f1": 0.09409888357256778 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "metrics": { "native_packet": { "recall": 1.0, "precision": 0.21515151515151515, "f1": 0.35411471321695764 }, "late_physical4096": { "recall": 1.0, "precision": 0.09726027397260274, "f1": 0.17727840199750314 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "repo": "jqlang/jq", "repo_url": "https://github.com/jqlang/jq.git", "language": "c", "metrics": { "native_packet": { "recall": 0.8706140350877193, "precision": 0.5416098226466576, "f1": 0.6677880571909168 }, "late_physical4096": { "recall": 1.0, "precision": 0.4351145038167939, "f1": 0.6063829787234043 } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "repo": "darkreader/darkreader", "repo_url": "https://github.com/darkreader/darkreader.git", "language": "typescript", "metrics": { "native_packet": { "recall": 1.0, "precision": 0.04246284501061571, "f1": 0.0814663951120163 }, "late_physical4096": { "recall": 1.0, "precision": 0.02518891687657431, "f1": 0.049140049140049144 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "repo": "nlohmann/json", "repo_url": "https://github.com/nlohmann/json.git", "language": "cpp", "metrics": { "native_packet": { "recall": 0.2608695652173913, "precision": 0.41317365269461076, "f1": 0.3198146002317497 }, "late_physical4096": { "recall": 0.3856332703213611, "precision": 0.30402384500745155, "f1": 0.33999999999999997 } } }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "repo": "catchorg/Catch2", "repo_url": "https://github.com/catchorg/Catch2.git", "language": "cpp", "metrics": { "native_packet": { "recall": 0.5, "precision": 0.15673981191222572, "f1": 0.23866348448687352 }, "late_physical4096": { "recall": 0.5, "precision": 0.08237232289950576, "f1": 0.1414427157001414 } } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "metrics": { "native_packet": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 }, "late_physical4096": { "recall": 0.18700787401574803, "precision": 0.12666666666666668, "f1": 0.15103338632750396 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "repo": "tokio-rs/bytes", "repo_url": "https://github.com/tokio-rs/bytes.git", "language": "rust", "metrics": { "native_packet": { "recall": 0.782608695652174, "precision": 0.05042016806722689, "f1": 0.09473684210526315 }, "late_physical4096": { "recall": 1.0, "precision": 0.03276353276353276, "f1": 0.06344827586206897 } } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "repo": "serverless/serverless", "repo_url": "https://github.com/serverless/serverless.git", "language": "javascript", "metrics": { "native_packet": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 }, "late_physical4096": { "recall": 0.8661710037174721, "precision": 0.38576158940397354, "f1": 0.5337915234822451 } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "repo": "nushell/nushell", "repo_url": "https://github.com/nushell/nushell.git", "language": "rust", "metrics": { "native_packet": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 }, "late_physical4096": { "recall": 0.6807760141093474, "precision": 0.13370280568063733, "f1": 0.22350897510133177 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "repo": "google/gson", "repo_url": "https://github.com/google/gson.git", "language": "java", "metrics": { "native_packet": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 }, "late_physical4096": { "recall": 0.9699646643109541, "precision": 0.973404255319149, "f1": 0.9716814159292034 } } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "metrics": { "native_packet": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 }, "late_physical4096": { "recall": 1.0, "precision": 0.04361370716510903, "f1": 0.0835820895522388 } } }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "repo": "alibaba/fastjson2", "repo_url": "https://github.com/alibaba/fastjson2.git", "language": "java", "metrics": { "native_packet": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 }, "late_physical4096": { "recall": 0.546875, "precision": 0.10542168674698796, "f1": 0.17676767676767677 } } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "repo": "flipt-io/flipt", "repo_url": "https://github.com/flipt-io/flipt.git", "language": "go", "metrics": { "native_packet": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 }, "late_physical4096": { "recall": 0.2745664739884393, "precision": 0.12532981530343007, "f1": 0.1721014492753623 } } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "repo": "axios/axios", "repo_url": "https://github.com/axios/axios.git", "language": "javascript", "metrics": { "native_packet": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 }, "late_physical4096": { "recall": 0.22058823529411764, "precision": 0.7038123167155426, "f1": 0.3358992302309307 } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "repo": "pydata/xarray", "repo_url": "https://github.com/pydata/xarray.git", "language": "python", "metrics": { "native_packet": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 }, "late_physical4096": { "recall": 0.5901639344262295, "precision": 0.34394904458598724, "f1": 0.4346076458752515 } } } ], "macro": { "native_packet": { "recall": 0.5839005987960625, "precision": 0.2990485575256086, "f1": 0.31965726996678134 }, "late_physical4096": { "recall": 0.6216168275411674, "precision": 0.24133008676068557, "f1": 0.2863575297180191 } }, "descriptive_bootstrap": { "recall": { "difference": 0.03771622874510494, "descriptive_repository_cluster_95": [ 0.014860570372834518, 0.0650507292305561 ] }, "precision": { "difference": -0.057718470764923036, "descriptive_repository_cluster_95": [ -0.09550488929577373, -0.026249844737664983 ] }, "f1": { "difference": -0.033299740248762255, "descriptive_repository_cluster_95": [ -0.055210142546839716, -0.014017128987312681 ] } }, "repository_clusters": 24, "languages": { "python": 3, "typescript": 3, "c": 3, "cpp": 3, "go": 3, "java": 3, "javascript": 3, "rust": 3 }, "complete_development_source_delivery_gate_passed": false, "prior_four_task_mode_conformance_gate_passed": false, "prior_four_task_overall_mechanism_gate_passed": false, "remaining20_new_api_sdk_usd": 2.8706633000000004, "failed_first_attempt_api_sdk_usd": 0.8501327999999999, "all_packet_input_api_sdk_usd": 4.631439, "new_sdk_sessions": 24, "all_attempted_packet_sdk_sessions": 36, "reused_prefix_sdk_usd": 4.4310748, "allocated_pipeline_sdk_usd": { "native_packet": 6.044755899999998, "late_physical4096": 6.5986999 }, "known_program_api_usage_value_usd": 43.22530093, "new_usage_missing": 0, "actual_thinking_sessions_remaining20": 21, "sdk_reported_turn_range_remaining20": [ 1, 3 ], "repair_calls": 0, "independent_gain_established": false, "next": "If complete source-delivery gate passes, a new independent source-only protocol on untouched IDs may test this materially new late policy; both previous validation cohorts remain sealed. Freeze population and source before any new calls. No repairs or official500 automatically.", "analysis_sha256": "651a69d2248aad6cc35a23d375671adc3901045719a92bac8a907c786f6dc88e", "followup_and_development_review_attempts_api_sdk_usd": 3.7813062000000004, "publication_field_audit_sha256": "0fe41629d034ef6bb088a21c15ee7cce2659c629b7db64a523885122304d4c24" }, "late_packet_direct_messages_development24": { "scope": "New source-input engineering protocol on all24 known development cases. One free token count and one paid Messages HTTP request per packet, with no agent CLI/automatic continuation/retries/fallback. Native investigation is shared, source opportunity was already scored: not new confirmatory quality or fresh paired adaptive agents. Every prior SDK failure/gate/status remains unchanged. No repairs/independent tasks/full500.", "protocol_sha256": "7f97e9a0cc118f8d5a535e9a7256927d05e2a084e50f593c7d72e84ce0f33727", "implementation_commit": "d2bf113623f201ad5a167126909bcee5dcc37ebb", "primary_cumulative": { "native_packet": { "recall": 0.5839005987960625, "precision": 0.2990485575256086, "f1": 0.31965726996678134, "official_valid": 24 }, "late_physical4096": { "recall": 0.6518330878939435, "precision": 0.19943888273772412, "f1": 0.2626349122326203, "official_valid": 24 } }, "secondary_final32k_packet": { "native_packet": { "recall": 0.5838269827772168, "precision": 0.2997097543934261, "f1": 0.32035153312096293, "official_valid": 24 }, "late_physical4096": { "recall": 0.6517594718750978, "precision": 0.20051483291782335, "f1": 0.26410389095769865, "official_valid": 24 } }, "descriptive_bootstrap": { "recall": { "difference": 0.06793248909788097, "descriptive_repository_cluster_95": [ 0.03234937556167766, 0.11360523776170922 ] }, "precision": { "difference": -0.09960967478788449, "descriptive_repository_cluster_95": [ -0.14615678365221785, -0.05850391862549016 ] }, "f1": { "difference": -0.05702235773416103, "descriptive_repository_cluster_95": [ -0.08571706173722854, -0.029896212103703638 ] } }, "repository_clusters": 24, "rows": [ { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "arm": "native_packet", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "status": "ok", "input_confirmed": true, "api_usd": 0.17606, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 38800, "provider_output_tokens": 1043, "packet_cl100k_tokens": 22266, "source_lines_in_confirmed_packet": 1499, "prefix_usd_allocated": 0.4211373, "allocated_pipeline_usd": 0.5971972999999999 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "arm": "late_physical4096", "repo": "facebook/zstd", "repo_url": "https://github.com/facebook/zstd.git", "language": "c", "status": "ok", "input_confirmed": true, "api_usd": 0.199244, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 45641, "provider_output_tokens": 834, "packet_cl100k_tokens": 26355, "source_lines_in_confirmed_packet": 1766, "prefix_usd_allocated": 0.4211373, "allocated_pipeline_usd": 0.6203813 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "arm": "native_packet", "repo": "nushell/nushell", "repo_url": "https://github.com/nushell/nushell.git", "language": "rust", "status": "ok", "input_confirmed": true, "api_usd": 0.232828, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 53302, "provider_output_tokens": 981, "packet_cl100k_tokens": 32749, "source_lines_in_confirmed_packet": 2709, "prefix_usd_allocated": 0.5918497000000001, "allocated_pipeline_usd": 0.8246777000000002 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "arm": "late_physical4096", "repo": "nushell/nushell", "repo_url": "https://github.com/nushell/nushell.git", "language": "rust", "status": "ok", "input_confirmed": true, "api_usd": 0.23284, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 53430, "provider_output_tokens": 956, "packet_cl100k_tokens": 32766, "source_lines_in_confirmed_packet": 2710, "prefix_usd_allocated": 0.5918497000000001, "allocated_pipeline_usd": 0.8246897000000002 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "native_packet", "repo": "pylint-dev/pylint", "repo_url": "https://github.com/pylint-dev/pylint.git", "language": "python", "status": "ok", "input_confirmed": true, "api_usd": 0.027576, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 4219, "provider_output_tokens": 535, "packet_cl100k_tokens": 2434, "source_lines_in_confirmed_packet": 222, "prefix_usd_allocated": 0.0882124, "allocated_pipeline_usd": 0.1157884 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "late_physical4096", "repo": "pylint-dev/pylint", "repo_url": "https://github.com/pylint-dev/pylint.git", "language": "python", "status": "ok", "input_confirmed": true, "api_usd": 0.054044, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 10611, "provider_output_tokens": 580, "packet_cl100k_tokens": 6514, "source_lines_in_confirmed_packet": 565, "prefix_usd_allocated": 0.0882124, "allocated_pipeline_usd": 0.1422564 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "arm": "native_packet", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "status": "ok", "input_confirmed": true, "api_usd": 0.069648, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 12967, "provider_output_tokens": 889, "packet_cl100k_tokens": 8012, "source_lines_in_confirmed_packet": 567, "prefix_usd_allocated": 0.1886935, "allocated_pipeline_usd": 0.2583415 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "arm": "late_physical4096", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "status": "ok", "input_confirmed": true, "api_usd": 0.09238, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 19415, "provider_output_tokens": 736, "packet_cl100k_tokens": 12085, "source_lines_in_confirmed_packet": 927, "prefix_usd_allocated": 0.1886935, "allocated_pipeline_usd": 0.2810735 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "native_packet", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "input_confirmed": true, "api_usd": 0.144084, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 32436, "provider_output_tokens": 717, "packet_cl100k_tokens": 20416, "source_lines_in_confirmed_packet": 1784, "prefix_usd_allocated": 0.38395170000000006, "allocated_pipeline_usd": 0.5280357 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "arm": "late_physical4096", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "input_confirmed": true, "api_usd": 0.169852, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 38963, "provider_output_tokens": 700, "packet_cl100k_tokens": 24481, "source_lines_in_confirmed_packet": 2144, "prefix_usd_allocated": 0.38395170000000006, "allocated_pipeline_usd": 0.5538037 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "native_packet", "repo": "fmtlib/fmt", "repo_url": "https://github.com/fmtlib/fmt.git", "language": "cpp", "status": "ok", "input_confirmed": true, "api_usd": 0.047752, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 8653, "provider_output_tokens": 657, "packet_cl100k_tokens": 5203, "source_lines_in_confirmed_packet": 366, "prefix_usd_allocated": 0.11077899999999999, "allocated_pipeline_usd": 0.15853099999999998 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "arm": "late_physical4096", "repo": "fmtlib/fmt", "repo_url": "https://github.com/fmtlib/fmt.git", "language": "cpp", "status": "ok", "input_confirmed": true, "api_usd": 0.073856, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 15069, "provider_output_tokens": 679, "packet_cl100k_tokens": 9289, "source_lines_in_confirmed_packet": 700, "prefix_usd_allocated": 0.11077899999999999, "allocated_pipeline_usd": 0.184635 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "arm": "native_packet", "repo": "future-architect/vuls", "repo_url": "https://github.com/future-architect/vuls.git", "language": "go", "status": "ok", "input_confirmed": true, "api_usd": 0.037268, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 6002, "provider_output_tokens": 663, "packet_cl100k_tokens": 3593, "source_lines_in_confirmed_packet": 259, "prefix_usd_allocated": 0.10579999999999998, "allocated_pipeline_usd": 0.14306799999999997 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "arm": "late_physical4096", "repo": "future-architect/vuls", "repo_url": "https://github.com/future-architect/vuls.git", "language": "go", "status": "ok", "input_confirmed": true, "api_usd": 0.064076, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 12234, "provider_output_tokens": 757, "packet_cl100k_tokens": 7659, "source_lines_in_confirmed_packet": 602, "prefix_usd_allocated": 0.10579999999999998, "allocated_pipeline_usd": 0.16987599999999997 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "arm": "native_packet", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "status": "ok", "input_confirmed": true, "api_usd": 0.069228, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 12997, "provider_output_tokens": 862, "packet_cl100k_tokens": 7741, "source_lines_in_confirmed_packet": 609, "prefix_usd_allocated": 0.166744, "allocated_pipeline_usd": 0.23597200000000002 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "arm": "late_physical4096", "repo": "fasterxml/jackson", "repo_url": "https://github.com/fasterxml/jackson-databind.git", "language": "java", "status": "ok", "input_confirmed": true, "api_usd": 0.096612, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 19913, "provider_output_tokens": 848, "packet_cl100k_tokens": 11830, "source_lines_in_confirmed_packet": 954, "prefix_usd_allocated": 0.166744, "allocated_pipeline_usd": 0.26335600000000003 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "arm": "native_packet", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "status": "ok", "input_confirmed": true, "api_usd": 0.119, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 25325, "provider_output_tokens": 885, "packet_cl100k_tokens": 15726, "source_lines_in_confirmed_packet": 1072, "prefix_usd_allocated": 0.2933673, "allocated_pipeline_usd": 0.4123673 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "arm": "late_physical4096", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "status": "ok", "input_confirmed": true, "api_usd": 0.1435, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 31840, "provider_output_tokens": 807, "packet_cl100k_tokens": 19809, "source_lines_in_confirmed_packet": 1350, "prefix_usd_allocated": 0.2933673, "allocated_pipeline_usd": 0.43686729999999996 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "arm": "native_packet", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "input_confirmed": true, "api_usd": 0.124128, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 27817, "provider_output_tokens": 643, "packet_cl100k_tokens": 17535, "source_lines_in_confirmed_packet": 1502, "prefix_usd_allocated": 0.3170146, "allocated_pipeline_usd": 0.4411426 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "arm": "late_physical4096", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "input_confirmed": true, "api_usd": 0.15036, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 34290, "provider_output_tokens": 660, "packet_cl100k_tokens": 21620, "source_lines_in_confirmed_packet": 1822, "prefix_usd_allocated": 0.3170146, "allocated_pipeline_usd": 0.4673746 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "arm": "native_packet", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "status": "ok", "input_confirmed": true, "api_usd": 0.113288, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 24112, "provider_output_tokens": 842, "packet_cl100k_tokens": 13838, "source_lines_in_confirmed_packet": 913, "prefix_usd_allocated": 0.2329812, "allocated_pipeline_usd": 0.3462692 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "arm": "late_physical4096", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "status": "ok", "input_confirmed": true, "api_usd": 0.138024, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 30741, "provider_output_tokens": 753, "packet_cl100k_tokens": 17912, "source_lines_in_confirmed_packet": 1194, "prefix_usd_allocated": 0.2329812, "allocated_pipeline_usd": 0.37100520000000003 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "arm": "native_packet", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "status": "ok", "input_confirmed": true, "api_usd": 0.03926, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 6930, "provider_output_tokens": 577, "packet_cl100k_tokens": 4175, "source_lines_in_confirmed_packet": 330, "prefix_usd_allocated": 0.08864910000000001, "allocated_pipeline_usd": 0.1279091 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "arm": "late_physical4096", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "status": "ok", "input_confirmed": true, "api_usd": 0.066148, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 13397, "provider_output_tokens": 628, "packet_cl100k_tokens": 8253, "source_lines_in_confirmed_packet": 730, "prefix_usd_allocated": 0.08864910000000001, "allocated_pipeline_usd": 0.15479710000000002 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "arm": "native_packet", "repo": "jqlang/jq", "repo_url": "https://github.com/jqlang/jq.git", "language": "c", "status": "ok", "input_confirmed": true, "api_usd": 0.083104, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 16541, "provider_output_tokens": 847, "packet_cl100k_tokens": 10142, "source_lines_in_confirmed_packet": 733, "prefix_usd_allocated": 0.1509037, "allocated_pipeline_usd": 0.23400769999999999 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "arm": "late_physical4096", "repo": "jqlang/jq", "repo_url": "https://github.com/jqlang/jq.git", "language": "c", "status": "ok", "input_confirmed": true, "api_usd": 0.103936, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 22684, "provider_output_tokens": 660, "packet_cl100k_tokens": 14234, "source_lines_in_confirmed_packet": 1048, "prefix_usd_allocated": 0.1509037, "allocated_pipeline_usd": 0.2548397 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "arm": "native_packet", "repo": "darkreader/darkreader", "repo_url": "https://github.com/darkreader/darkreader.git", "language": "typescript", "status": "ok", "input_confirmed": true, "api_usd": 0.05042, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 9240, "provider_output_tokens": 673, "packet_cl100k_tokens": 5960, "source_lines_in_confirmed_packet": 471, "prefix_usd_allocated": 0.1315079, "allocated_pipeline_usd": 0.1819279 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "arm": "late_physical4096", "repo": "darkreader/darkreader", "repo_url": "https://github.com/darkreader/darkreader.git", "language": "typescript", "status": "ok", "input_confirmed": true, "api_usd": 0.076468, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 15442, "provider_output_tokens": 735, "packet_cl100k_tokens": 10029, "source_lines_in_confirmed_packet": 794, "prefix_usd_allocated": 0.1315079, "allocated_pipeline_usd": 0.2079759 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "arm": "native_packet", "repo": "nlohmann/json", "repo_url": "https://github.com/nlohmann/json.git", "language": "cpp", "status": "ok", "input_confirmed": true, "api_usd": 0.041044, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 6851, "provider_output_tokens": 682, "packet_cl100k_tokens": 4183, "source_lines_in_confirmed_packet": 334, "prefix_usd_allocated": 0.1176269, "allocated_pipeline_usd": 0.1586709 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "arm": "late_physical4096", "repo": "nlohmann/json", "repo_url": "https://github.com/nlohmann/json.git", "language": "cpp", "status": "ok", "input_confirmed": true, "api_usd": 0.068444, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 13376, "provider_output_tokens": 747, "packet_cl100k_tokens": 8251, "source_lines_in_confirmed_packet": 671, "prefix_usd_allocated": 0.1176269, "allocated_pipeline_usd": 0.1860709 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "arm": "native_packet", "repo": "catchorg/Catch2", "repo_url": "https://github.com/catchorg/Catch2.git", "language": "cpp", "status": "ok", "input_confirmed": true, "api_usd": 0.038808, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 6652, "provider_output_tokens": 610, "packet_cl100k_tokens": 3898, "source_lines_in_confirmed_packet": 319, "prefix_usd_allocated": 0.0842546, "allocated_pipeline_usd": 0.1230626 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "arm": "late_physical4096", "repo": "catchorg/Catch2", "repo_url": "https://github.com/catchorg/Catch2.git", "language": "cpp", "status": "ok", "input_confirmed": true, "api_usd": 0.070032, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 13948, "provider_output_tokens": 712, "packet_cl100k_tokens": 7967, "source_lines_in_confirmed_packet": 607, "prefix_usd_allocated": 0.0842546, "allocated_pipeline_usd": 0.1542866 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "arm": "native_packet", "repo": "tokio-rs/bytes", "repo_url": "https://github.com/tokio-rs/bytes.git", "language": "rust", "status": "ok", "input_confirmed": true, "api_usd": 0.051948, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 9142, "provider_output_tokens": 769, "packet_cl100k_tokens": 4935, "source_lines_in_confirmed_packet": 356, "prefix_usd_allocated": 0.1245299, "allocated_pipeline_usd": 0.1764779 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "arm": "late_physical4096", "repo": "tokio-rs/bytes", "repo_url": "https://github.com/tokio-rs/bytes.git", "language": "rust", "status": "ok", "input_confirmed": true, "api_usd": 0.082676, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 15484, "provider_output_tokens": 1037, "packet_cl100k_tokens": 9014, "source_lines_in_confirmed_packet": 701, "prefix_usd_allocated": 0.1245299, "allocated_pipeline_usd": 0.2072059 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "arm": "native_packet", "repo": "serverless/serverless", "repo_url": "https://github.com/serverless/serverless.git", "language": "javascript", "status": "ok", "input_confirmed": true, "api_usd": 0.074812, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 15308, "provider_output_tokens": 679, "packet_cl100k_tokens": 8305, "source_lines_in_confirmed_packet": 603, "prefix_usd_allocated": 0.1445584, "allocated_pipeline_usd": 0.21937040000000002 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "arm": "late_physical4096", "repo": "serverless/serverless", "repo_url": "https://github.com/serverless/serverless.git", "language": "javascript", "status": "ok", "input_confirmed": true, "api_usd": 0.104904, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 22526, "provider_output_tokens": 740, "packet_cl100k_tokens": 12375, "source_lines_in_confirmed_packet": 934, "prefix_usd_allocated": 0.1445584, "allocated_pipeline_usd": 0.2494624 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "arm": "native_packet", "repo": "google/gson", "repo_url": "https://github.com/google/gson.git", "language": "java", "status": "ok", "input_confirmed": true, "api_usd": 0.056064, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 10951, "provider_output_tokens": 613, "packet_cl100k_tokens": 6571, "source_lines_in_confirmed_packet": 563, "prefix_usd_allocated": 0.159743, "allocated_pipeline_usd": 0.215807 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "arm": "late_physical4096", "repo": "google/gson", "repo_url": "https://github.com/google/gson.git", "language": "java", "status": "ok", "input_confirmed": true, "api_usd": 0.083352, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 17748, "provider_output_tokens": 618, "packet_cl100k_tokens": 10644, "source_lines_in_confirmed_packet": 925, "prefix_usd_allocated": 0.159743, "allocated_pipeline_usd": 0.243095 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "arm": "native_packet", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "input_confirmed": true, "api_usd": 0.04092, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 6945, "provider_output_tokens": 657, "packet_cl100k_tokens": 4138, "source_lines_in_confirmed_packet": 321, "prefix_usd_allocated": 0.08079030000000001, "allocated_pipeline_usd": 0.12171030000000001 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "arm": "late_physical4096", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "input_confirmed": true, "api_usd": 0.055448, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 11052, "provider_output_tokens": 562, "packet_cl100k_tokens": 6672, "source_lines_in_confirmed_packet": 520, "prefix_usd_allocated": 0.08079030000000001, "allocated_pipeline_usd": 0.1362383 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "arm": "native_packet", "repo": "alibaba/fastjson2", "repo_url": "https://github.com/alibaba/fastjson2.git", "language": "java", "status": "ok", "input_confirmed": true, "api_usd": 0.039612, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 7048, "provider_output_tokens": 571, "packet_cl100k_tokens": 4406, "source_lines_in_confirmed_packet": 332, "prefix_usd_allocated": 0.0967964, "allocated_pipeline_usd": 0.1364084 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "arm": "late_physical4096", "repo": "alibaba/fastjson2", "repo_url": "https://github.com/alibaba/fastjson2.git", "language": "java", "status": "ok", "input_confirmed": true, "api_usd": 0.069332, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 13538, "provider_output_tokens": 759, "packet_cl100k_tokens": 8484, "source_lines_in_confirmed_packet": 681, "prefix_usd_allocated": 0.0967964, "allocated_pipeline_usd": 0.1661284 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "arm": "native_packet", "repo": "flipt-io/flipt", "repo_url": "https://github.com/flipt-io/flipt.git", "language": "go", "status": "ok", "input_confirmed": true, "api_usd": 0.07382, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 15215, "provider_output_tokens": 648, "packet_cl100k_tokens": 8925, "source_lines_in_confirmed_packet": 758, "prefix_usd_allocated": 0.13847730000000003, "allocated_pipeline_usd": 0.21229730000000002 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "arm": "late_physical4096", "repo": "flipt-io/flipt", "repo_url": "https://github.com/flipt-io/flipt.git", "language": "go", "status": "ok", "input_confirmed": true, "api_usd": 0.098968, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 21742, "provider_output_tokens": 600, "packet_cl100k_tokens": 12982, "source_lines_in_confirmed_packet": 1109, "prefix_usd_allocated": 0.13847730000000003, "allocated_pipeline_usd": 0.23744530000000003 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "arm": "native_packet", "repo": "axios/axios", "repo_url": "https://github.com/axios/axios.git", "language": "javascript", "status": "ok", "input_confirmed": true, "api_usd": 0.036204, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 5731, "provider_output_tokens": 664, "packet_cl100k_tokens": 3651, "source_lines_in_confirmed_packet": 341, "prefix_usd_allocated": 0.0839265, "allocated_pipeline_usd": 0.1201305 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "arm": "late_physical4096", "repo": "axios/axios", "repo_url": "https://github.com/axios/axios.git", "language": "javascript", "status": "ok", "input_confirmed": true, "api_usd": 0.060736, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 12184, "provider_output_tokens": 600, "packet_cl100k_tokens": 7722, "source_lines_in_confirmed_packet": 702, "prefix_usd_allocated": 0.0839265, "allocated_pipeline_usd": 0.1446625 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "arm": "native_packet", "repo": "pydata/xarray", "repo_url": "https://github.com/pydata/xarray.git", "language": "python", "status": "ok", "input_confirmed": true, "api_usd": 0.043056, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 7624, "provider_output_tokens": 628, "packet_cl100k_tokens": 4113, "source_lines_in_confirmed_packet": 311, "prefix_usd_allocated": 0.1287801, "allocated_pipeline_usd": 0.1718361 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "arm": "late_physical4096", "repo": "pydata/xarray", "repo_url": "https://github.com/pydata/xarray.git", "language": "python", "status": "ok", "input_confirmed": true, "api_usd": 0.06958, "paid_message_requests": 1, "api_http_requests": 2, "response_complete": true, "stop_reason": "end_turn", "observed_thinking_blocks": 1, "provider_input_tokens": 13980, "provider_output_tokens": 683, "packet_cl100k_tokens": 8146, "source_lines_in_confirmed_packet": 663, "prefix_usd_allocated": 0.1287801, "allocated_pipeline_usd": 0.1983601 } ], "gates": { "all48_source_packets_normal_and_confirmed": true, "exactly_one_paid_request_each": true, "known_usage": true, "all_official_scores_valid": true, "runtime_source_input_driver_hashes_exact": true, "all24_selected_positions_retained": true, "within_reservation": true }, "complete24_source_delivery_engineering_gate_passed": true, "new_api_usage_estimate_usd": 4.2547440000000005, "paid_message_requests": 48, "free_token_count_requests": 96, "complete_review_responses": 48, "stop_reason_counts": { "end_turn": 48 }, "sdk_sessions": 0, "reused_prefix_sdk_usd": 4.4310748, "allocated_pipeline_usd": { "native_packet": 6.2610068, "late_physical4096": 6.855886799999999 }, "prior_failed_sdk_attempts_api_usd": 4.631439, "all_packet_delivery_api_usage_usd": 8.886183, "known_program_api_usage_value_usd": 47.480044930000005, "cost_scope": "New direct Messages usage is distinct from SDK usage; no new SDK sessions. Both pipeline arms allocate full historical native prefix, never double-count it as new spending. Count endpoint is free. Prices frozen4/20 per million standard/global/uncached tokens, actual account bill and credit deductions unverified.", "repair_calls": 0, "independent_gain_established": false, "prior_sdk_gates_unchanged": true, "next": "Only a passing complete24 engineering gate permits freezing a new independent source-only protocol on untouched IDs. Both old32 cohorts remain sealed. No repair/full500 launch automatically; verified500 remains a separate measured budget decision.", "analysis_sha256": "7fe5571c6b983bd7c46ab8e03acb25fd8d69be5227f068bc45eb5f0149387bda", "token_preflight_sha256": "2422b0526e6a0e02967ba071413fa403ad9e9e77768a7feb00f27042f6ffa6d3", "root_cause_audit_sha256": "94101620c98f773094524eabe915463085bab704b8126903029461096d798cf1", "model_requested": "claude-opus-5-5", "actual_main_models": [ "claude-opus-5-5" ], "profile": { "endpoint": "https://api.anthropic.com", "effort": "medium", "thinking": "adaptive", "max_output_tokens": 2048, "tools": [], "max_retries": 0, "follow_redirects": false, "priced_profile": "standard/global/uncached", "usd_per_million_input_output": [ 4, 20 ] }, "httpx_versions": [ "0.28.1" ] }, "late_packet_sdk_root_cause_audit": { "scope": "Root cause and auxiliary billing audit only; no thought contents, source score changes or new calls. Requested disabled thinking is unsupported by Opus5.5;512 output tokens can truncate thinking plus answer and CLI can continue. Terminal model_usage also contains background Haiku; its cost was already included in SDK totals, so no ledger increase.", "official_model_capability_url": "https://platform.claude.com/docs/en/models/opus-5-5/whats-new-opus-5-5", "official_pricing_url": "https://platform.claude.com/docs/en/models/opus-5-5/overview", "official_count_tokens_url": "https://platform.claude.com/docs/en/build-with-claude/token-counting", "checked_date": "2026-10-17", "current_api_prices_usd_per_million": { "input": 4, "output": 20 }, "rows": [ { "study": "contextbench-late-packet-input-4-20261017", "sessions_with_model_usage": 4, "model_usage_usd": { "claude-haiku-5-5": 0.0051862, "claude-opus-5-5": 0.8449466 }, "sum_matches_terminal_cost_sessions": 4 }, { "study": "contextbench-late-packet-input-v2-4-20261017", "sessions_with_model_usage": 8, "model_usage_usd": { "claude-haiku-5-5": 0.0151773, "claude-opus-5-5": 0.8954656000000001 }, "sum_matches_terminal_cost_sessions": 8 }, { "study": "contextbench-late-packet-input-development20-20261017", "sessions_with_model_usage": 24, "model_usage_usd": { "claude-haiku-5-5": 0.0487389, "claude-opus-5-5": 2.8219243999999994 }, "sum_matches_terminal_cost_sessions": 24 } ], "new_api_cost_usd": 0, "previous_scores_and_gates_unchanged": true }, "late_packet_opportunity_delivery_audit": { "scope": "Descriptive source geometry only. Intermediate decoding/repacking may omit boundary blank lines; actual source scores use only confirmed exact packet lines. Keep original opportunity and actual-input scores unchanged. No strategy/driver/source/annotation-denominator changes, new model calls or new credits.", "rows": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "earlier_opportunity_source_points": 567, "actual_confirmed_source_points": 565, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "earlier_opportunity_source_points": 930, "actual_confirmed_source_points": 928, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__b8c88ac8", "earlier_opportunity_source_points": 2152, "actual_confirmed_source_points": 2145, "opportunity_points_not_credited": 7, "missing_points_blank_source_lines": 7, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__8ebb764e", "earlier_opportunity_source_points": 703, "actual_confirmed_source_points": 701, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__720b4d92", "earlier_opportunity_source_points": 605, "actual_confirmed_source_points": 602, "opportunity_points_not_credited": 3, "missing_points_blank_source_lines": 3, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__07d7817b", "earlier_opportunity_source_points": 955, "actual_confirmed_source_points": 955, "opportunity_points_not_credited": 0, "missing_points_blank_source_lines": 0, "unexpected_extra_points": 0 }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__4eb0e647", "earlier_opportunity_source_points": 1354, "actual_confirmed_source_points": 1352, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__3be02163", "earlier_opportunity_source_points": 1825, "actual_confirmed_source_points": 1824, "opportunity_points_not_credited": 1, "missing_points_blank_source_lines": 1, "unexpected_extra_points": 0 }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__4f3cb6be", "earlier_opportunity_source_points": 1198, "actual_confirmed_source_points": 1195, "opportunity_points_not_credited": 3, "missing_points_blank_source_lines": 3, "unexpected_extra_points": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__60bc9e86", "earlier_opportunity_source_points": 731, "actual_confirmed_source_points": 730, "opportunity_points_not_credited": 1, "missing_points_blank_source_lines": 1, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__cdbc5890", "earlier_opportunity_source_points": 1048, "actual_confirmed_source_points": 1048, "opportunity_points_not_credited": 0, "missing_points_blank_source_lines": 0, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2fb50735", "earlier_opportunity_source_points": 795, "actual_confirmed_source_points": 794, "opportunity_points_not_credited": 1, "missing_points_blank_source_lines": 1, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__7c9ef76c", "earlier_opportunity_source_points": 673, "actual_confirmed_source_points": 671, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__cpp__maintenance__bugfix__e43d9aa5", "earlier_opportunity_source_points": 609, "actual_confirmed_source_points": 607, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "earlier_opportunity_source_points": 1768, "actual_confirmed_source_points": 1767, "opportunity_points_not_credited": 1, "missing_points_blank_source_lines": 1, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__03fdad45", "earlier_opportunity_source_points": 704, "actual_confirmed_source_points": 702, "opportunity_points_not_credited": 2, "missing_points_blank_source_lines": 2, "unexpected_extra_points": 0 }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__f4541fb1", "earlier_opportunity_source_points": 940, "actual_confirmed_source_points": 935, "opportunity_points_not_credited": 5, "missing_points_blank_source_lines": 5, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "earlier_opportunity_source_points": 3248, "actual_confirmed_source_points": 3244, "opportunity_points_not_credited": 4, "missing_points_blank_source_lines": 4, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__747c7f60", "earlier_opportunity_source_points": 930, "actual_confirmed_source_points": 926, "opportunity_points_not_credited": 4, "missing_points_blank_source_lines": 4, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__6d48775e", "earlier_opportunity_source_points": 520, "actual_confirmed_source_points": 520, "opportunity_points_not_credited": 0, "missing_points_blank_source_lines": 0, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__java__maintenance__bugfix__8b7cc5e0", "earlier_opportunity_source_points": 684, "actual_confirmed_source_points": 681, "opportunity_points_not_credited": 3, "missing_points_blank_source_lines": 3, "unexpected_extra_points": 0 }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__997c7afd", "earlier_opportunity_source_points": 1116, "actual_confirmed_source_points": 1109, "opportunity_points_not_credited": 7, "missing_points_blank_source_lines": 7, "unexpected_extra_points": 0 }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__98bbaed2", "earlier_opportunity_source_points": 705, "actual_confirmed_source_points": 702, "opportunity_points_not_credited": 3, "missing_points_blank_source_lines": 3, "unexpected_extra_points": 0 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__42c77239", "earlier_opportunity_source_points": 672, "actual_confirmed_source_points": 666, "opportunity_points_not_credited": 6, "missing_points_blank_source_lines": 6, "unexpected_extra_points": 0 } ], "missing_opportunity_points": 63, "all_missing_points_blank": true, "unexpected_extra_points": 0, "actual_analysis_sha256": "7fe5571c6b983bd7c46ab8e03acb25fd8d69be5227f068bc45eb5f0149387bda", "new_model_calls": 0, "new_api_cost_usd": 0, "source_points_omitted_from_actual_packets_or_prefix": 63, "additional_points_lost_in_prediction_parsing": 0 }, "late_source_independent32_prospective_selection": { "scope": "Historical metadata-only selection record before inference. Source/annotation fields describe that selection stage; terminal amended32 results are reported separately. Six languages,not repository-held-out or representative500.", "selection_protocol_sha256": "64c1bf9a8e478eaf0aeae555499cd9b939082ae7d29cf7fc8d1be425a25bb139", "selection_sha256": "e9053b3d09786c4b5ebc86795f1f6359bcab77fa1a7dc362cb6a4ebd274a1dac", "runtime_input_receipt_sha256": "20e6dcdc4bf6b93464e5bdcdfa102175bfdbf6e6634e08b3d3ee7cfa8e894509", "tasks": 32, "languages": { "typescript": 4, "python": 12, "javascript": 4, "c": 4, "rust": 4, "go": 4 }, "repository_count": 23, "candidate_pool_at_selection": 324, "unreserved_verified_candidates": 292, "annotation_fields_loaded": false, "reference_patches_loaded": false, "new_paid_requests": 0, "open_gates": [], "all32_exact_sources_prepared": true, "prepared_source_readiness_sha256": "3e35e0cb35bcd77afdd4895486b8465d5dafe1db6d041446861332f6747908e3", "runtime_gold_empty": true, "state": "selection_closed_all32_scored_and_sealed" }, "late_source_independent32_frozen_protocol": { "scope": "Fresh independent32 task-level source-strategy validation of the frozen late physical4096 policy. Six languages/23 repository names; no C++/Java,not repository-held-out or representative full500 distribution. A fresh native investigation per task is shared across two final source-packet arms,not two fresh adaptive investigations. Both earlier32 cohorts sealed. No repairs/native tests/full500.", "population": [ "Multi-SWE-Bench__typescript__maintenance__bugfix__8fcb53e6", "SWE-Bench-Verified__python__maintenance__bugfix__dc72c033", "SWE-Bench-Verified__python__maintenance__bugfix__d561b333", "SWE-Bench-Verified__python__maintenance__bugfix__51f3cc91", "SWE-Bench-Verified__python__maintenance__bugfix__9dcac3ec", "SWE-Bench-Pro__python__maintenance__bugfix__b9f9961f", "SWE-PolyBench__python__evolution__feature__66f97093", "Multi-SWE-Bench__javascript__maintenance__bugfix__1081c568", "SWE-Bench-Verified__python__maintenance__bugfix__34e61891", "Multi-SWE-Bench__c__maintenance__bugfix__634fe9b8", "Multi-SWE-Bench__rust__maintenance__bugfix__92158fba", "Multi-SWE-Bench__c__maintenance__bugfix__39769692", "Multi-SWE-Bench__typescript__maintenance__bugfix__2aa6fa4c", "Multi-SWE-Bench__go__maintenance__bugfix__3d85271b", "SWE-Bench-Pro__python__maintenance__bugfix__89932d58", "Multi-SWE-Bench__go__maintenance__bugfix__3c9b757f", "Multi-SWE-Bench__go__maintenance__bugfix__52c152ba", "SWE-PolyBench__typescript__maintenance__bugfix__48e5846a", "Multi-SWE-Bench__javascript__maintenance__bugfix__72488b59", "Multi-SWE-Bench__go__maintenance__bugfix__2fea5f6b", "SWE-Bench-Verified__python__maintenance__bugfix__4f685b8e", "SWE-PolyBench__javascript__maintenance__bugfix__4d090fc6", "SWE-PolyBench__python__maintenance__bugfix__b110ef6c", "SWE-Bench-Pro__python__maintenance__bugfix__ab946da6", "Multi-SWE-Bench__rust__maintenance__bugfix__0e4346f7", "Multi-SWE-Bench__c__maintenance__bugfix__1e195cd6", "SWE-PolyBench__typescript__maintenance__bugfix__708894b2", "Multi-SWE-Bench__rust__maintenance__bugfix__ae8b362a", "Multi-SWE-Bench__c__maintenance__bugfix__45da4482", "Multi-SWE-Bench__rust__maintenance__bugfix__d3133dfb", "SWE-Bench-Pro__javascript__maintenance__bugfix__09eb0d6d", "SWE-Bench-Verified__python__maintenance__bugfix__1ce4c38e" ], "languages": { "typescript": 4, "python": 12, "javascript": 4, "c": 4, "rust": 4, "go": 4 }, "arms": [ "native_packet", "late_physical4096" ], "selection_sha256": "e9053b3d09786c4b5ebc86795f1f6359bcab77fa1a7dc362cb6a4ebd274a1dac", "parent_engineering_analysis_sha256": "7fe5571c6b983bd7c46ab8e03acb25fd8d69be5227f068bc45eb5f0149387bda", "implementation_commit": "d2bf113623f201ad5a167126909bcee5dcc37ebb", "model": "claude-opus-5-5", "policy": "Exactly parent d2bf1136 helper bytes,including documented boundary-blank omissions. Native read/grep/glob only,no graph,BM25,Jev,file-map or first-read augmentation. Project only actual version-verified native tool replies followed by an assistant,then choose source from already observed files using final-handoff file order,64-line/3000-character tiling,nearest-span order and global200-window file round-robin cap. <=2MiB files,4K novel source before observed source within the same32K cl100k final cap. Extra used source/model tokens and cost are part of the treatment. No selection from annotations.", "native_profile": { "backend": "claude-agent-sdk", "route": "anthropic-api", "effort": "medium", "max_turns": 20, "max_output_tokens_requested": 4096, "timeout_s": 600, "case_sdk_guard_usd": 0.75, "headroom_usd": 0.25, "source_policy": "native", "grep_view": "native" }, "consumer_profile": { "backend": "direct-messages", "endpoint": "https://api.anthropic.com", "thinking": { "type": "adaptive" }, "effort": "medium", "max_output_tokens": 2048, "timeout_s": 180, "request_guard_usd": 0.5, "headroom_usd": 0.1, "paid_requests_per_packet": 1, "free_count_requests_per_packet": 1, "http_retries": 0, "follow_redirects": false, "pricing_usd_per_million_input_output": [ 4, 20 ], "pricing_profile": "standard/global/uncached" }, "budget": { "reservation_usd": 24, "known_prior_program_api_usage_value_usd": 47.48004493, "authorized_api_ceiling_usd": 100, "forecast_from_development_shared_pipeline_usd": 11.581091733333333, "max_concurrent_pipelines": 2, "max_concurrent_consumer_requests": 4, "complete_pipeline_window_reservation_usd": 2.2, "experiment_retries": 0, "fallback": false, "actual_account_bill_and_credit_deduction_verified": false }, "stop": "Reserve each complete pipeline at2.2 (native.75+.25;two source messages each.5+.1),at most two pipelines per window. Smaller windows only for reservation. Drain reserved siblings. Stop subsequent windows on missing/uncertain usage,source/runtime/model/route/tool failure,unexpected native terminal or source-consumer error,per-native usage>1 or per-message>.6. Preserve every error/not-run case. No session retry,replacement,model fallback or interim scoring. Guards are post-response/estimated bounds,not provider account hard caps.", "accepted_native_terminals": "Raw success/ok or expected error_max_turns/error_max_budget_usd with known usage,correct actual main model/API route and no unavailable tools. SDK iteration may leave raw status=error after an expected budget terminal; retain that error and report it explicitly,while source-only processing may continue from confirmed observations. This prospectively defined handling does not relabel older failures or imply a completed investigation.", "accepted_consumer_terminals": "Normal HTTP200 assistant Message at end_turn or max_tokens,exact fixed source/prompt/model/version/route,known uncached standard/global usage and one paid HTTP message. Capped responses confirm input but remain incomplete answers. Reject all other statuses/tools/model/price profiles; no retry.", "source_credit": "Retain raw native attempts verbatim. Before packet construction and scoring,remove only tool replies with no subsequent assistant event from a labeled confirmed-prefix projection and recompute its trajectory. A prepared packet gets no source credit until an actual consumer assistant response plus terminal usage/final source validation. Only confirmed exact source contributes; no credit from requested ranges,assistant summaries or undelivered output.", "annotation_isolation": "Runtime metadata/issues contain explicit empty gold_context. No annotation/patch/test-label columns during inference or packet generation. Only after terminal execution and immutable output receipts,materialize selected gold_context in a separate evaluation cohort. Freeze all confirmed predictions before official scoring. Keep all32 selected positions,including failures/not-run; invalid/missing results remain zero in task-macro metrics.", "primary": "All32 paired macro cumulative confirmed-source line Recall difference. Require lower endpoint>0 in two-sided repository-cluster95 bootstrap (5000 draws,seed20261017),all32 native receipts accepted under the stated terminal policy,all64 source consumers confirmed with known usage/model/route/source/runtime receipts,32 valid official cumulative and final-packet scores per arm,complete costs within24. Report Precision,case-macro F1,source volume,raw errors/caps and costs; F1 nondecrease is not a veto.", "bootstrap": "Canonical lowercase repository URLs with trailing slash/.git removed. Resample whole repository clusters with replacement,retaining all tasks in each selected cluster; compute task-macro paired differences. Report actual cluster count and balanced-six-language population limits.", "cost_allocation": "Each logical arm allocates its full native investigation cost plus its own packet consumer. Actual new spend pays each native prefix once plus both consumers; never double-count shared-prefix allocations. SDK sessions,Messages inference requests,free counts and underlying SDK HTTP counts are distinct; auxiliary SDK model_usage remains in total costs.", "secondary": "Final32K source-packet and equal-language macro scores separately,never replacing failed primary. Report capped/incomplete review replies separately from source-input confirmation.", "promotion": "Seal new32 after scoring. Only a passing independent gate permits a separately measured verified500 quality/cost/source/readiness plan. No repair/native-harness or full500 batch starts automatically. Failure rejects the unchanged policy for promotion; neither old32 cohort is recycled.", "local_verification": { "unchanged_parent_runtime": true, "focused_retrieval_tests_passed": 104, "mock_http_boundary_tests": 17, "unpaid_prefix_pipeline_preflight": true, "all32_exact_source_ready": true }, "protocol_sha256": "0b84a4a4e39a20449366e079277a3658bb99e0b0da02f907a6c7e346905b632e", "state": "stopped_local_packing_error_before_labels", "outcomes_available": true, "interim_quality_scoring": false, "raw_runtime_and_driver_receipts_archived": true, "quality_scored": false, "original_operational_gate_passed": false, "completed_native_sessions": 8, "completed_message_requests": 14, "complete_pairs": 7, "known_api_usage_usd": 2.5663486, "model_retries": 0 }, "late_source_independent32_encoding_amendment": { "scope": "Pre-label operational amendment of the same untouched32 cohort. Original run failed locally in source packing after8 native investigations/14 source messages,with no selected annotations or quality scores. Preserve that original failure,all inputs/outputs and the exact source-only terminal audit. This is not a new sample or a successful reclassification of the original strict execution.", "parent_protocol_sha256": "0b84a4a4e39a20449366e079277a3658bb99e0b0da02f907a6c7e346905b632e", "parent_execution_sha256": "1d9f5f88557ebbb0ce960b418210a2402d063c600d4a7e466b8d41bf0f448151", "parent_prelabel_audit_sha256": "84fb369113343a3f0ef509fd422b81595830e8a4671380161d6c5ff639f27894", "original_operational_gate_passed": false, "selection_sha256": "e9053b3d09786c4b5ebc86795f1f6359bcab77fa1a7dc362cb6a4ebd274a1dac", "policy_correction": "Treat source tokenizer marker strings as ordinary text in cl100k packet packing/validation via disallowed_special=(). No file/window/order/query/model/route/output/turn/source/token-cap/primary/interval/F1-gate changes. Generic valid-text handling,not a reference-label or outcome-driven selection change.", "source_query_implementation_unchanged": true, "completed_call_replay_byte_identical": [ "Multi-SWE-Bench__typescript__maintenance__bugfix__8fcb53e6", "SWE-Bench-Verified__python__maintenance__bugfix__dc72c033", "SWE-Bench-Verified__python__maintenance__bugfix__d561b333", "SWE-Bench-Verified__python__maintenance__bugfix__51f3cc91", "SWE-Bench-Verified__python__maintenance__bugfix__9dcac3ec", "SWE-Bench-Pro__python__maintenance__bugfix__b9f9961f", "Multi-SWE-Bench__javascript__maintenance__bugfix__1081c568" ], "native_prefix_reuse_only": [ "SWE-PolyBench__python__evolution__feature__66f97093" ], "unchanged_completed_pipelines": 7, "already_completed_native_sessions": 8, "already_completed_message_requests": 14, "new_native_sessions_planned": 24, "new_messages_planned": 50, "experiment_model_retries": 0, "native_or_message_attempt_replacement": false, "annotation_access_before_amendment": false, "old_known_native_sdk_usd": 1.4335246000000001, "old_known_messages_api_usd": 1.1328240000000003, "old_known_api_total_usd": 2.5663486000000004, "whole32_reservation_usd": 24, "remaining_new_reservation_usd": 21.4336514, "known_program_before_whole32_usd": 47.48004493, "authorized_program_api_ceiling_usd": 100, "validation": "7 completed packet pairs replay byte-identically without paid calls; failed Transformers candidate pool now packs and validates normally; a literal in an over-budget candidate need not enter the final packet. All raw source receipts retained. Reuse every already completed model call; execute only previously unstarted native investigations/consumers. Freeze all amended code/drivers/metrics before any further model calls.", "quality": "Keep original all32 source Recall lower95>0 and full receipt/official-score/cost gates. Read selected annotations only after combined terminal outputs and source-prediction freeze. Report amended32 as a label-blind implementation correction,not original protocol success. Seal this cohort after scoring; both older32 cohorts remain sealed.", "new_paid_requests_in_preparation": 0, "new_api_cost_usd": 0, "protocol_sha256": "75a7579f4f0f379e23a556e869d7132de2113cb29c5e033a0f85e28ad7ce09ad", "runtime_commit": "8d5e0b877d4e2709a1bb407f6487d3fd9d3b9453", "state": "finished_scored_and_sealed", "focused_tests": 129, "whole32_api_usage_usd": 11.183725800000001, "new_encoding_phase_api_usage_usd": 8.6173772, "quality_gate_passed": true, "selected_gold_read_only_after_terminal_source_prediction_freeze": true }, "late_source_independent32_amended_result": { "scope": "Same independent32 under a disclosed pre-label literal-token encoding correction. Original strict execution stopped locally after8 native calls/14 consumers and remains operationally failed/unscored. No labels,model retries,case replacement or score-driven changes. Reuse7 byte-identical completed pipelines and the accepted failed-packing native prefix; run only previously unstarted calls. Original primary metrics/intervals/terminal/source policies unchanged.", "protocol_sha256": "75a7579f4f0f379e23a556e869d7132de2113cb29c5e033a0f85e28ad7ce09ad", "implementation_commit": "8d5e0b877d4e2709a1bb407f6487d3fd9d3b9453", "selection_sha256": "e9053b3d09786c4b5ebc86795f1f6359bcab77fa1a7dc362cb6a4ebd274a1dac", "primary_cumulative": { "native_packet": { "recall": 0.7494390839904986, "precision": 0.22728208253297102, "f1": 0.3013120498589982, "official_valid": 32 }, "late_physical4096": { "recall": 0.79092782160193, "precision": 0.14834402468472668, "f1": 0.22315314579488282, "official_valid": 32 } }, "secondary_final32k_packet": { "native_packet": { "recall": 0.7494390839904986, "precision": 0.22743810145068366, "f1": 0.3014676350244081, "official_valid": 32 }, "late_physical4096": { "recall": 0.79092782160193, "precision": 0.14841269050518446, "f1": 0.22322584474656026, "official_valid": 32 } }, "paired_differences": { "recall": { "difference": 0.04148873761143135, "repository_cluster_95": [ 0.01992324096330084, 0.06849514133270226 ] }, "precision": { "difference": -0.07893805784824434, "repository_cluster_95": [ -0.11960049899118805, -0.0526663567158232 ] }, "f1": { "difference": -0.07815890406411535, "repository_cluster_95": [ -0.1113190866908781, -0.05353474625137731 ] } }, "equal_language_macro_secondary": { "native_packet": { "recall": 0.7459643999824993, "precision": 0.20704786753357526, "f1": 0.2805047633563323 }, "late_physical4096": { "recall": 0.7860031101924517, "precision": 0.14218603464758098, "f1": 0.2130616191397512 } }, "languages": { "typescript": 4, "python": 12, "javascript": 4, "c": 4, "rust": 4, "go": 4 }, "repository_clusters": 22, "rows": [ { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__8fcb53e6", "arm": "native_packet", "repo": "mui/material", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.36695160000000004, "messages_api_usd": 0.14774, "pipeline_usd_allocated": 0.5146916, "provider_input_tokens": 32755, "provider_output_tokens": 836, "context_cl100k_tokens": 20393, "confirmed_packet_source_lines": 1845, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4075, "candidate_windows": 60, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__8fcb53e6", "recall": 0.23255813953488372, "precision": 0.005417118093174431, "f1": 0.010587612493382742 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__8fcb53e6", "arm": "late_physical4096", "repo": "mui/material", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.36695160000000004, "messages_api_usd": 0.1707, "pipeline_usd_allocated": 0.5376516, "provider_input_tokens": 38770, "provider_output_tokens": 781, "context_cl100k_tokens": 24450, "confirmed_packet_source_lines": 2158, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4075, "candidate_windows": 60, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__8fcb53e6", "recall": 0.37209302325581395, "precision": 0.007410838351088467, "f1": 0.014532243415077202 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__dc72c033", "arm": "native_packet", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.08638570000000001, "messages_api_usd": 0.035136, "pipeline_usd_allocated": 0.12152170000000001, "provider_input_tokens": 5599, "provider_output_tokens": 637, "context_cl100k_tokens": 3316, "confirmed_packet_source_lines": 259, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 10, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__dc72c033", "recall": 0.627906976744186, "precision": 0.2084942084942085, "f1": 0.31304347826086953 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__dc72c033", "arm": "late_physical4096", "repo": "django/django", "repo_url": "https://github.com/django/django.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.08638570000000001, "messages_api_usd": 0.062228, "pipeline_usd_allocated": 0.14861370000000002, "provider_input_tokens": 11922, "provider_output_tokens": 727, "context_cl100k_tokens": 7402, "confirmed_packet_source_lines": 566, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 10, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__dc72c033", "recall": 0.627906976744186, "precision": 0.09540636042402827, "f1": 0.1656441717791411 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__d561b333", "arm": "native_packet", "repo": "sympy/sympy", "repo_url": "https://github.com/sympy/sympy.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0824807, "messages_api_usd": 0.03292, "pipeline_usd_allocated": 0.1154007, "provider_input_tokens": 5530, "provider_output_tokens": 540, "context_cl100k_tokens": 3370, "confirmed_packet_source_lines": 183, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4087, "candidate_windows": 115, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__d561b333", "recall": 0.40397350993377484, "precision": 0.6666666666666666, "f1": 0.5030927835051546 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__d561b333", "arm": "late_physical4096", "repo": "sympy/sympy", "repo_url": "https://github.com/sympy/sympy.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0824807, "messages_api_usd": 0.059436, "pipeline_usd_allocated": 0.1419167, "provider_input_tokens": 11284, "provider_output_tokens": 715, "context_cl100k_tokens": 7408, "confirmed_packet_source_lines": 462, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4087, "candidate_windows": 115, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__d561b333", "recall": 0.40397350993377484, "precision": 0.26406926406926406, "f1": 0.3193717277486911 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__51f3cc91", "arm": "native_packet", "repo": "scikit-learn/scikit-learn", "repo_url": "https://github.com/scikit-learn/scikit-learn.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0670289, "messages_api_usd": 0.028404, "pipeline_usd_allocated": 0.0954329, "provider_input_tokens": 3696, "provider_output_tokens": 681, "context_cl100k_tokens": 1705, "confirmed_packet_source_lines": 100, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4096, "candidate_windows": 40, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__51f3cc91", "recall": 0.7222222222222222, "precision": 0.39, "f1": 0.5064935064935066 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__51f3cc91", "arm": "late_physical4096", "repo": "scikit-learn/scikit-learn", "repo_url": "https://github.com/scikit-learn/scikit-learn.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0670289, "messages_api_usd": 0.051392, "pipeline_usd_allocated": 0.1184209, "provider_input_tokens": 9988, "provider_output_tokens": 572, "context_cl100k_tokens": 5783, "confirmed_packet_source_lines": 373, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4096, "candidate_windows": 40, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__51f3cc91", "recall": 1.0, "precision": 0.1447721179624665, "f1": 0.2529274004683841 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__9dcac3ec", "arm": "native_packet", "repo": "sphinx-doc/sphinx", "repo_url": "https://github.com/sphinx-doc/sphinx.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.046490199999999995, "messages_api_usd": 0.02666, "pipeline_usd_allocated": 0.0731502, "provider_input_tokens": 4065, "provider_output_tokens": 520, "context_cl100k_tokens": 2477, "confirmed_packet_source_lines": 167, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 1695, "candidate_windows": 3, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__9dcac3ec", "recall": 1.0, "precision": 0.08982035928143713, "f1": 0.16483516483516483 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__9dcac3ec", "arm": "late_physical4096", "repo": "sphinx-doc/sphinx", "repo_url": "https://github.com/sphinx-doc/sphinx.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.046490199999999995, "messages_api_usd": 0.037212, "pipeline_usd_allocated": 0.0837022, "provider_input_tokens": 6738, "provider_output_tokens": 513, "context_cl100k_tokens": 4162, "confirmed_packet_source_lines": 314, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 1695, "candidate_windows": 3, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__9dcac3ec", "recall": 1.0, "precision": 0.04777070063694268, "f1": 0.0911854103343465 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__b9f9961f", "arm": "native_packet", "repo": "ansible/ansible", "repo_url": "https://github.com/ansible/ansible.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.358661, "messages_api_usd": 0.158584, "pipeline_usd_allocated": 0.517245, "provider_input_tokens": 35016, "provider_output_tokens": 926, "context_cl100k_tokens": 21636, "confirmed_packet_source_lines": 1682, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4088, "candidate_windows": 129, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__b9f9961f", "recall": 0.9147398843930635, "precision": 0.3761140819964349, "f1": 0.5330526315789473 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__b9f9961f", "arm": "late_physical4096", "repo": "ansible/ansible", "repo_url": "https://github.com/ansible/ansible.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.358661, "messages_api_usd": 0.183532, "pipeline_usd_allocated": 0.542193, "provider_input_tokens": 41643, "provider_output_tokens": 848, "context_cl100k_tokens": 25706, "confirmed_packet_source_lines": 1987, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4088, "candidate_windows": 129, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__b9f9961f", "recall": 0.9263005780346821, "precision": 0.3224346076458752, "f1": 0.4783582089552239 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__66f97093", "arm": "native_packet", "repo": "huggingface/transformers", "repo_url": "https://github.com/huggingface/transformers.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.27480720000000003, "messages_api_usd": 0.11206, "pipeline_usd_allocated": 0.3868672, "provider_input_tokens": 24455, "provider_output_tokens": 712, "context_cl100k_tokens": 14533, "confirmed_packet_source_lines": 964, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 200, "skipped": [ { "file": "tests/models/whisper/test_modeling_whisper.py", "overlong_source_lines": [ 2312, 2399, 2432, 2471, 2473, 2474, 2525, 2527, 2528 ] } ], "metrics": { "instance_id": "SWE-PolyBench__python__evolution__feature__66f97093", "recall": 0.7451298701298701, "precision": 0.47564766839378236, "f1": 0.5806451612903226 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__66f97093", "arm": "late_physical4096", "repo": "huggingface/transformers", "repo_url": "https://github.com/huggingface/transformers.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.27480720000000003, "messages_api_usd": 0.138164, "pipeline_usd_allocated": 0.41297120000000004, "provider_input_tokens": 31016, "provider_output_tokens": 705, "context_cl100k_tokens": 18623, "confirmed_packet_source_lines": 1299, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 200, "skipped": [ { "file": "tests/models/whisper/test_modeling_whisper.py", "overlong_source_lines": [ 2312, 2399, 2432, 2471, 2473, 2474, 2525, 2527, 2528 ] } ], "metrics": { "instance_id": "SWE-PolyBench__python__evolution__feature__66f97093", "recall": 0.7857142857142857, "precision": 0.3723076923076923, "f1": 0.5052192066805845 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__1081c568", "arm": "native_packet", "repo": "sveltejs/svelte", "repo_url": "https://github.com/sveltejs/svelte.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1507193, "messages_api_usd": 0.05698, "pipeline_usd_allocated": 0.2076993, "provider_input_tokens": 10270, "provider_output_tokens": 795, "context_cl100k_tokens": 6741, "confirmed_packet_source_lines": 621, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4094, "candidate_windows": 16, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__1081c568", "recall": 1.0, "precision": 0.12721417069243157, "f1": 0.2257142857142857 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__1081c568", "arm": "late_physical4096", "repo": "sveltejs/svelte", "repo_url": "https://github.com/sveltejs/svelte.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1507193, "messages_api_usd": 0.0819, "pipeline_usd_allocated": 0.2326193, "provider_input_tokens": 16235, "provider_output_tokens": 848, "context_cl100k_tokens": 10829, "confirmed_packet_source_lines": 1004, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4094, "candidate_windows": 16, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__1081c568", "recall": 1.0, "precision": 0.07868525896414343, "f1": 0.1458910433979686 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__34e61891", "arm": "native_packet", "repo": "matplotlib/matplotlib", "repo_url": "https://github.com/matplotlib/matplotlib.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.19122000000000003, "messages_api_usd": 0.0648, "pipeline_usd_allocated": 0.25602, "provider_input_tokens": 12780, "provider_output_tokens": 684, "context_cl100k_tokens": 7490, "confirmed_packet_source_lines": 544, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4085, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__34e61891", "recall": 1.0, "precision": 0.04220183486238532, "f1": 0.08098591549295775 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__34e61891", "arm": "late_physical4096", "repo": "matplotlib/matplotlib", "repo_url": "https://github.com/matplotlib/matplotlib.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.19122000000000003, "messages_api_usd": 0.090516, "pipeline_usd_allocated": 0.28173600000000004, "provider_input_tokens": 19199, "provider_output_tokens": 686, "context_cl100k_tokens": 11565, "confirmed_packet_source_lines": 879, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4085, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__34e61891", "recall": 1.0, "precision": 0.026136363636363635, "f1": 0.05094130675526024 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__634fe9b8", "arm": "native_packet", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.11738380000000001, "messages_api_usd": 0.048004, "pipeline_usd_allocated": 0.1653878, "provider_input_tokens": 8601, "provider_output_tokens": 680, "context_cl100k_tokens": 5241, "confirmed_packet_source_lines": 526, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4081, "candidate_windows": 59, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__634fe9b8", "recall": 1.0, "precision": 0.2718631178707224, "f1": 0.42750373692077726 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__634fe9b8", "arm": "late_physical4096", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.11738380000000001, "messages_api_usd": 0.074292, "pipeline_usd_allocated": 0.1916758, "provider_input_tokens": 15153, "provider_output_tokens": 684, "context_cl100k_tokens": 9296, "confirmed_packet_source_lines": 952, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4081, "candidate_windows": 59, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__634fe9b8", "recall": 1.0, "precision": 0.15021008403361344, "f1": 0.26118721461187216 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__92158fba", "arm": "native_packet", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.3057653, "messages_api_usd": 0.120344, "pipeline_usd_allocated": 0.4261093, "provider_input_tokens": 25681, "provider_output_tokens": 881, "context_cl100k_tokens": 16426, "confirmed_packet_source_lines": 1344, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__92158fba", "recall": 0.3470715835140998, "precision": 0.35634743875278396, "f1": 0.3516483516483517 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__92158fba", "arm": "late_physical4096", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.3057653, "messages_api_usd": 0.145216, "pipeline_usd_allocated": 0.45098130000000003, "provider_input_tokens": 32004, "provider_output_tokens": 860, "context_cl100k_tokens": 20512, "confirmed_packet_source_lines": 1666, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__92158fba", "recall": 0.41576283441793205, "precision": 0.3445176752546435, "f1": 0.3768020969855832 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__39769692", "arm": "native_packet", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.5277449, "messages_api_usd": 0.19236, "pipeline_usd_allocated": 0.7201048999999999, "provider_input_tokens": 43020, "provider_output_tokens": 1014, "context_cl100k_tokens": 26410, "confirmed_packet_source_lines": 2359, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4090, "candidate_windows": 180, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__39769692", "recall": 0.6723926380368098, "precision": 0.23220338983050848, "f1": 0.3451968503937008 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__39769692", "arm": "late_physical4096", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.5277449, "messages_api_usd": 0.21826, "pipeline_usd_allocated": 0.7460049, "provider_input_tokens": 49470, "provider_output_tokens": 1019, "context_cl100k_tokens": 30478, "confirmed_packet_source_lines": 2723, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4090, "candidate_windows": 180, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__39769692", "recall": 0.7607361963190185, "precision": 0.2276064610866373, "f1": 0.35038146369030804 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2aa6fa4c", "arm": "native_packet", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0627302, "messages_api_usd": 0.025024, "pipeline_usd_allocated": 0.0877542, "provider_input_tokens": 3066, "provider_output_tokens": 638, "context_cl100k_tokens": 1859, "confirmed_packet_source_lines": 159, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 1390, "candidate_windows": 4, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2aa6fa4c", "recall": 1.0, "precision": 0.12578616352201258, "f1": 0.223463687150838 } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2aa6fa4c", "arm": "late_physical4096", "repo": "vuejs/core", "repo_url": "https://github.com/vuejs/core.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0627302, "messages_api_usd": 0.032796, "pipeline_usd_allocated": 0.0955262, "provider_input_tokens": 5229, "provider_output_tokens": 594, "context_cl100k_tokens": 3243, "confirmed_packet_source_lines": 291, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 1390, "candidate_windows": 4, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__2aa6fa4c", "recall": 1.0, "precision": 0.06872852233676977, "f1": 0.12861736334405147 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3d85271b", "arm": "native_packet", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1569107, "messages_api_usd": 0.060656, "pipeline_usd_allocated": 0.2175667, "provider_input_tokens": 12374, "provider_output_tokens": 558, "context_cl100k_tokens": 7363, "confirmed_packet_source_lines": 527, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4083, "candidate_windows": 21, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3d85271b", "recall": 1.0, "precision": 0.013282732447817837, "f1": 0.026217228464419477 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3d85271b", "arm": "late_physical4096", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1569107, "messages_api_usd": 0.08658, "pipeline_usd_allocated": 0.2434907, "provider_input_tokens": 18730, "provider_output_tokens": 583, "context_cl100k_tokens": 11440, "confirmed_packet_source_lines": 871, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4083, "candidate_windows": 21, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3d85271b", "recall": 1.0, "precision": 0.008036739380022962, "f1": 0.015945330296127564 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__89932d58", "arm": "native_packet", "repo": "qutebrowser/qutebrowser", "repo_url": "https://github.com/qutebrowser/qutebrowser.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.18463279999999999, "messages_api_usd": 0.084332, "pipeline_usd_allocated": 0.2689648, "provider_input_tokens": 17453, "provider_output_tokens": 726, "context_cl100k_tokens": 10624, "confirmed_packet_source_lines": 889, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 83, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__89932d58", "recall": 0.9324324324324325, "precision": 0.1550561797752809, "f1": 0.2658959537572254 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__89932d58", "arm": "late_physical4096", "repo": "qutebrowser/qutebrowser", "repo_url": "https://github.com/qutebrowser/qutebrowser.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.18463279999999999, "messages_api_usd": 0.109512, "pipeline_usd_allocated": 0.2941448, "provider_input_tokens": 23883, "provider_output_tokens": 699, "context_cl100k_tokens": 14650, "confirmed_packet_source_lines": 1207, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 83, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__89932d58", "recall": 0.9391891891891891, "precision": 0.11506622516556292, "f1": 0.20501474926253688 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3c9b757f", "arm": "native_packet", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0840865, "messages_api_usd": 0.046864, "pipeline_usd_allocated": 0.1309505, "provider_input_tokens": 9181, "provider_output_tokens": 507, "context_cl100k_tokens": 5262, "confirmed_packet_source_lines": 359, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4093, "candidate_windows": 23, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3c9b757f", "recall": 0.9891304347826086, "precision": 0.25348189415041783, "f1": 0.4035476718403548 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3c9b757f", "arm": "late_physical4096", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.0840865, "messages_api_usd": 0.075392, "pipeline_usd_allocated": 0.1594785, "provider_input_tokens": 15848, "provider_output_tokens": 600, "context_cl100k_tokens": 9349, "confirmed_packet_source_lines": 682, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4093, "candidate_windows": 23, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__3c9b757f", "recall": 0.9891304347826086, "precision": 0.13343108504398826, "f1": 0.23514211886304906 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__52c152ba", "arm": "native_packet", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.08232690000000001, "messages_api_usd": 0.04848, "pipeline_usd_allocated": 0.1308069, "provider_input_tokens": 8835, "provider_output_tokens": 657, "context_cl100k_tokens": 5262, "confirmed_packet_source_lines": 494, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 942, "candidate_windows": 2, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__52c152ba", "recall": 1.0, "precision": 0.022267206477732792, "f1": 0.04356435643564356 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__52c152ba", "arm": "late_physical4096", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.08232690000000001, "messages_api_usd": 0.054892, "pipeline_usd_allocated": 0.1372189, "provider_input_tokens": 10323, "provider_output_tokens": 680, "context_cl100k_tokens": 6202, "confirmed_packet_source_lines": 588, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 942, "candidate_windows": 2, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__52c152ba", "recall": 1.0, "precision": 0.01870748299319728, "f1": 0.03672787979966611 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__48e5846a", "arm": "native_packet", "repo": "mui/material-ui", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.203075, "messages_api_usd": 0.06914, "pipeline_usd_allocated": 0.272215, "provider_input_tokens": 13620, "provider_output_tokens": 733, "context_cl100k_tokens": 8163, "confirmed_packet_source_lines": 643, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4089, "candidate_windows": 133, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__48e5846a", "recall": 0.6666666666666666, "precision": 0.1150855365474339, "f1": 0.1962864721485411 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__48e5846a", "arm": "late_physical4096", "repo": "mui/material-ui", "repo_url": "https://github.com/mui/material-ui.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.203075, "messages_api_usd": 0.095496, "pipeline_usd_allocated": 0.29857100000000003, "provider_input_tokens": 19779, "provider_output_tokens": 819, "context_cl100k_tokens": 12213, "confirmed_packet_source_lines": 1005, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4089, "candidate_windows": 133, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__48e5846a", "recall": 0.7027027027027027, "precision": 0.07761194029850746, "f1": 0.13978494623655913 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__72488b59", "arm": "native_packet", "repo": "iamkun/dayjs", "repo_url": "https://github.com/iamkun/dayjs.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.10366740000000002, "messages_api_usd": 0.028192, "pipeline_usd_allocated": 0.13185940000000002, "provider_input_tokens": 3958, "provider_output_tokens": 618, "context_cl100k_tokens": 2376, "confirmed_packet_source_lines": 155, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4093, "candidate_windows": 13, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__72488b59", "recall": 0.9705882352941176, "precision": 0.21153846153846154, "f1": 0.34736842105263155 } }, { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__72488b59", "arm": "late_physical4096", "repo": "iamkun/dayjs", "repo_url": "https://github.com/iamkun/dayjs.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.10366740000000002, "messages_api_usd": 0.053248, "pipeline_usd_allocated": 0.1569154, "provider_input_tokens": 9892, "provider_output_tokens": 684, "context_cl100k_tokens": 6438, "confirmed_packet_source_lines": 484, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4093, "candidate_windows": 13, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__javascript__maintenance__bugfix__72488b59", "recall": 0.9705882352941176, "precision": 0.06804123711340206, "f1": 0.12716763005780343 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__2fea5f6b", "arm": "native_packet", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1008184, "messages_api_usd": 0.043152, "pipeline_usd_allocated": 0.1439704, "provider_input_tokens": 7113, "provider_output_tokens": 735, "context_cl100k_tokens": 4252, "confirmed_packet_source_lines": 328, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 3719, "candidate_windows": 9, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__2fea5f6b", "recall": 0.6818181818181818, "precision": 0.27439024390243905, "f1": 0.391304347826087 } }, { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__2fea5f6b", "arm": "late_physical4096", "repo": "cli/cli", "repo_url": "https://github.com/cli/cli.git", "language": "go", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1008184, "messages_api_usd": 0.064528, "pipeline_usd_allocated": 0.1653464, "provider_input_tokens": 13132, "provider_output_tokens": 600, "context_cl100k_tokens": 7922, "confirmed_packet_source_lines": 620, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 3719, "candidate_windows": 9, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__go__maintenance__bugfix__2fea5f6b", "recall": 0.6818181818181818, "precision": 0.14516129032258066, "f1": 0.2393617021276596 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__4f685b8e", "arm": "native_packet", "repo": "pydata/xarray", "repo_url": "https://github.com/pydata/xarray.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.08013029999999999, "messages_api_usd": 0.032464, "pipeline_usd_allocated": 0.11259429999999998, "provider_input_tokens": 5006, "provider_output_tokens": 622, "context_cl100k_tokens": 3053, "confirmed_packet_source_lines": 230, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4096, "candidate_windows": 81, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__4f685b8e", "recall": 0.85, "precision": 0.4434782608695652, "f1": 0.5828571428571429 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__4f685b8e", "arm": "late_physical4096", "repo": "pydata/xarray", "repo_url": "https://github.com/pydata/xarray.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.08013029999999999, "messages_api_usd": 0.059756, "pipeline_usd_allocated": 0.1398863, "provider_input_tokens": 11479, "provider_output_tokens": 692, "context_cl100k_tokens": 7131, "confirmed_packet_source_lines": 573, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4096, "candidate_windows": 81, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__4f685b8e", "recall": 0.85, "precision": 0.17801047120418848, "f1": 0.29437229437229434 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__4d090fc6", "arm": "native_packet", "repo": "serverless/serverless", "repo_url": "https://github.com/serverless/serverless.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.07807080000000001, "messages_api_usd": 0.043564, "pipeline_usd_allocated": 0.12163480000000002, "provider_input_tokens": 7561, "provider_output_tokens": 666, "context_cl100k_tokens": 3599, "confirmed_packet_source_lines": 298, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 2577, "candidate_windows": 5, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__4d090fc6", "recall": 0.5186335403726708, "precision": 0.5604026845637584, "f1": 0.5387096774193548 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__4d090fc6", "arm": "late_physical4096", "repo": "serverless/serverless", "repo_url": "https://github.com/serverless/serverless.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.07807080000000001, "messages_api_usd": 0.06156, "pipeline_usd_allocated": 0.1396308, "provider_input_tokens": 11785, "provider_output_tokens": 721, "context_cl100k_tokens": 6146, "confirmed_packet_source_lines": 511, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 2577, "candidate_windows": 5, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__4d090fc6", "recall": 0.6211180124223602, "precision": 0.3913894324853229, "f1": 0.48019207683073234 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__b110ef6c", "arm": "native_packet", "repo": "keras-team/keras", "repo_url": "https://github.com/keras-team/keras.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1402427, "messages_api_usd": 0.059408, "pipeline_usd_allocated": 0.19965070000000001, "provider_input_tokens": 11852, "provider_output_tokens": 600, "context_cl100k_tokens": 6234, "confirmed_packet_source_lines": 536, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4090, "candidate_windows": 25, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__b110ef6c", "recall": 0.7761194029850746, "precision": 0.38733705772811916, "f1": 0.5167701863354037 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__b110ef6c", "arm": "late_physical4096", "repo": "keras-team/keras", "repo_url": "https://github.com/keras-team/keras.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.1402427, "messages_api_usd": 0.086416, "pipeline_usd_allocated": 0.2266587, "provider_input_tokens": 18484, "provider_output_tokens": 624, "context_cl100k_tokens": 10306, "confirmed_packet_source_lines": 869, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4090, "candidate_windows": 25, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__b110ef6c", "recall": 0.9738805970149254, "precision": 0.3, "f1": 0.45869947275922673 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__ab946da6", "arm": "native_packet", "repo": "internetarchive/openlibrary", "repo_url": "https://github.com/internetarchive/openlibrary.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.16063159999999999, "messages_api_usd": 0.050252, "pipeline_usd_allocated": 0.21088359999999998, "provider_input_tokens": 9318, "provider_output_tokens": 649, "context_cl100k_tokens": 5707, "confirmed_packet_source_lines": 471, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4087, "candidate_windows": 119, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__ab946da6", "recall": 0.3125, "precision": 0.16913319238900634, "f1": 0.2194787379972565 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__ab946da6", "arm": "late_physical4096", "repo": "internetarchive/openlibrary", "repo_url": "https://github.com/internetarchive/openlibrary.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.16063159999999999, "messages_api_usd": 0.073564, "pipeline_usd_allocated": 0.2341956, "provider_input_tokens": 15351, "provider_output_tokens": 608, "context_cl100k_tokens": 9638, "confirmed_packet_source_lines": 807, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4087, "candidate_windows": 119, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__ab946da6", "recall": 0.328125, "precision": 0.103831891223733, "f1": 0.15774647887323945 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__0e4346f7", "arm": "native_packet", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.13849519999999998, "messages_api_usd": 0.0673, "pipeline_usd_allocated": 0.20579519999999998, "provider_input_tokens": 13485, "provider_output_tokens": 668, "context_cl100k_tokens": 8270, "confirmed_packet_source_lines": 661, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4090, "candidate_windows": 167, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__0e4346f7", "recall": 1.0, "precision": 0.1573373676248109, "f1": 0.27189542483660134 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__0e4346f7", "arm": "late_physical4096", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.13849519999999998, "messages_api_usd": 0.09288, "pipeline_usd_allocated": 0.2313752, "provider_input_tokens": 19875, "provider_output_tokens": 669, "context_cl100k_tokens": 12354, "confirmed_packet_source_lines": 991, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4090, "candidate_windows": 167, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__0e4346f7", "recall": 1.0, "precision": 0.10494450050454086, "f1": 0.18995433789954339 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__1e195cd6", "arm": "native_packet", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.32426489999999997, "messages_api_usd": 0.115248, "pipeline_usd_allocated": 0.4395129, "provider_input_tokens": 23872, "provider_output_tokens": 988, "context_cl100k_tokens": 14551, "confirmed_packet_source_lines": 1175, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4095, "candidate_windows": 152, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__1e195cd6", "recall": 1.0, "precision": 0.02127659574468085, "f1": 0.04166666666666667 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__1e195cd6", "arm": "late_physical4096", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.32426489999999997, "messages_api_usd": 0.139512, "pipeline_usd_allocated": 0.46377689999999994, "provider_input_tokens": 30463, "provider_output_tokens": 883, "context_cl100k_tokens": 18601, "confirmed_packet_source_lines": 1560, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4095, "candidate_windows": 152, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__1e195cd6", "recall": 1.0, "precision": 0.016025641025641024, "f1": 0.031545741324921134 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__708894b2", "arm": "native_packet", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.2209847, "messages_api_usd": 0.10468, "pipeline_usd_allocated": 0.32566470000000003, "provider_input_tokens": 22365, "provider_output_tokens": 761, "context_cl100k_tokens": 13220, "confirmed_packet_source_lines": 823, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4094, "candidate_windows": 143, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__708894b2", "recall": 0.8532110091743119, "precision": 0.22600243013365734, "f1": 0.3573487031700288 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__708894b2", "arm": "late_physical4096", "repo": "microsoft/vscode", "repo_url": "https://github.com/microsoft/vscode.git", "language": "typescript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.2209847, "messages_api_usd": 0.132712, "pipeline_usd_allocated": 0.3536967, "provider_input_tokens": 29343, "provider_output_tokens": 767, "context_cl100k_tokens": 17308, "confirmed_packet_source_lines": 1111, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4094, "candidate_windows": 143, "skipped": [], "metrics": { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__708894b2", "recall": 0.8990825688073395, "precision": 0.17641764176417643, "f1": 0.2949586155003763 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__ae8b362a", "arm": "native_packet", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.2149613, "messages_api_usd": 0.094288, "pipeline_usd_allocated": 0.3092493, "provider_input_tokens": 19492, "provider_output_tokens": 816, "context_cl100k_tokens": 12144, "confirmed_packet_source_lines": 983, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 90, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__ae8b362a", "recall": 0.6009615384615384, "precision": 0.3814852492370295, "f1": 0.46670815183571873 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__ae8b362a", "arm": "late_physical4096", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.2149613, "messages_api_usd": 0.120396, "pipeline_usd_allocated": 0.33535729999999997, "provider_input_tokens": 25894, "provider_output_tokens": 841, "context_cl100k_tokens": 16222, "confirmed_packet_source_lines": 1351, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4092, "candidate_windows": 90, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__ae8b362a", "recall": 0.7596153846153846, "precision": 0.35085122131754254, "f1": 0.4799999999999999 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__45da4482", "arm": "native_packet", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.20329819999999998, "messages_api_usd": 0.08966, "pipeline_usd_allocated": 0.2929582, "provider_input_tokens": 19000, "provider_output_tokens": 683, "context_cl100k_tokens": 11050, "confirmed_packet_source_lines": 637, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4095, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__45da4482", "recall": 0.352, "precision": 0.13793103448275862, "f1": 0.1981981981981982 } }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__45da4482", "arm": "late_physical4096", "repo": "ponylang/ponyc", "repo_url": "https://github.com/ponylang/ponyc.git", "language": "c", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.20329819999999998, "messages_api_usd": 0.115264, "pipeline_usd_allocated": 0.3185622, "provider_input_tokens": 25331, "provider_output_tokens": 697, "context_cl100k_tokens": 15119, "confirmed_packet_source_lines": 990, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4095, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__45da4482", "recall": 0.4, "precision": 0.10090817356205853, "f1": 0.16116035455278002 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__d3133dfb", "arm": "native_packet", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.2547614, "messages_api_usd": 0.121788, "pipeline_usd_allocated": 0.37654940000000003, "provider_input_tokens": 26897, "provider_output_tokens": 710, "context_cl100k_tokens": 17191, "confirmed_packet_source_lines": 1435, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4081, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__d3133dfb", "recall": 0.6, "precision": 0.05006954102920723, "f1": 0.09242618741976892 } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__d3133dfb", "arm": "late_physical4096", "repo": "clap-rs/clap", "repo_url": "https://github.com/clap-rs/clap.git", "language": "rust", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.2547614, "messages_api_usd": 0.14784, "pipeline_usd_allocated": 0.4026014, "provider_input_tokens": 33335, "provider_output_tokens": 725, "context_cl100k_tokens": 21262, "confirmed_packet_source_lines": 1744, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4081, "candidate_windows": 200, "skipped": [], "metrics": { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__d3133dfb", "recall": 0.6, "precision": 0.0412135088723526, "f1": 0.0771290840921264 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__09eb0d6d", "arm": "native_packet", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.24191710000000002, "messages_api_usd": 0.097012, "pipeline_usd_allocated": 0.33892910000000004, "provider_input_tokens": 19973, "provider_output_tokens": 856, "context_cl100k_tokens": 12083, "confirmed_packet_source_lines": 661, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4088, "candidate_windows": 200, "skipped": [ { "file": "test/user.js", "overlong_source_lines": [ 71 ] } ], "metrics": { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__09eb0d6d", "recall": 0.3786610878661088, "precision": 0.27382753403933435, "f1": 0.3178226514486392 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__09eb0d6d", "arm": "late_physical4096", "repo": "NodeBB/NodeBB", "repo_url": "https://github.com/NodeBB/NodeBB.git", "language": "javascript", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.24191710000000002, "messages_api_usd": 0.1202, "pipeline_usd_allocated": 0.3621171, "provider_input_tokens": 26490, "provider_output_tokens": 712, "context_cl100k_tokens": 16165, "confirmed_packet_source_lines": 952, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4088, "candidate_windows": 200, "skipped": [ { "file": "test/user.js", "overlong_source_lines": [ 71 ] } ], "metrics": { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__09eb0d6d", "recall": 0.4686192468619247, "precision": 0.23529411764705882, "f1": 0.3132867132867133 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1ce4c38e", "arm": "native_packet", "repo": "astropy/astropy", "repo_url": "https://github.com/astropy/astropy.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.10534910000000001, "messages_api_usd": 0.04968, "pipeline_usd_allocated": 0.15502910000000003, "provider_input_tokens": 9870, "provider_output_tokens": 510, "context_cl100k_tokens": 5516, "confirmed_packet_source_lines": 482, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4086, "candidate_windows": 27, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1ce4c38e", "recall": 0.8333333333333334, "precision": 0.05186721991701245, "f1": 0.09765625 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1ce4c38e", "arm": "late_physical4096", "repo": "astropy/astropy", "repo_url": "https://github.com/astropy/astropy.git", "language": "python", "status": "ok", "confirmed_input": true, "response_complete": true, "stop_reason": "end_turn", "native_status": "ok", "native_budget_terminal": false, "native_sdk_usd_allocated": 0.10534910000000001, "messages_api_usd": 0.076164, "pipeline_usd_allocated": 0.1815131, "provider_input_tokens": 16131, "provider_output_tokens": 582, "context_cl100k_tokens": 9600, "confirmed_packet_source_lines": 781, "paid_message_requests": 1, "free_count_requests": 1, "extra_envelope_tokens": 4086, "candidate_windows": 27, "skipped": [], "metrics": { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1ce4c38e", "recall": 0.8333333333333334, "precision": 0.03201024327784891, "f1": 0.06165228113440197 } } ], "gates": { "all32_native_terminal_receipts_accepted": true, "all64_source_consumers_confirmed": true, "exactly_one_paid_message_each": true, "all_usage_known": true, "all_source_model_route_runtime_receipts_verified": true, "all32_selected_positions_retained": true, "gold_only_read_after_terminal_source_prediction_freeze": true, "all_official_scores_valid": true, "positive_lower_recall_interval": true, "within_reservation": true }, "independent_gate_passed": true, "new_native_sdk_usage_usd": 5.716993799999999, "new_messages_api_usage_usd": 5.466732000000001, "new_combined_api_usage_usd": 11.183725800000001, "encoding_phase_new_api_usd": 8.6173772, "parent_pre_encoding_api_usd": 2.5663486000000004, "native_sdk_sessions": 32, "paid_message_requests": 64, "free_token_count_requests": 64, "allocated_pipeline_usd": { "native_packet": 8.072169799999998, "late_physical4096": 8.8285498 }, "known_program_api_usage_value_usd": 58.663770729999996, "complete_review_responses": 64, "native_raw_status_counts": { "ok": 32 }, "consumer_stop_reason_counts": { "end_turn": 64 }, "unknown_usage_cases": 0, "original_execution_operational_gate_passed": false, "prelabel_encoding_amendment_sha256": "e4ac3346613cc02d0d32e4b699a7b866168a992ad1f4c8894a08042daae6357f", "model_retries": 0, "runtime_gold_empty": true, "annotation_materialization_sha256": "f4833cc425b22ae3b79ebd688b94fb85930937aef9db9ef048f6c11aae6684a9", "repair_calls": 0, "official500_started": false, "new32_sealed": true, "both_old32_still_sealed": true, "next": "Seal amended32 after scoring and retain original operational failure. Only passing the unchanged all32 Recall/interval/source/usage gates permits a separately measured full500 source+solver/readiness/budget plan. No automatic repair or full500 calls; both older32 stay sealed.", "sdk_terminal_usage_models": [ "claude-haiku-5-5", "claude-opus-5-5" ], "auxiliary_usage_already_in_sdk_total_usd": { "claude-haiku-5-5": 0.006325600000000001 }, "observed_candidate_vs_native_allocated_cost_difference_usd": 0.7563800000000018, "observed_candidate_vs_native_cost_difference_per_task_usd": 0.023636875000000057 }, "post_late_source_validation_untouched_pool": { "verified_candidates_remaining": 292, "new32_sealed": true, "both_older32_cohorts_sealed": true, "development_population_unchanged": 105, "source": "Post-sealing metadata-only audit of verified/full datasets,excluding known IDs,original IDs and exact source commits across aliases. No further issue/source/annotation fields read.", "policy_tuning_on_validation_allowed": false, "remaining_verified_per_language": { "python": 194, "javascript": 36, "typescript": 40, "c": 3, "go": 18, "rust": 1 }, "remaining_default_only_candidates": 623, "additional_source_identity_exclusions_beyond32": 0 }, "verified500_measured_cost_and_submission_plan": { "scope": "Historical small-sample arithmetic and formal submission requirements, retained without changing prior estimates. Current canonical500 context-only dispatch protocol and live state are in verified500_context_execution; no quality or repair result is implied.", "remaining_authorized_api_usage_usd": 40.90601126999999, "unweighted500_arithmetic_only": { "api_investigation_plus_candidate_review_usd": 137.94609062499995, "api_investigation_plus_two_reviews_usd": 174.74571562500003, "api_investigation_plus_candidate_batch_review_usd": 113.63705937499999, "max_investigation_plus_candidate_standard_review_api_usd": 48.618062499999986, "max_investigation_plus_candidate_batch_review_api_usd": 24.309031249999993 }, "conditional_reuse_of_exact32_context_results": { "eligible_context_positions": 32, "remaining_positions": 468, "reuse_condition": "Preserve exactly the context policy/model/profile/source-credit semantics and audit all per-case versions/receipts into the full-run protocol. Reuse every matching selected case,never select by its score. Solver outcomes remain unavailable.", "no_completed_call_rebilled": true, "unweighted_remaining468_api_investigation_and_standard_review_usd": 129.11754082499996, "unweighted_remaining468_api_investigation_and_batch_review_usd": 106.36428757499999, "unweighted_remaining468_max_investigation_and_batch_review_api_usd": 22.753253249999997, "other_historical_outputs": "Audit profile/source/terminal parity separately before any reuse. No speculative reuse credit or output replacement in the reservation." }, "covered_language_reweighting": { "per_language": { "c": { "sample_size": 4, "verified500_tasks": 23, "mean_investigation_sdk_usd": 0.29317295, "mean_candidate_standard_review_usd": 0.136832, "covered_tasks_investigation_api_projection_usd": 6.74297785, "covered_tasks_review_standard_api_projection_usd": 3.147136 }, "go": { "sample_size": 4, "verified500_tasks": 40, "mean_investigation_sdk_usd": 0.106035625, "mean_candidate_standard_review_usd": 0.070348, "covered_tasks_investigation_api_projection_usd": 4.241425, "covered_tasks_review_standard_api_projection_usd": 2.8139199999999995 }, "javascript": { "sample_size": 4, "verified500_tasks": 60, "mean_investigation_sdk_usd": 0.14359365000000002, "mean_candidate_standard_review_usd": 0.07922699999999999, "covered_tasks_investigation_api_projection_usd": 8.615619, "covered_tasks_review_standard_api_projection_usd": 4.75362 }, "python": { "sample_size": 12, "verified500_tasks": 266, "mean_investigation_sdk_usd": 0.14817168333333333, "mean_candidate_standard_review_usd": 0.08565766666666667, "covered_tasks_investigation_api_projection_usd": 39.41366776666666, "covered_tasks_review_standard_api_projection_usd": 22.784939333333334 }, "rust": { "sample_size": 4, "verified500_tasks": 20, "mean_investigation_sdk_usd": 0.2284958, "mean_candidate_standard_review_usd": 0.126583, "covered_tasks_investigation_api_projection_usd": 4.569916, "covered_tasks_review_standard_api_projection_usd": 2.53166 }, "typescript": { "sample_size": 4, "verified500_tasks": 66, "mean_investigation_sdk_usd": 0.21343537500000004, "mean_candidate_standard_review_usd": 0.107926, "covered_tasks_investigation_api_projection_usd": 14.086734750000003, "covered_tasks_review_standard_api_projection_usd": 7.123116 } }, "covered_tasks": 475, "uncovered_languages_no_cost_imputation": { "cpp": 10, "java": 15 }, "covered_tasks_investigation_api_usd": 77.67034036666666, "covered_tasks_standard_review_api_usd": 43.154391333333336, "covered_tasks_batch_review_api_usd": 21.577195666666668, "note": "Only475 tasks in six observed languages. No extrapolation for25 C++/Java tasks; sample counts4 or12 are small. Max subscription has usage limits and an unmeasured elapsed-time/quota budget. A nominal SDK dollar estimate is not a Max cash bill." }, "batch_option": { "primary_source": "https://platform.claude.com/docs/en/build-with-claude/batch-processing", "source_checked_date": "2026-10-17", "opus55_input_output_usd_per_million": [ 2, 10 ], "standard_discount_factor": 0.5, "may_take_up_to_hours": 24, "directly_batchable_component": "Independent final packet Messages after native source investigations are complete. Interactive SDK/tool turns do not automatically receive the batch discount.", "implementation_and_receipt_parity_verified": true, "batch_submitted": true, "verification_scope": "52local tests plus one real8-item Batch from4fresh Max native investigations on known development tasks. This verifies the small route,not whole500 source/token/quota/budget readiness.", "engineering_api_usage_usd": 0.430218 }, "max_route_option": { "local_supported_cli_subscription_auth_confirmed": true, "quota_available_verified": false, "same_policy_profile_source_receipt_parity_required": true, "new_api_route_fallback": false, "frozen32_route_unchanged": true, "small_development_native_profile_verified": true, "verified_Max_native_positions": 4 }, "full500_budget_gate_passed": false, "leaderboard_repository_revision": "a40dfa6b16f14cafd2b1cc46faf0969b38ef03b9", "required_artifacts": [ "metadata.json", "results.json", "README.md", "logs/", "trajectories/" ], "actual_issue_resolution_outcomes_required_for_pass_at_1": true, "per_committed_file_cap_mib": 50, "external_links_or_lfs_pointers_suffice": false, "open_gates": [ "Finishonly257unclaimed natives,reuse242sourcequalifiedprefixes/56pairs,retain1originalfailedzero-source/unmeteredcase. Preserveallfailedstrictgates andSDKrejectedactions/errors;eachrejectedaction0source,onlyexactSDKrejectionsqualify.", "BeforeonepaidBatchcreate,countALL443exacteligiblecandidateinputs andfulloutputcap/margin/headroom<=same40USDreserve;all500positionsretained.", "Sealall500predictions/rawreceiptsbeforeGold;reportfixed500RPF1andvalidity/errors separately,discloseoperationalqualifications/knowncases,no repair/freshindependent500claim.", "Real mini SWE-agent/native resolution outcomes and supported native grading for Pass@1.", "Complete original raw trajectories,failed attempts,evaluation and outcome archives committed within official file limits." ], "new_model_calls": 0, "new_api_cost_usd": 0, "official500_started": false, "context500_open_gates": [ "Finishonly257unclaimed natives,reuse242sourcequalifiedprefixes/56pairs,retain1originalfailedzero-source/unmeteredcase. Preserveallfailedstrictgates andSDKrejectedactions/errors;eachrejectedaction0source,onlyexactSDKrejectionsqualify.", "BeforeonepaidBatchcreate,countALL443exacteligiblecandidateinputs andfulloutputcap/margin/headroom<=same40USDreserve;all500positionsretained.", "Sealall500predictions/rawreceiptsbeforeGold;reportfixed500RPF1andvalidity/errors separately,discloseoperationalqualifications/knowncases,no repair/freshindependent500claim." ], "official_submission_additional_gates": [ "Real mini SWE-agent/native resolution outcomes and supported native grading for Pass@1.", "Complete original raw trajectories,failed attempts,evaluation and outcome archives committed within official file limits." ], "context_quality_can_be_evaluated_before_repairs": true, "current_conditional_reuse_of_exact56_context_results": { "scope": "Historical56-pair reuse scenario,retained without new rebilling. Current quota-order continuation additionally reuses36accepted initial500prefixes,retains1failedzero-source case andcontinues375unstarted IDs;see verified500_context_execution.", "eligible_context_positions": 56, "remaining_positions": 444, "unweighted_remaining444_max_investigation_and_batch_review_api_usd": 21.58641975, "estimate_basis": "Same32-task sample arithmetic,not representative forecast or whole-run reservation.", "repairs_required_before_context_evaluation": false } }, "max_batch_development4_route_verification": { "scope": "Four known development tasks,complete Max-native -> unchanged late physical4K compiler -> two-arm Batch source-review engineering. No confirmatory quality/retrieval scoring,repairs,native tests,unseen IDs or official500. This checks the cheaper execution route,not a new context strategy.", "protocol_sha256": "4cb3ebd851d30369e90b57d94d7d825d8ff5328ffb6f5cb202b87e236e71d2b7", "implementation_commit": "4143a04507c512b5839a028e6cf70b0dfb0ba4c7", "gates": { "all4_Max_native_receipts_accepted": true, "all8_batch_inputs_confirmed": true, "all_main_models_Opus55": true, "all_usage_known": true, "within_2_api_reservation": true, "one_create_intent_and_acknowledged_identity": true, "all8_positions_retained": true, "source_runtime_prompt_receipts_verified": true, "no_independent_tasks_or_Gold_used": true }, "engineering_gate_passed": true, "batch_rows": [ { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "arm": "late_physical4096", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.044702, "standard_equivalent_same_actual_tokens_usd": 0.089404, "provider_usage": { "input_tokens": 18331, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 804, "output_tokens_details": { "thinking_tokens": 375 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 11561, "source_lines": 910 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "late_physical4096", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.027894, "standard_equivalent_same_actual_tokens_usd": 0.055788, "provider_usage": { "input_tokens": 10602, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 669, "output_tokens_details": { "thinking_tokens": 267 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 6512, "source_lines": 549 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "arm": "native_packet", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.070916, "standard_equivalent_same_actual_tokens_usd": 0.141832, "provider_usage": { "input_tokens": 30678, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 956, "output_tokens_details": { "thinking_tokens": 529 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 18629, "source_lines": 1642 }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "arm": "native_packet", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.03236, "standard_equivalent_same_actual_tokens_usd": 0.06472, "provider_usage": { "input_tokens": 11945, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 847, "output_tokens_details": { "thinking_tokens": 444 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 7474, "source_lines": 565 }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "arm": "late_physical4096", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.083304, "standard_equivalent_same_actual_tokens_usd": 0.166608, "provider_usage": { "input_tokens": 37032, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 924, "output_tokens_details": { "thinking_tokens": 475 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 22697, "source_lines": 1972 }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "arm": "native_packet", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.013972, "standard_equivalent_same_actual_tokens_usd": 0.027944, "provider_usage": { "input_tokens": 4206, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 556, "output_tokens_details": { "thinking_tokens": 182 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 2428, "source_lines": 206 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "arm": "late_physical4096", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.08508, "standard_equivalent_same_actual_tokens_usd": 0.17016, "provider_usage": { "input_tokens": 38230, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 862, "output_tokens_details": { "thinking_tokens": 389 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 22294, "source_lines": 1415 }, { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "arm": "native_packet", "type": "succeeded", "confirmed_input": true, "response_complete": true, "api_usage_usd": 0.07199, "standard_equivalent_same_actual_tokens_usd": 0.14398, "provider_usage": { "input_tokens": 31735, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0, "cache_creation": { "ephemeral_5m_input_tokens": 0, "ephemeral_1h_input_tokens": 0 }, "output_tokens": 852, "output_tokens_details": { "thinking_tokens": 394 }, "service_tier": "batch", "inference_geo": "global", "speed": "standard" }, "stop_reason": "end_turn", "actual_model": "claude-opus-5-5", "context_tokens": 18211, "source_lines": 1138 } ], "new_max_nominal_sdk_usd": 1.2485557, "new_batch_api_usage_usd": 0.430218, "known_program_api_usage_value_usd": 59.09398873000001, "new_max_sdk_sessions": 4, "new_batch_message_requests": 8, "free_token_count_requests": 8, "batch_create_posts": 1, "models": { "claude-opus-5-5": 8 }, "stop_reasons": { "end_turn": 8 }, "complete_review_responses": 8, "standard_equivalent_same_actual_batch_tokens_usd": 0.860436, "quality_scores_computed": false, "repair_calls": 0, "official500_started": false, "all_three32_remain_sealed": true, "unknown_batch_usage": false, "next": "Passing this small engineering study permits exact-source/count/quota and separately scoped verified500 context planning,not automatic500/repairs. Reuse every exact-profile output independent of score;never replace failed attempts.", "native_receipts": [ { "instance_id": "Multi-SWE-Bench__c__maintenance__bugfix__0f495035", "status": "ok", "terminal": "success", "accepted_budget_terminal": false, "models": [ "claude-opus-5-5" ], "routes": [ "none" ], "nominal_sdk_usd": 0.4879477, "known_usage": true, "accepted": true, "raw_error_preserved": false, "exit_code": 0, "sha256": "1f90b86d3d940988e298c292d5060e1d8e55deaa2bdba3f476257ab04e2b457a", "language": "c", "unconfirmed_tool_replies_excluded": 0, "sdk_model_usage": { "claude-haiku-5-5": { "inputTokens": 1694, "outputTokens": 21, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.0001799, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-5-5", "provider": "firstParty", "costBasis": "list" }, "claude-opus-5-5": { "inputTokens": 20, "outputTokens": 5866, "cacheReadInputTokens": 209239, "cacheCreationInputTokens": 41065, "webSearchRequests": 0, "costUSD": 0.4877678, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 716, "canonicalModel": "claude-opus-5-5", "provider": "firstParty", "costBasis": "list" } } }, { "instance_id": "Multi-SWE-Bench__rust__maintenance__bugfix__16860730", "status": "ok", "terminal": "success", "accepted_budget_terminal": false, "models": [ "claude-opus-5-5" ], "routes": [ "none" ], "nominal_sdk_usd": 0.4467497, "known_usage": true, "accepted": true, "raw_error_preserved": false, "exit_code": 0, "sha256": "95432dd2c761bd2b4301ec25cb07bdbaa6b836427b90b0b79452774447ad070e", "language": "rust", "unconfirmed_tool_replies_excluded": 0, "sdk_model_usage": { "claude-haiku-5-5": { "inputTokens": 2570, "outputTokens": 21, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.0002675, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-5-5", "provider": "firstParty", "costBasis": "list" }, "claude-opus-5-5": { "inputTokens": 20, "outputTokens": 4939, "cacheReadInputTokens": 213511, "cacheCreationInputTokens": 38115, "webSearchRequests": 0, "costUSD": 0.4464822, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 446, "canonicalModel": "claude-opus-5-5", "provider": "firstParty", "costBasis": "list" } } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1409977d", "status": "ok", "terminal": "success", "accepted_budget_terminal": false, "models": [ "claude-opus-5-5" ], "routes": [ "none" ], "nominal_sdk_usd": 0.09340180000000001, "known_usage": true, "accepted": true, "raw_error_preserved": false, "exit_code": 0, "sha256": "32e2353fe5973be77a84317fff3b483a7bef26aac194c46517480d80011a2b94", "language": "python", "unconfirmed_tool_replies_excluded": 0, "sdk_model_usage": { "claude-haiku-5-5": { "inputTokens": 1439, "outputTokens": 21, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.0001544, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-5-5", "provider": "firstParty", "costBasis": "list" }, "claude-opus-5-5": { "inputTokens": 10, "outputTokens": 1897, "cacheReadInputTokens": 25937, "cacheCreationInputTokens": 6260, "webSearchRequests": 0, "costUSD": 0.0932474, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 297, "canonicalModel": "claude-opus-5-5", "provider": "firstParty", "costBasis": "list" } } }, { "instance_id": "Multi-SWE-Bench__typescript__maintenance__bugfix__7c41997b", "status": "ok", "terminal": "success", "accepted_budget_terminal": false, "models": [ "claude-opus-5-5" ], "routes": [ "none" ], "nominal_sdk_usd": 0.2204565, "known_usage": true, "accepted": true, "raw_error_preserved": false, "exit_code": 0, "sha256": "b859001b2ed010a4f80cfbd1e5c09308acbee423de4896f4dad4a98986787fbe", "language": "typescript", "unconfirmed_tool_replies_excluded": 0, "sdk_model_usage": { "claude-haiku-5-5": { "inputTokens": 1191, "outputTokens": 26, "cacheReadInputTokens": 0, "cacheCreationInputTokens": 0, "webSearchRequests": 0, "costUSD": 0.0001321, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 0, "canonicalModel": "claude-haiku-5-5", "provider": "firstParty", "costBasis": "list" }, "claude-opus-5-5": { "inputTokens": 14, "outputTokens": 3795, "cacheReadInputTokens": 65562, "cacheCreationInputTokens": 16407, "webSearchRequests": 0, "costUSD": 0.2203244, "contextWindow": 1000000, "maxOutputTokens": 128000, "thinkingTokens": 671, "canonicalModel": "claude-opus-5-5", "provider": "firstParty", "costBasis": "list" } } } ], "runtime_gold_empty": true, "new_dataset_annotation_or_reference_patch_columns_loaded": false, "focused_local_tests": 52, "batch_mock_tests": 29, "protocol_api_reservation_usd": 2, "count_forecast_with_headroom_usd": 0.5903436000000001 }, "verified500_source_preparation_audit": { "scope": "Read-only current source coverage from existing native-source manifest records only. No whole500 model run,clones,source mutation,issue/source text,Gold,reference patches or labels read. Absence here does not prove absence from every unrelated local cache. No availability-based benchmark task filtering.", "dataset_sha256": "e9dcfd504cbfb849ac815a79040c793d0d92f94eecc9b5a4ee3e1445a2f8a791", "metadata_columns_read": [ "instance_id", "original_inst_id", "repo", "repo_url", "base_commit", "language" ], "selected_population": 500, "ready_count": 119, "not_verified_count": 381, "languages": { "python": { "total": 266, "ready": 26 }, "javascript": { "total": 60, "ready": 15 }, "go": { "total": 40, "ready": 15 }, "typescript": { "total": 66, "ready": 15 }, "c": { "total": 23, "ready": 15 }, "cpp": { "total": 10, "ready": 7 }, "java": { "total": 15, "ready": 11 }, "rust": { "total": 20, "ready": 15 } }, "unique_verified_checkout_count": 119, "verified_allocated_kib": 8765296, "elapsed_s": 3.9584085941314697, "new_model_calls": 0, "new_api_cost_usd": 0, "full500_model_run_started": false, "gate_passed": false, "source_capacity_point_arithmetic": { "cached_ready": 119, "unverified": 381, "missing_with_repo_sample": 358, "missing_with_only_language_sample": { "python": 10, "go": 4, "typescript": 2, "java": 3, "javascript": 1, "rust": 3 }, "new_checkout_point_bytes": 45716062208, "cache_free_bytes": 42643873792, "source_staging_minimum_free_bytes": 12884901888, "projected_free_after_point_bytes": -3072188416, "new_api_cost_usd": 0 }, "inventory_stage": "Initial119-source inventory, retained as historical evidence; current parent readiness is recorded in verified500_source_gitlinks_continuation." }, "verified500_metric_mapping_audit": { "scope": "Pinned official source/schema audit only,no task examples,dataset labels or model calls.", "leaderboard_source": "https://github.com/ContextBench/contextbench.github.io/blob/a40dfa6b16f14cafd2b1cc46faf0969b38ef03b9/src/lib/leaderboard.ts", "leaderboard_sha256": "cdbdf039c5909321d4b96f2d169015d9e95ffab542cc72ca934516e39ce7ff1e", "official_recall_key": "performance.line.recall", "official_f1_key": "performance.line.f1", "default_rank_key": "line_recall", "current_frozen_source_metric_matches_board_level": true, "evaluator_source": "https://github.com/EuniAI/ContextBench/blob/1436c28a8eb95496da4ea69ad458b9f8a8eb7d61/contextbench/evaluate.py", "evaluator_sha256": "a07e211ce72ee155c69c1e3f194526d559f1b1b152528153fa9ff0082d52b22d", "evaluator_levels": [ "file", "symbol", "span", "line" ], "website_required_levels": [ "file", "block", "line" ], "block_mapping_verified": true, "open_gate": "Full500 still requires complete population,original artifacts and real native resolution outcomes. Metric-level and macro aggregation naming bridge is verified.", "new_model_calls": 0, "new_api_cost_usd": 0, "block_source_metric": "symbol", "aggregation": "Per-task metrics,then arithmetic mean;F1 per-task before averaging. Derive per-task F1 from R/P only when absent from the pinned evaluator raw field;never harmonic mean of macro Precision and Recall.", "block_definition_primary_source": "https://arxiv.org/html/2602.05892v3", "canonical_aggregation_source": "https://github.com/ContextBench/contextbench.github.io/blob/a40dfa6b16f14cafd2b1cc46faf0969b38ef03b9/submissions/2026-09-12_openjiuwen-jiuwenswarm/aggregate.py", "canonical_aggregation_sha256": "2843c5a449d25fa31073fa88845b935cd7e7783a3dff874f326b5afdcf722903" }, "sealed32_canonical_aggregation_audit": { "scope": "Posthoc aggregation/schema verification on already sealed32 official cumulative raw reports. No model calls,new annotation read,primary gate change,policy tuning or leaderboard claim.", "field_mapping": { "file": "file", "block": "symbol", "line": "line" }, "inputs_sha256": { "verification/contextbench-late-source-independent32-encoding-20261017/evaluation/native_packet/cumulative/metrics.json": "0f0ceeba76b7ece63b8e6870e2beb0ba51524d751b9643bd799b90aab4e431e5", "verification/contextbench-late-source-independent32-encoding-20261017/evaluation/late_physical4096/cumulative/metrics.json": "cef2e447c5f0a3931153e98ee6a0f707f0e330260c2ea5826f6781431fd56cf1" }, "task_macro": { "native_packet": { "file": { "recall": 0.8542277167277167, "precision": 0.3598378981699415, "f1": 0.45409263801712313, "evaluated_cases": 32 }, "block": { "recall": 0.8283589354768236, "precision": 0.17203410213938755, "f1": 0.2575456817266009, "evaluated_cases": 32 }, "line": { "recall": 0.7494390839904986, "precision": 0.22728208253297102, "f1": 0.3013120498589982, "evaluated_cases": 32 } }, "late_physical4096": { "file": { "recall": 0.8542277167277167, "precision": 0.3598378981699415, "f1": 0.45409263801712313, "evaluated_cases": 32 }, "block": { "recall": 0.8359978243657125, "precision": 0.13077616018772778, "f1": 0.2062729156718217, "evaluated_cases": 32 }, "line": { "recall": 0.79092782160193, "precision": 0.14834402468472668, "f1": 0.2231531457948828, "evaluated_cases": 32 } } }, "matches_frozen_primary_line_R_P_F1": true, "new_api_cost_usd": 0 }, "verified500_source_preparation_execution": { "scope": "Terminal source-only preparation of all canonical verified500 positions. Source readiness is not a benchmark model attempt or source input consumed by a model. Failed and unstarted positions stay in the500 denominator and are never replaced.", "runtime_commit": "ccb49d538f35e76042b9d85772459d080332fb5d", "protocol_sha256": "38c3127c29e1aee43d90ee2928266356a0b1f70c00ca52e428ee71386b2124b8", "execution_sha256": "e0ecb20a9583b3b732958ac7487c9313500b4e696ce0f6d0aafcb01c5f30101c", "manifest_sha256": "25e43f72e6e6dd79b9a810e995774a93d478d05992350008130ca469a97c3362", "state": "finished", "population": 500, "ready": 452, "errors": 3, "not_started": 45, "source_gate_passed": false, "methods": { "audited_existing_link": 119, "borrowed_git_objects": 318, "fresh_shallow_fetch": 15, "error": 3 }, "new_allocated_bytes": 29848334336, "cache_free_bytes": 12795011072, "minimum_free_bytes_before_dispatch": 12884901888, "workers": 2, "stop_reason": "source disk floor before dispatch", "languages": { "c": { "ready": 23 }, "cpp": { "ready": 10 }, "go": { "ready": 40 }, "java": { "ready": 15 }, "javascript": { "ready": 60 }, "rust": { "ready": 20 }, "typescript": { "ready": 63 }, "python": { "ready": 221, "error": 3 } }, "new_model_calls": 0, "new_api_usage_usd": 0, "all_three32_sealed": true, "full500_inference_or_repairs_started": false, "timestamp_utc": "2026-10-17T17:31:38.696869+00:00" }, "verified500_existing_packet_reuse_audit": { "scope": "Read-only code/profile explanation after strict reuse audit rejected24 whole-file hashes. Retain that failed strict gate. No labels, quality scores, compiler/window/model changes, task filtering or new calls. Semantic eligibility requires all other strict checks plus a proven exact two-keyword postprocessing-only change; never allow an arbitrary implementation difference.", "plan_sha256": "186940ccdc4d20ac5fe1e344173302e6410625baa1c196dfcb8ef56fcbb847cc", "audit_sha256": "40c5a49b78cd24d909364c23fdeb319008cd7202f8bb8711981efdbf91a2f107", "strict_audit_sha256": "0bee18a3e73294668d312153fd7a003dcf2a41d50ad46cf2e7e35587742ac3ca", "original_strict_whole_file_gate_passed": false, "strict_whole_file_eligible_positions": 32, "postprocessing_encoding_only_difference_positions": 24, "positions_audited": 56, "eligible_positions": 56, "development_positions": 24, "independent_positions": 32, "remaining_positions": 444, "all56_gate_passed": true, "inactive_native_grep_tokens_disclosed": [ 2048, 4096 ], "duplicate_Max_engineering_positions_add_new_unique_positions": false, "new_quality_scores": false, "new_model_calls": 0, "new_api_usage_usd": 0, "whole500_run_authorized": false, "remaining444_max_batch_arithmetic_only_usd": 21.58641975 }, "verified500_source_gitlinks_continuation": { "scope": "All canonical500 exact parent snapshots, no model/API/issue/annotation/repair work. Preserve the original strict source failures and unstarted positions; this is a new source-only correctness continuation, never a model retry or reclassification of the original strict source gate. Unmapped gitlinks stay empty and explicitly unavailable; do not fabricate child bytes or URLs, exclude tasks or alter gold denominators.", "runtime_commit": "50519a3a33030b5181675c5b3993949a5468d75a", "protocol_sha256": "87b5c0232d4818442e72b56d47719935c64c73591dde076dac358b1c9a4a102e", "execution_sha256": "441f85a0e90b482140de384031652c8036e96fb8550d8a9be7c1d6dd57b880e3", "manifest_sha256": "e4e5579cd7d63342439dc2727123c01d7ace256862b47b3b4ad135eb9cf23b97", "parent_ready_count": 500, "source_errors": 0, "not_started": 0, "parent_snapshot_gate_passed": true, "complete_source_availability_gate_passed": false, "cases_with_unavailable_gitlinks": 3, "unavailable_gitlinks": { "SWE-PolyBench__python__maintenance__bugfix__52a91dd8": [ { "commit": "7e525a3b91ddf2cc0d9b3add5928460449c1d600", "path": "docs/modules/indexes/document_loaders/examples/example_data/test_repo1", "reason": "unregistered_gitlink" } ], "SWE-PolyBench__python__maintenance__bugfix__aaf309d2": [ { "commit": "7e525a3b91ddf2cc0d9b3add5928460449c1d600", "path": "docs/modules/indexes/document_loaders/examples/example_data/test_repo1", "reason": "unregistered_gitlink" } ], "SWE-PolyBench__python__maintenance__bugfix__b52bbc24": [ { "commit": "7e525a3b91ddf2cc0d9b3add5928460449c1d600", "path": "docs/modules/indexes/document_loaders/examples/example_data/test_repo1", "reason": "unregistered_gitlink" } ] }, "original_strict_source_gate_passed": false, "original_ready": 452, "original_errors": 3, "original_unstarted": 45, "original_floor_gib": 12, "continuation_floor_gib": 8, "continuation_workers": 2, "owned_nondonor_object_repacks": 16, "saved_object_bytes": 1958182912, "cache_free_bytes": 9670189056, "new_model_calls": 0, "new_api_usage_usd": 0, "new_quality_scores": false, "all_three32_sealed": true, "full500_inference_or_repairs_started": false }, "verified500_owned_object_compaction": { "state": "finished", "owned_nondonor_borrowers": 16, "saved_object_bytes": 1958182912, "source_refs_config_alternates_and_donors_unchanged": true, "native_connectivity_audits_passed": true, "new_model_calls": 0, "new_api_usage_usd": 0 }, "verified500_context_execution": { "observed_at_utc": "2026-10-17T20:28:44.876467+00:00", "state": "running", "phase": "SDK_rejected_action_source_receipt_continuation", "runtime_commit": "1b0ad5ac835ce4ce8a2ed46e0b6fa47ce27ade58", "protocol_sha256": "4daea3d3d39ce1fd9a1f06e653c32edfaeaac899cba415aab3750d083a9f7b0d", "population": 500, "qualified_source_prefixes_reused": 242, "complete_context_pairs_reused": 56, "original_failed_zero_source_positions": 1, "original_unknown_Max_terminal_cases": 1, "capacity_parent_completed": 53, "capacity_parent_strict_attempt_gate_accepted": 52, "capacity_parent_original_strict_gate_remains_failed": true, "capacity_parent_source_receipts_qualified": 53, "SDK_rejected_unavailable_actions": 1, "rejected_action_source_credit": 0, "tool_alias_rewrite_or_execution_added": false, "current_native_positions": 257, "current_completed": 122, "current_accepted": 122, "current_remaining": 135, "all500_native_terminal_records": 365, "new_candidate_consumers": 443, "known_new_Max_nominal_sdk_usd": 46.58501650000001, "new_Max_terminal_metering_gaps": 1, "new_API_usage_usd": 0, "known_prior_API_usage_usd": 59.09398873, "same_API_reservation_usd": 40, "API_ceiling_usd": 100, "new_Batch_or_count_requests": 0, "new_quality_scores": true, "new_repairs": 0, "fresh_independent500": false, "all_three32_cohorts_sealed": true, "source_model_prompt_compiler_budget_unchanged": true, "new_native_baseline_scored_cases": 122, "new_candidate_scored_cases": 0, "full500_paired_score_available": false, "original_all500_before_first_new_Gold_schedule_followed": false, "interim_protocol_sha256": "853d663ca2afcd72ddda4af9e525835cd7a9422fc967c57b65009652637f5a7d" }, "verified500_additional_native_prefix_reuse": { "scope": "Metadata/profile/source/code-path audit of every matching original prefix, without new calls, quality scoring, label access or favorable-case selection. Original strict empty advertised-catalog requirement remains failed; separate exact validated-reference catalog equality qualifies all32. All sealed study quality conclusions remain unchanged.", "additional_qualified_prefixes": 32, "complete_pairs_reused": 56, "total_native_prefixes_reused": 88, "fresh_native_positions": 412, "new_candidate_consumers": 444, "native_packet_consumers_omitted": 444, "baseline": "Confirmed cumulative native source already consumed by investigator. All56 original native_packet source unions equal native prefix; new candidate adds one actual final input. Do not claim matched final-review reasoning or repair gain, or credit an unsent native packet.", "new_model_calls": 0, "new_api_usage_usd": 0 }, "max_included_only_runtime_guard": { "runtime_commit": "1b0ad5ac835ce4ce8a2ed46e0b6fa47ce27ade58", "default_enabled": false, "flag": "--max-included-only", "requires": "Existing first-party Max auth,init apiKeySource=none and explicit included-only RateLimitEvent before terminal case acceptance. Keep pending raw output and drain terminal usage when event order differs; reject an explicit unsafe event immediately. A missing event rejects after metering,without source credit.", "does_not_change_sdk_query_options": true, "first_request_may_precede_quota_event": true, "is_provider_billing_cap": false, "changes_account_settings": false, "model_api_fallback": false, "sdk_boundary_tests_passed": 36, "relevant_unit_tests_passed": 142, "billed_tests_run": false }, "verified500_quota_order_amendment": { "scope": "Prospective operational correctness correction before any new500 annotation access,not a quality/policy/model/task replacement. Original frozen cf2f2e47/e8f6b675 execution stays failed at37attempts/36accepted/1unmetered interruption. Continue only375unstarted canonical IDs under21456878/1b0ad5ac; retain failed case withzero source in botharms andall500denominators.", "cause": "AssistantMessage arrived before RateLimitEvent; the original optional guard interrupted before terminal usage. Missing-event validation now occurs after draining the SDK stream; explicit unsafe rate events still stop immediately.", "SDK_options_query_AST_equal": true, "sdk_boundary_tests_passed": 36, "relevant_local_tests_passed": 142, "new_model_retries": 0, "new_gold_reference_or_test_reads": 0, "original_unknown_Max_terminal_usage_not_replaced_by_partial_tokens": true }, "verified500_reused_raw_archive_preparation": { "scope": "Partial local raw archive of fixed terminal reuse only,not complete500 or a formal submission. No new model calls/annotations/quality scoring. Preserve exact original bytes,including the interrupted native attempt;local source paths remain publication candidates and are not automatically committed/published.", "native_attempts": 125, "native_interrupted_attempts_retained": 1, "already_consumed_standard_messages": 112, "archived_original_files": 464, "gzip_tar_stream_parts": 3, "compressed_bytes": 7964272, "largest_part_bytes": 4258086, "per_part_limit_bytes": 52428800, "all_original_byte_hashes_roundtrip_verified": true, "plan_sha256": "d0f75f708b757873a6447283b842b4307d0f24e0371bd87431f7fd4395dc0f0d", "archive_index_sha256": "0bcebd5001f64e98723c016748607619e7956465b9e2fce84e7290a8c001cfa0", "verification_sha256": "0c47338d9ee7024798395f913ce1f52223a6363619d75d649baf90d6a1f0fa8a", "full500_official_artifact_gate_passed": false, "new_model_calls": 0, "new_annotation_reads": 0, "new_API_usage_usd": 0, "canonical_evaluator_source_binding": { "positions": 500, "retained_source_aliases": 119, "all_targets_match_recorded_sources": true, "audit_sha256": "483b26e1498eb0e3e584ed06263ee20065e5322dc06a3dce65bb592043a323a5" } }, "verified500_capacity_continuation": { "scope": "Prospective artifact-capacity continuation only,all model/source/compiler/consumer/metricsettings unchanged. Preserve parent65accepted raw/state andoriginal1failedzero-source case; no modelretry ornew annotations.", "parent_protocol_sha256": "2145687887c3aa4def05a6cdc9417718ec8d4030e209e5d86d98c44e712eb92d", "parent_exec_terminal_exit": 1, "parent_completed_known_usage_accepted": 65, "parent_unreconciled_intents": 0, "capacity_protocol_sha256": "ac478f8956de831e7776212176f1dac617e4e2888a54362348c57fa3d926253a", "fresh_native_positions": 310, "prefixes_reused": 189, "new_candidate_inputs": 443, "artifact_admission_floor_bytes": 4294967296, "large_volume_output_and_TMPDIR_only": true, "source_repositories_unmoved": true, "first_actual_case_terminal_and_included_only_verified": true, "source_prefix_roundtrips_passed": 189, "prior_full_context_pair_source_parity_passed": 56, "new_model_failures_due_to_capacity_stop": 0, "new_API_usage_usd": 0, "new_quality_scores": false }, "verified500_raw_archive_bundle": { "scope": "Partial original raw bundle:243 disjoint native attempts,including1original interrupted zero-source case and1original strict rejected-action gate retained;112standard consumer attempts. Source qualification contributes zero source for the SDK-rejected action. Three independently roundtrip-verified archive groups. Current257native,new443Batch,quality/outcome/fullpublication remain open.", "native_positions": 243, "failed_zero_source_native_positions": 1, "capacity_original_strict_attempt_gate_failures": 1, "consumed_standard_message_attempts": 112, "verified_file_members_with_cross_group_duplicates": 796, "distinct_member_names": 713, "gzip_parts": 7, "compressed_bytes": 11568526, "largest_part_bytes": 4258086, "all_parts_below_50MiB": true, "overlapping_members_byte_identical": true, "previous_partial_index_sha256": "eac01d64a6202950e9c9fd2315a829f4619deb0ec79ae0f992ba938e1f93941f", "full500_submission_ready": false, "new_model_calls": 0, "new_annotation_reads": 0, "new_API_usage_usd": 0, "partial_bundle_index_sha256": "98a7ce52d0d89afaad397197b1507c5bf0b42c5fb6e6f3ce66963cd8b70648ef", "additional53_archive_index_sha256": "53e344b9689332b53b24ea54bfc544fdef2c562a14cd275d4a3637e866e02bae", "additional53_verification_sha256": "be35301bb064fc1ad31219c3c5d05cbe4842ed371d7c0c6f75eec8bf3ca5af17", "new_quality_scores": false }, "verified500_rejected_action_source_qualification": { "scope": "Label-blind source receipt qualification only. Keep original strict no-unavailable-attempt gate failed; no modelretry,toolalias rewrite,newcapability ornewsource from SDK-rejected action. Registered/executed source boundary remains read/grep/glob;other confirmed inputs survive an unsuccessful model action.", "original_strict_gate_passed": false, "source_qualification_before_Gold": true, "qualified_parent_streams": 53, "strict_parent_accepted": 52, "SDK_error": "No such tool available: mcp__source:read", "SDK_error_is_error": true, "registered_source_tools": [ "mcp__source__read", "mcp__source__grep", "mcp__source__glob" ], "rejected_action_source_credit": 0, "no_alias_mapping_or_extra_execution": true, "native_geometry_equal_without_rejected_action": true, "compiled_packet_bytes_equal_without_rejected_action": true, "adversarial_boundaries_rejected": 5, "all53_streams_checked": true, "all242_prefixes_replay": true, "all56_prior_pairs_replay": true, "qualification_sha256": "aee417dd780a427e9738b6088dcbaa35cecc6c04af43e02c8a9cc844909736e4", "new_model_calls": 0, "new_Gold_or_quality_reads": false }, "verified500_native_interim_baseline": { "population": 122, "native_prefix_only": true, "candidate_comparison_available": false, "failure_accounted_snapshot_macro": { "file": { "recall": 0.8031858348726338, "precision": 0.3562485256717264, "f1": 0.44327477356524747 }, "block": { "recall": 0.8178157043431532, "precision": 0.26038132580387824, "f1": 0.35072697039911144 }, "line": { "recall": 0.7680559332548911, "precision": 0.30752983776988435, "f1": 0.3899852469923677 } }, "official_valid_cases": 122, "official_failed_cases": [], "language_counts": { "python": 58, "javascript": 34, "go": 4, "typescript": 26 }, "repository_clusters": 16, "known_baseline_Max_nominal_sdk_usd": 27.262828100000007, "Max_nominal_is_cash_bill": false, "new_model_calls": 0, "new_API_usage_usd": 0, "native_model_retries": 0, "repair_calls": 0, "Pass_at_1": null, "fresh_independent_sample": false, "whole500_result": false, "original_all500_before_first_new_Gold_schedule_followed": false, "protocol_sha256": "853d663ca2afcd72ddda4af9e525835cd7a9422fc967c57b65009652637f5a7d", "predictions_sha256": "241eb07350b327ef47892d4f2edacfabf04fee55e3761e9b0c90bd9f3f140e24", "metrics_sha256": "9ae946101239c6b7ca7451e76ccba05ff2281e5bb999095d97897b6cfccc995f", "annotation_access_sha256": "7d408207be56209592e9f65338ec507c8c217939e59b8433c377dbd7bbd11043", "scope": "Descriptive interim native-prefix baseline on every completed case in one atomic running-state snapshot. Completion-order subset, not random, not all500, not independent validation, not candidate gain or repair success. All snapshot source predictions sealed before their new Gold access.", "frozen_at_utc": "2026-10-17T20:28:44.876467+00:00", "analysis_sha256": "9fb214d5663b6747c52d9a1d1856a90201d666271db8ba9f317e628ffd5fc207", "selection": "All completed source-accepted rows at a single prelabel snapshot; no quality selection.", "cases": [ { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__1eaeabf4", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 1.0, "f1": 1.0 }, "block": { "recall": 1.0, "precision": 0.8, "f1": 0.888888888888889 }, "line": { "recall": 0.26744186046511625, "precision": 0.6216216216216216, "f1": 0.37398373983739835 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__ac127191", "official_valid": true, "error": null, "file": { "recall": 0.6, "precision": 0.75, "f1": 0.6666666666666665 }, "block": { "recall": 0.25, "precision": 0.3, "f1": 0.2727272727272727 }, "line": { "recall": 0.48104956268221577, "precision": 0.4714285714285714, "f1": 0.4761904761904762 } }, { "instance_id": "SWE-Bench-Verified__python__maintenance__bugfix__3c7d37b7", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "line": { "recall": 0.6994047619047619, "precision": 0.8131487889273357, "f1": 0.752 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__71253eae", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.27586206896551724, "f1": 0.4324324324324324 }, "block": { "recall": 1.0, "precision": 0.10256410256410256, "f1": 0.18604651162790695 }, "line": { "recall": 0.7659574468085106, "precision": 0.16997167138810199, "f1": 0.2782071097372489 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__53ca6a30", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.6666666666666666, "f1": 0.8 }, "block": { "recall": 1.0, "precision": 0.9411764705882353, "f1": 0.9696969696969697 }, "line": { "recall": 0.9926739926739927, "precision": 0.8138138138138138, "f1": 0.8943894389438944 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__39a9746b", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.12, "f1": 0.20689655172413793 }, "block": { "recall": 0.7142857142857143, "precision": 0.1282051282051282, "f1": 0.21739130434782608 }, "line": { "recall": 0.749498997995992, "precision": 0.27540500736377027, "f1": 0.4028002154011847 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__7c4a11fb", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.6666666666666666, "f1": 0.8 }, "block": { "recall": 1.0, "precision": 0.4117647058823529, "f1": 0.5833333333333334 }, "line": { "recall": 0.9041095890410958, "precision": 0.38596491228070173, "f1": 0.5409836065573771 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__40a717e5", "official_valid": true, "error": null, "file": { "recall": 0.6923076923076923, "precision": 0.42857142857142855, "f1": 0.5294117647058824 }, "block": { "recall": 1.0, "precision": 0.425, "f1": 0.5964912280701754 }, "line": { "recall": 0.847457627118644, "precision": 0.29093931837073983, "f1": 0.43316831683168316 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__739b23db", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2777777777777778, "f1": 0.4347826086956522 }, "block": { "recall": 0.8421052631578947, "precision": 0.23529411764705882, "f1": 0.367816091954023 }, "line": { "recall": 0.5592105263157895, "precision": 0.1501766784452297, "f1": 0.23676880222841223 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__64fffdfa", "official_valid": true, "error": null, "file": { "recall": 0.8333333333333334, "precision": 0.45454545454545453, "f1": 0.5882352941176471 }, "block": { "recall": 0.6666666666666666, "precision": 0.07407407407407407, "f1": 0.13333333333333333 }, "line": { "recall": 0.5734265734265734, "precision": 0.15922330097087378, "f1": 0.2492401215805471 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__ac8400d9", "official_valid": true, "error": null, "file": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "block": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "line": { "recall": 0.0, "precision": 0.0, "f1": 0.0 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__6efcf999", "official_valid": true, "error": null, "file": { "recall": 0.631578947368421, "precision": 0.4444444444444444, "f1": 0.5217391304347826 }, "block": { "recall": 0.4782608695652174, "precision": 0.12643678160919541, "f1": 0.2 }, "line": { "recall": 0.32941176470588235, "precision": 0.10467289719626169, "f1": 0.1588652482269504 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__cf01f471", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.5, "f1": 0.5 }, "block": { "recall": 0.9090909090909091, "precision": 0.4166666666666667, "f1": 0.5714285714285715 }, "line": { "recall": 0.82, "precision": 0.47766990291262135, "f1": 0.603680981595092 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__18d7bbbc", "official_valid": true, "error": null, "file": { "recall": 0.8181818181818182, "precision": 0.28125, "f1": 0.41860465116279066 }, "block": { "recall": 0.8444444444444444, "precision": 0.3220338983050847, "f1": 0.4662576687116564 }, "line": { "recall": 0.5942211055276382, "precision": 0.45133587786259544, "f1": 0.5130151843817787 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__20f502e0", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.5, "f1": 0.5 }, "block": { "recall": 0.7692307692307693, "precision": 0.20408163265306123, "f1": 0.3225806451612903 }, "line": { "recall": 0.9021164021164021, "precision": 0.4577181208053691, "f1": 0.6073018699910953 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__d3c56cc5", "official_valid": true, "error": null, "file": { "recall": 0.875, "precision": 0.5185185185185185, "f1": 0.6511627906976744 }, "block": { "recall": 0.8095238095238095, "precision": 0.3695652173913043, "f1": 0.5074626865671641 }, "line": { "recall": 0.8959854014598541, "precision": 0.30649188514357056, "f1": 0.4567441860465117 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__5b2cf9bb", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3, "f1": 0.4615384615384615 }, "block": { "recall": 0.8421052631578947, "precision": 0.2962962962962963, "f1": 0.4383561643835616 }, "line": { "recall": 0.8790697674418605, "precision": 0.11659469463294263, "f1": 0.2058823529411765 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__84fdadf0", "official_valid": true, "error": null, "file": { "recall": 0.6666666666666666, "precision": 0.15384615384615385, "f1": 0.25 }, "block": { "recall": 1.0, "precision": 0.09302325581395349, "f1": 0.1702127659574468 }, "line": { "recall": 0.9318181818181818, "precision": 0.03996101364522417, "f1": 0.07663551401869158 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__e2b70931", "official_valid": true, "error": null, "file": { "recall": 0.9090909090909091, "precision": 0.7692307692307693, "f1": 0.8333333333333333 }, "block": { "recall": 0.7878787878787878, "precision": 0.6842105263157895, "f1": 0.732394366197183 }, "line": { "recall": 0.6301369863013698, "precision": 0.4316996871741397, "f1": 0.5123762376237623 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__2464eadb", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.16666666666666666, "f1": 0.2857142857142857 }, "block": { "recall": 1.0, "precision": 0.03571428571428571, "f1": 0.0689655172413793 }, "line": { "recall": 1.0, "precision": 0.21284403669724772, "f1": 0.3509833585476551 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__c1f435de", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.42857142857142855, "f1": 0.4615384615384615 }, "block": { "recall": 0.8571428571428571, "precision": 0.16666666666666666, "f1": 0.27906976744186046 }, "line": { "recall": 0.7098976109215017, "precision": 0.2950354609929078, "f1": 0.41683366733466937 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__64469377", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.42857142857142855, "f1": 0.5454545454545454 }, "block": { "recall": 0.9473684210526315, "precision": 0.3157894736842105, "f1": 0.47368421052631576 }, "line": { "recall": 0.456, "precision": 0.21923076923076923, "f1": 0.2961038961038961 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__607fc4ff", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.2631578947368421, "f1": 0.3448275862068966 }, "block": { "recall": 0.7307692307692307, "precision": 0.16666666666666666, "f1": 0.2714285714285714 }, "line": { "recall": 0.6551724137931034, "precision": 0.22478206724782068, "f1": 0.334724153917478 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__4c132bfd", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "line": { "recall": 0.9925373134328358, "precision": 0.21212121212121213, "f1": 0.34954007884362687 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__d9d9c3a6", "official_valid": true, "error": null, "file": { "recall": 0.5714285714285714, "precision": 0.27586206896551724, "f1": 0.37209302325581395 }, "block": { "recall": 0.7777777777777778, "precision": 0.5, "f1": 0.6086956521739131 }, "line": { "recall": 0.6861538461538461, "precision": 0.5234741784037559, "f1": 0.5938748335552596 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__450d75e0", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3076923076923077, "f1": 0.47058823529411764 }, "block": { "recall": 1.0, "precision": 0.23809523809523808, "f1": 0.3846153846153846 }, "line": { "recall": 0.9254658385093167, "precision": 0.2450657894736842, "f1": 0.38751625487646296 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__7c2efe8a", "official_valid": true, "error": null, "file": { "recall": 0.6666666666666666, "precision": 0.12121212121212122, "f1": 0.20512820512820512 }, "block": { "recall": 1.0, "precision": 0.20987654320987653, "f1": 0.346938775510204 }, "line": { "recall": 0.8379888268156425, "precision": 0.20920502092050208, "f1": 0.33482142857142855 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__512d556b", "official_valid": true, "error": null, "file": { "recall": 0.6666666666666666, "precision": 0.2222222222222222, "f1": 0.3333333333333333 }, "block": { "recall": 1.0, "precision": 0.16666666666666666, "f1": 0.2857142857142857 }, "line": { "recall": 0.7228915662650602, "precision": 0.22304832713754646, "f1": 0.34090909090909094 } }, { "instance_id": "SWE-Bench-Pro__go__maintenance__bugfix__4df06349", "official_valid": true, "error": null, "file": { "recall": 0.8888888888888888, "precision": 0.5714285714285714, "f1": 0.6956521739130435 }, "block": { "recall": 0.8, "precision": 0.29850746268656714, "f1": 0.43478260869565216 }, "line": { "recall": 0.7882882882882883, "precision": 0.14767932489451477, "f1": 0.24875621890547264 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__31f13b61", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.23076923076923078, "f1": 0.3529411764705882 }, "block": { "recall": 0.5581395348837209, "precision": 0.25, "f1": 0.34532374100719426 }, "line": { "recall": 0.5071283095723014, "precision": 0.38131699846860645, "f1": 0.43531468531468526 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__1cae51cc", "official_valid": true, "error": null, "file": { "recall": 0.46153846153846156, "precision": 0.11764705882352941, "f1": 0.1875 }, "block": { "recall": 0.15789473684210525, "precision": 0.046875, "f1": 0.07228915662650602 }, "line": { "recall": 0.19455252918287938, "precision": 0.08361204013377926, "f1": 0.11695906432748536 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__36b8c9ee", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.2962962962962963, "f1": 0.43243243243243246 }, "block": { "recall": 0.3953488372093023, "precision": 0.18888888888888888, "f1": 0.2556390977443609 }, "line": { "recall": 0.17435897435897435, "precision": 0.255, "f1": 0.20710659898477157 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__62badbbf", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.13636363636363635, "f1": 0.24000000000000002 }, "block": { "recall": 1.0, "precision": 0.22448979591836735, "f1": 0.36666666666666664 }, "line": { "recall": 0.9527027027027027, "precision": 0.14461538461538462, "f1": 0.25111308993766696 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__81c5c6fa", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5384615384615384, "f1": 0.7000000000000001 }, "block": { "recall": 0.9615384615384616, "precision": 0.4166666666666667, "f1": 0.5813953488372093 }, "line": { "recall": 0.7791932059447984, "precision": 0.5381231671554252, "f1": 0.6366001734605378 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__0c4f8d13", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.8, "f1": 0.888888888888889 }, "block": { "recall": 1.0, "precision": 0.6451612903225806, "f1": 0.7843137254901961 }, "line": { "recall": 0.911340206185567, "precision": 0.5538847117794486, "f1": 0.6890101325019486 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__5ec14acf", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "block": { "recall": 1.0, "precision": 0.078125, "f1": 0.14492753623188406 }, "line": { "recall": 1.0, "precision": 0.07242990654205607, "f1": 0.13507625272331153 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__07bb383a", "official_valid": true, "error": null, "file": { "recall": 0.8333333333333334, "precision": 0.2, "f1": 0.3225806451612903 }, "block": { "recall": 0.7, "precision": 0.175, "f1": 0.27999999999999997 }, "line": { "recall": 0.6762075134168157, "precision": 0.5220994475138122, "f1": 0.5892439594699922 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__6ebb54dc", "official_valid": true, "error": null, "file": { "recall": 0.3333333333333333, "precision": 0.2857142857142857, "f1": 0.30769230769230765 }, "block": { "recall": 0.9310344827586207, "precision": 0.421875, "f1": 0.5806451612903226 }, "line": { "recall": 0.967487684729064, "precision": 0.49898373983739835, "f1": 0.6583975863224941 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__942d0b14", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.6, "f1": 0.5454545454545454 }, "block": { "recall": 0.6, "precision": 0.2608695652173913, "f1": 0.36363636363636365 }, "line": { "recall": 0.4384858044164038, "precision": 0.25112917795844625, "f1": 0.31935669155657664 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__fcb506a5", "official_valid": true, "error": null, "file": { "recall": 0.9, "precision": 0.47368421052631576, "f1": 0.6206896551724138 }, "block": { "recall": 0.8363636363636363, "precision": 0.5348837209302325, "f1": 0.652482269503546 }, "line": { "recall": 0.8061336254107339, "precision": 0.5631216526396328, "f1": 0.6630630630630631 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__462b957d", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.5294117647058824, "f1": 0.6923076923076924 }, "line": { "recall": 0.8112449799196787, "precision": 0.6516129032258065, "f1": 0.7227191413237924 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__5fdd12f6", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2222222222222222, "f1": 0.3636363636363636 }, "block": { "recall": 1.0, "precision": 0.20754716981132076, "f1": 0.34375000000000006 }, "line": { "recall": 1.0, "precision": 0.3252032520325203, "f1": 0.49079754601226994 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__95b4abc1", "official_valid": true, "error": null, "file": { "recall": 0.7272727272727273, "precision": 0.16666666666666666, "f1": 0.2711864406779661 }, "block": { "recall": 0.8245614035087719, "precision": 0.23857868020304568, "f1": 0.37007874015748027 }, "line": { "recall": 0.6142433234421365, "precision": 0.20679320679320679, "f1": 0.30941704035874434 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__e31ec45c", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3793103448275862, "f1": 0.5499999999999999 }, "block": { "recall": 0.5869565217391305, "precision": 0.34615384615384615, "f1": 0.43548387096774194 }, "line": { "recall": 0.5851063829787234, "precision": 0.3142857142857143, "f1": 0.4089219330855019 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__1bb5c0b0", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.6, "f1": 0.6666666666666665 }, "block": { "recall": 0.8461538461538461, "precision": 0.28205128205128205, "f1": 0.4230769230769231 }, "line": { "recall": 0.7529761904761905, "precision": 0.32603092783505155, "f1": 0.45503597122302164 } }, { "instance_id": "SWE-Bench-Pro__javascript__maintenance__bugfix__5c001746", "official_valid": true, "error": null, "file": { "recall": 0.5833333333333334, "precision": 0.5833333333333334, "f1": 0.5833333333333334 }, "block": { "recall": 0.5714285714285714, "precision": 0.4, "f1": 0.47058823529411764 }, "line": { "recall": 0.5756676557863502, "precision": 0.44241733181299886, "f1": 0.5003223726627982 } }, { "instance_id": "SWE-Bench-Pro__python__maintenance__bugfix__e6efdd87", "official_valid": true, "error": null, "file": { "recall": 0.7272727272727273, "precision": 0.5333333333333333, "f1": 0.6153846153846153 }, "block": { "recall": 0.8333333333333334, "precision": 0.33707865168539325, "f1": 0.48 }, "line": { "recall": 0.6891891891891891, "precision": 0.39666666666666667, "f1": 0.5035260930888575 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__1ad00901", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2, "f1": 0.33333333333333337 }, "block": { "recall": 1.0, "precision": 0.1111111111111111, "f1": 0.19999999999999998 }, "line": { "recall": 1.0, "precision": 0.07575757575757576, "f1": 0.14084507042253522 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__9d623e73", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.4, "f1": 0.5714285714285715 }, "block": { "recall": 1.0, "precision": 0.17857142857142858, "f1": 0.30303030303030304 }, "line": { "recall": 0.9085923217550275, "precision": 0.41624790619765495, "f1": 0.5709362435381965 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__e21304a6", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.07692307692307693, "f1": 0.13333333333333336 }, "block": { "recall": 0.6666666666666666, "precision": 0.10526315789473684, "f1": 0.18181818181818182 }, "line": { "recall": 0.927710843373494, "precision": 0.26101694915254237, "f1": 0.40740740740740733 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__d3babcfe", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3125, "f1": 0.47619047619047616 }, "block": { "recall": 1.0, "precision": 0.18571428571428572, "f1": 0.3132530120481928 }, "line": { "recall": 0.958974358974359, "precision": 0.13689604685212298, "f1": 0.23959000640614994 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__78039f77", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.3, "f1": 0.4285714285714285 }, "block": { "recall": 0.5625, "precision": 0.25, "f1": 0.34615384615384615 }, "line": { "recall": 0.53156146179402, "precision": 0.367816091954023, "f1": 0.43478260869565216 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__d1cb6b19", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.2857142857142857, "f1": 0.4210526315789473 }, "block": { "recall": 0.8461538461538461, "precision": 0.2558139534883721, "f1": 0.3928571428571428 }, "line": { "recall": 0.5354107648725213, "precision": 0.6116504854368932, "f1": 0.5709969788519638 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__eb1411ba", "official_valid": true, "error": null, "file": { "recall": 0.8333333333333334, "precision": 0.5555555555555556, "f1": 0.6666666666666667 }, "block": { "recall": 0.8, "precision": 0.3, "f1": 0.4363636363636363 }, "line": { "recall": 0.6423357664233577, "precision": 0.2678843226788432, "f1": 0.3780880773361976 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__b9ce6a70", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.16666666666666666, "f1": 0.2857142857142857 }, "block": { "recall": 1.0, "precision": 0.5625, "f1": 0.72 }, "line": { "recall": 0.9675675675675676, "precision": 0.7937915742793792, "f1": 0.8721071863580999 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__c6d3e230", "official_valid": true, "error": null, "file": { "recall": 0.6666666666666666, "precision": 0.5, "f1": 0.5714285714285715 }, "block": { "recall": 0.5714285714285714, "precision": 0.3333333333333333, "f1": 0.4210526315789474 }, "line": { "recall": 0.40634920634920635, "precision": 0.5224489795918368, "f1": 0.45714285714285713 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__11e78a8d", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.5, "f1": 0.6153846153846154 }, "block": { "recall": 0.8888888888888888, "precision": 0.5714285714285714, "f1": 0.6956521739130435 }, "line": { "recall": 0.7219512195121951, "precision": 0.8757396449704142, "f1": 0.7914438502673797 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__9ca0e415", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.3333333333333333, "f1": 0.4 }, "block": { "recall": 0.6, "precision": 0.5294117647058824, "f1": 0.5625 }, "line": { "recall": 0.4301675977653631, "precision": 0.2883895131086142, "f1": 0.3452914798206278 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__8cc426dd", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.75, "f1": 0.75 }, "block": { "recall": 0.9523809523809523, "precision": 0.47619047619047616, "f1": 0.634920634920635 }, "line": { "recall": 0.9692832764505119, "precision": 0.5579567779960707, "f1": 0.7082294264339151 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__7e3e2bb9", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.18181818181818182, "f1": 0.2962962962962963 }, "block": { "recall": 0.631578947368421, "precision": 0.16216216216216217, "f1": 0.25806451612903225 }, "line": { "recall": 0.8923444976076556, "precision": 0.18483647175421208, "f1": 0.30623973727422005 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__63513ada", "official_valid": true, "error": null, "file": { "recall": 0.4, "precision": 1.0, "f1": 0.5714285714285715 }, "block": { "recall": 0.6923076923076923, "precision": 0.9, "f1": 0.7826086956521738 }, "line": { "recall": 0.7516129032258064, "precision": 0.4794238683127572, "f1": 0.5854271356783919 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__c26fd1ab", "official_valid": true, "error": null, "file": { "recall": 0.16666666666666666, "precision": 0.14285714285714285, "f1": 0.15384615384615383 }, "block": { "recall": 0.2, "precision": 0.07692307692307693, "f1": 0.1111111111111111 }, "line": { "recall": 0.15763546798029557, "precision": 0.47761194029850745, "f1": 0.23703703703703707 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__e647c8ce", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.17647058823529413, "f1": 0.2608695652173913 }, "block": { "recall": 0.5, "precision": 0.15, "f1": 0.23076923076923075 }, "line": { "recall": 0.1456953642384106, "precision": 0.19298245614035087, "f1": 0.16603773584905657 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__27e1903f", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.125, "f1": 0.2222222222222222 }, "block": { "recall": 0.75, "precision": 0.2, "f1": 0.31578947368421056 }, "line": { "recall": 0.8230088495575221, "precision": 0.36328125, "f1": 0.5040650406504065 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__1e29a469", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.42857142857142855, "f1": 0.6 }, "block": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "line": { "recall": 1.0, "precision": 0.20234604105571846, "f1": 0.3365853658536585 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__108b2d38", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.3333333333333333, "f1": 0.4 }, "block": { "recall": 0.5555555555555556, "precision": 0.16666666666666666, "f1": 0.2564102564102564 }, "line": { "recall": 0.7291666666666666, "precision": 0.2679938744257274, "f1": 0.3919372900335946 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__9b08d665", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.17647058823529413, "f1": 0.3 }, "block": { "recall": 0.5, "precision": 0.12, "f1": 0.1935483870967742 }, "line": { "recall": 0.34615384615384615, "precision": 0.19834710743801653, "f1": 0.2521891418563923 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__21086305", "official_valid": true, "error": null, "file": { "recall": 0.25, "precision": 0.5, "f1": 0.3333333333333333 }, "block": { "recall": 0.6153846153846154, "precision": 0.4444444444444444, "f1": 0.5161290322580646 }, "line": { "recall": 0.5277777777777778, "precision": 0.45103857566765576, "f1": 0.48640000000000005 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__3b6e6f3a", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2727272727272727, "f1": 0.42857142857142855 }, "block": { "recall": 1.0, "precision": 0.2777777777777778, "f1": 0.4347826086956522 }, "line": { "recall": 0.9213483146067416, "precision": 0.2939068100358423, "f1": 0.44565217391304346 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__c017c9ba", "official_valid": true, "error": null, "file": { "recall": 0.625, "precision": 0.4166666666666667, "f1": 0.5 }, "block": { "recall": 0.36363636363636365, "precision": 0.21052631578947367, "f1": 0.26666666666666666 }, "line": { "recall": 0.17579250720461095, "precision": 0.30808080808080807, "f1": 0.22385321100917432 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__463d4713", "official_valid": true, "error": null, "file": { "recall": 0.25, "precision": 1.0, "f1": 0.4 }, "block": { "recall": 0.42857142857142855, "precision": 1.0, "f1": 0.6 }, "line": { "recall": 0.4470588235294118, "precision": 0.6333333333333333, "f1": 0.5241379310344828 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__d6eb75d7", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.15384615384615385, "f1": 0.23529411764705882 }, "block": { "recall": 0.5555555555555556, "precision": 0.13157894736842105, "f1": 0.2127659574468085 }, "line": { "recall": 0.3646723646723647, "precision": 0.2064516129032258, "f1": 0.26364572605561276 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__10ab7842", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.6666666666666666, "f1": 0.8 }, "block": { "recall": 0.75, "precision": 0.6, "f1": 0.6666666666666665 }, "line": { "recall": 0.8518518518518519, "precision": 0.5373831775700935, "f1": 0.6590257879656161 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__59dd675b", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.35294117647058826, "f1": 0.48 }, "block": { "recall": 0.7142857142857143, "precision": 0.32608695652173914, "f1": 0.4477611940298507 }, "line": { "recall": 0.7137546468401487, "precision": 0.5106382978723404, "f1": 0.5953488372093023 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__0f1fd789", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "line": { "recall": 1.0, "precision": 0.361328125, "f1": 0.5308464849354376 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__a65e43ee", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 0.8846153846153846, "precision": 0.3333333333333333, "f1": 0.4842105263157894 }, "line": { "recall": 0.8932384341637011, "precision": 0.5218295218295218, "f1": 0.6587926509186351 } }, { "instance_id": "SWE-PolyBench__javascript__maintenance__bugfix__9c9b931e", "official_valid": true, "error": null, "file": { "recall": 0.4444444444444444, "precision": 0.5, "f1": 0.47058823529411764 }, "block": { "recall": 0.4090909090909091, "precision": 0.391304347826087, "f1": 0.4 }, "line": { "recall": 0.5493562231759657, "precision": 0.5412262156448203, "f1": 0.5452609158679447 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__7d106697", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "block": { "recall": 1.0, "precision": 0.07894736842105263, "f1": 0.14634146341463414 }, "line": { "recall": 1.0, "precision": 0.07364975450081833, "f1": 0.1371951219512195 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__22615f27", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 1.0, "precision": 0.2, "f1": 0.33333333333333337 }, "line": { "recall": 1.0, "precision": 0.08562197092084006, "f1": 0.15773809523809523 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__16d1ff7a", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.1, "f1": 0.18181818181818182 }, "line": { "recall": 1.0, "precision": 0.07679738562091504, "f1": 0.1426403641881639 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__48468065", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "block": { "recall": 1.0, "precision": 0.17647058823529413, "f1": 0.3 }, "line": { "recall": 0.9772727272727273, "precision": 0.15925925925925927, "f1": 0.2738853503184714 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__2bb4ea7a", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "block": { "recall": 1.0, "precision": 0.09090909090909091, "f1": 0.16666666666666669 }, "line": { "recall": 1.0, "precision": 0.13310580204778158, "f1": 0.23493975903614459 } }, { "instance_id": "SWE-PolyBench__typescript__evolution__feature__684a3415", "official_valid": true, "error": null, "file": { "recall": 0.2, "precision": 0.125, "f1": 0.15384615384615385 }, "block": { "recall": 0.25, "precision": 0.1, "f1": 0.14285714285714288 }, "line": { "recall": 0.6436781609195402, "precision": 0.3128491620111732, "f1": 0.42105263157894735 } }, { "instance_id": "SWE-PolyBench__typescript__evolution__feature__07098661", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.3333333333333333, "f1": 0.46153846153846156 }, "block": { "recall": 0.9166666666666666, "precision": 0.13253012048192772, "f1": 0.23157894736842105 }, "line": { "recall": 0.9178082191780822, "precision": 0.10014947683109118, "f1": 0.18059299191374661 } }, { "instance_id": "SWE-PolyBench__typescript__evolution__feature__c2e9e5ff", "official_valid": true, "error": null, "file": { "recall": 0.6, "precision": 0.75, "f1": 0.6666666666666665 }, "block": { "recall": 0.75, "precision": 0.23076923076923078, "f1": 0.3529411764705882 }, "line": { "recall": 0.8811881188118812, "precision": 0.29276315789473684, "f1": 0.43950617283950616 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__18d389e5", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.3333333333333333, "f1": 0.4 }, "block": { "recall": 0.5714285714285714, "precision": 0.0975609756097561, "f1": 0.16666666666666669 }, "line": { "recall": 0.6419753086419753, "precision": 0.1187214611872146, "f1": 0.20038535645472058 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__0307e40c", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.07142857142857142, "f1": 0.13333333333333333 }, "block": { "recall": 1.0, "precision": 0.07692307692307693, "f1": 0.14285714285714288 }, "line": { "recall": 1.0, "precision": 0.0711743772241993, "f1": 0.132890365448505 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__2d1b4a72", "official_valid": true, "error": null, "file": { "recall": 0.3333333333333333, "precision": 0.16666666666666666, "f1": 0.2222222222222222 }, "block": { "recall": 0.0, "precision": 0.0, "f1": 0.0 }, "line": { "recall": 0.3333333333333333, "precision": 0.046153846153846156, "f1": 0.08108108108108109 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__52180d42", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2, "f1": 0.33333333333333337 }, "block": { "recall": 1.0, "precision": 0.0, "f1": 0.0 }, "line": { "recall": 1.0, "precision": 0.1566579634464752, "f1": 0.2708803611738149 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__1de1bd3c", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.1, "f1": 0.18181818181818182 }, "block": { "recall": 1.0, "precision": 0.057692307692307696, "f1": 0.1090909090909091 }, "line": { "recall": 1.0, "precision": 0.16929133858267717, "f1": 0.2895622895622896 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__6659fd5a", "official_valid": true, "error": null, "file": { "recall": 0.5, "precision": 0.5, "f1": 0.5 }, "block": { "recall": 0.6666666666666666, "precision": 0.14285714285714285, "f1": 0.23529411764705882 }, "line": { "recall": 0.6284153005464481, "precision": 0.3582554517133956, "f1": 0.4563492063492064 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__27393dd8", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.25, "f1": 0.4 }, "block": { "recall": 1.0, "precision": 0.07142857142857142, "f1": 0.13333333333333333 }, "line": { "recall": 1.0, "precision": 0.10144927536231885, "f1": 0.1842105263157895 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__7dfdbbd1", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.16666666666666666, "f1": 0.2857142857142857 }, "block": { "recall": 1.0, "precision": 0.2, "f1": 0.33333333333333337 }, "line": { "recall": 1.0, "precision": 0.1958762886597938, "f1": 0.3275862068965517 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__2bfb742a", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 1.0, "precision": 0.2702702702702703, "f1": 0.4255319148936171 }, "line": { "recall": 0.9743589743589743, "precision": 0.2544642857142857, "f1": 0.40353982300884955 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__7827bc22", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.25, "f1": 0.4 }, "block": { "recall": 0.6666666666666666, "precision": 0.03571428571428571, "f1": 0.06779661016949153 }, "line": { "recall": 0.8181818181818182, "precision": 0.09977827050997783, "f1": 0.17786561264822134 } }, { "instance_id": "SWE-PolyBench__typescript__evolution__feature__34826a6a", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.26666666666666666, "f1": 0.4 }, "block": { "recall": 0.8, "precision": 0.11428571428571428, "f1": 0.19999999999999998 }, "line": { "recall": 0.6036585365853658, "precision": 0.22654462242562928, "f1": 0.329450915141431 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__678fa217", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.1111111111111111, "f1": 0.19999999999999998 }, "block": { "recall": 1.0, "precision": 0.017857142857142856, "f1": 0.03508771929824561 }, "line": { "recall": 1.0, "precision": 0.05792349726775956, "f1": 0.10950413223140495 } }, { "instance_id": "SWE-PolyBench__typescript__evolution__feature__d520ba22", "official_valid": true, "error": null, "file": { "recall": 0.6666666666666666, "precision": 0.13333333333333333, "f1": 0.2222222222222222 }, "block": { "recall": 1.0, "precision": 0.1590909090909091, "f1": 0.2745098039215686 }, "line": { "recall": 0.904, "precision": 0.2198443579766537, "f1": 0.3536776212832551 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__3864c137", "official_valid": true, "error": null, "file": { "recall": 0.6666666666666666, "precision": 0.6666666666666666, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.1875, "f1": 0.3157894736842105 }, "line": { "recall": 0.8, "precision": 0.23645320197044334, "f1": 0.3650190114068441 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__15589352", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.16666666666666666, "f1": 0.2857142857142857 }, "block": { "recall": 1.0, "precision": 0.047619047619047616, "f1": 0.0909090909090909 }, "line": { "recall": 1.0, "precision": 0.15940685820203893, "f1": 0.2749800159872102 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__6688f2d9", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.25, "f1": 0.4 }, "block": { "recall": 1.0, "precision": 0.2222222222222222, "f1": 0.3636363636363636 }, "line": { "recall": 1.0, "precision": 0.23780487804878048, "f1": 0.3842364532019704 } }, { "instance_id": "SWE-PolyBench__typescript__evolution__feature__41cd3842", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.16666666666666666, "f1": 0.2857142857142857 }, "block": { "recall": 1.0, "precision": 0.15, "f1": 0.2608695652173913 }, "line": { "recall": 1.0, "precision": 0.054409005628517824, "f1": 0.10320284697508897 } }, { "instance_id": "SWE-PolyBench__typescript__maintenance__bugfix__42165c4e", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.3333333333333333, "f1": 0.46153846153846156 }, "block": { "recall": 0.6666666666666666, "precision": 0.07142857142857142, "f1": 0.12903225806451613 }, "line": { "recall": 0.7073170731707317, "precision": 0.06812842599843383, "f1": 0.12428571428571429 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__ed58622a", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.09090909090909091, "f1": 0.16666666666666669 }, "block": { "recall": 1.0, "precision": 0.0392156862745098, "f1": 0.07547169811320754 }, "line": { "recall": 1.0, "precision": 0.10482921083627797, "f1": 0.18976545842217482 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__faa6b3ac", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.09090909090909091, "f1": 0.16666666666666669 }, "block": { "recall": 1.0, "precision": 0.05434782608695652, "f1": 0.10309278350515463 }, "line": { "recall": 1.0, "precision": 0.17307692307692307, "f1": 0.29508196721311475 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__9a72b241", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2857142857142857, "f1": 0.4444444444444445 }, "block": { "recall": 1.0, "precision": 0.12195121951219512, "f1": 0.21739130434782605 }, "line": { "recall": 0.8596491228070176, "precision": 0.23671497584541062, "f1": 0.3712121212121212 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__5188c6bc", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.25, "f1": 0.4 }, "block": { "recall": 1.0, "precision": 0.05128205128205128, "f1": 0.09756097560975609 }, "line": { "recall": 1.0, "precision": 0.025594149908592323, "f1": 0.049910873440285206 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__214ce29d", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.08333333333333333, "f1": 0.15384615384615385 }, "block": { "recall": 1.0, "precision": 0.08928571428571429, "f1": 0.1639344262295082 }, "line": { "recall": 1.0, "precision": 0.1079913606911447, "f1": 0.1949317738791423 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__2f88019c", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.25, "f1": 0.4 }, "line": { "recall": 1.0, "precision": 0.3148148148148148, "f1": 0.4788732394366197 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__dbd7cde5", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.5, "f1": 0.6666666666666666 }, "block": { "recall": 1.0, "precision": 0.2857142857142857, "f1": 0.4444444444444445 }, "line": { "recall": 1.0, "precision": 0.49557522123893805, "f1": 0.6627218934911243 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__453fb844", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.14285714285714285, "f1": 0.25 }, "block": { "recall": 1.0, "precision": 0.07407407407407407, "f1": 0.13793103448275862 }, "line": { "recall": 1.0, "precision": 0.09688013136288999, "f1": 0.17664670658682635 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__5518cb7f", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 0.8333333333333334, "precision": 0.17857142857142858, "f1": 0.29411764705882354 }, "line": { "recall": 0.9150943396226415, "precision": 0.2878338278931751, "f1": 0.43792325056433407 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__8bb50331", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.3333333333333333, "f1": 0.47058823529411764 }, "block": { "recall": 0.967741935483871, "precision": 0.5084745762711864, "f1": 0.6666666666666666 }, "line": { "recall": 0.5258064516129032, "precision": 0.21168831168831168, "f1": 0.3018518518518518 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__e3c9c53c", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.125, "f1": 0.2222222222222222 }, "block": { "recall": 1.0, "precision": 0.043478260869565216, "f1": 0.08333333333333333 }, "line": { "recall": 1.0, "precision": 0.20618556701030927, "f1": 0.3418803418803419 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__4a329645", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.6666666666666666, "f1": 0.8 }, "block": { "recall": 0.9130434782608695, "precision": 1.0, "f1": 0.9545454545454545 }, "line": { "recall": 0.855191256830601, "precision": 0.9178885630498533, "f1": 0.8854314002828855 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__5acd9675", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.25, "f1": 0.4 }, "block": { "recall": 1.0, "precision": 0.1568627450980392, "f1": 0.2711864406779661 }, "line": { "recall": 1.0, "precision": 0.5272108843537415, "f1": 0.690423162583519 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__9ff56151", "official_valid": true, "error": null, "file": { "recall": 0.75, "precision": 0.3333333333333333, "f1": 0.46153846153846156 }, "block": { "recall": 0.8461538461538461, "precision": 0.1506849315068493, "f1": 0.2558139534883721 }, "line": { "recall": 0.7480314960629921, "precision": 0.3327495621716287, "f1": 0.4606060606060606 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__17a093c8", "official_valid": true, "error": null, "file": { "recall": 0.4, "precision": 1.0, "f1": 0.5714285714285715 }, "block": { "recall": 0.8, "precision": 1.0, "f1": 0.888888888888889 }, "line": { "recall": 0.8297297297297297, "precision": 0.953416149068323, "f1": 0.8872832369942195 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__ebb79d55", "official_valid": true, "error": null, "file": { "recall": 0.8, "precision": 0.5, "f1": 0.6153846153846154 }, "block": { "recall": 0.8571428571428571, "precision": 0.2057142857142857, "f1": 0.3317972350230415 }, "line": { "recall": 0.8143564356435643, "precision": 0.4177777777777778, "f1": 0.5522450692404531 } }, { "instance_id": "SWE-PolyBench__python__evolution__feature__b4302840", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.2, "f1": 0.33333333333333337 }, "block": { "recall": 1.0, "precision": 0.3888888888888889, "f1": 0.56 }, "line": { "recall": 1.0, "precision": 0.3405088062622309, "f1": 0.5080291970802919 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__91e6ed53", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 1.0, "precision": 0.09090909090909091, "f1": 0.16666666666666669 }, "line": { "recall": 1.0, "precision": 0.12142857142857143, "f1": 0.21656050955414013 } }, { "instance_id": "SWE-PolyBench__python__maintenance__bugfix__ac5893a5", "official_valid": true, "error": null, "file": { "recall": 1.0, "precision": 0.3333333333333333, "f1": 0.5 }, "block": { "recall": 1.0, "precision": 0.13043478260869565, "f1": 0.23076923076923078 }, "line": { "recall": 1.0, "precision": 0.09465648854961832, "f1": 0.1729428172942817 } } ], "CodeNib_components_used": [ "default read/grep/glob source tools", "repository-bound source adapters", "actual-consumed-source trajectory conversion", "snapshot/usage receipts and official evaluator bridge" ], "not_enabled": [ "semantic graph", "vector retrieval", "BM25", "Jev reranking", "late4K final-context consumer" ], "enhancement_is_later_matched_source_packing_comparison": true }, "verified500_same122_official_comparison": { "scope": "Same122 IDs, same local Gold/source snapshots, same pinned official evaluator/environment; compare two different model+agent systems, not CodeNib treatment benefit.", "population": 122, "our_native": { "population": 122, "official_valid_cases": 122, "official_failed_cases": [], "failure_accounted_snapshot_macro": { "file": { "recall": 0.8031858348726338, "precision": 0.3562485256717264, "f1": 0.44327477356524747 }, "block": { "recall": 0.8178157043431532, "precision": 0.26038132580387824, "f1": 0.35072697039911144 }, "line": { "recall": 0.7680559332548911, "precision": 0.30752983776988435, "f1": 0.3899852469923677 } } }, "official_published_same122": { "population": 122, "official_valid_cases": 121, "official_failed_cases": [ "SWE-Bench-Pro__python__maintenance__bugfix__84fdadf0" ], "failure_accounted_snapshot_macro": { "file": { "recall": 0.7646786166091601, "precision": 0.4115735189053502, "f1": 0.4637008811885822 }, "block": { "recall": 0.802263652981903, "precision": 0.2503276867782366, "f1": 0.33661294161526745 }, "line": { "recall": 0.7882829073733001, "precision": 0.21106891356016186, "f1": 0.29617444896085793 } } }, "official_rescored_same122": { "population": 122, "official_valid_cases": 122, "official_failed_cases": [], "failure_accounted_snapshot_macro": { "file": { "recall": 0.7674108570463186, "precision": 0.4148294901851989, "f1": 0.46644728029467275 }, "block": { "recall": 0.804399961177463, "precision": 0.24932312319125916, "f1": 0.33491978766693953 }, "line": { "recall": 0.7923812680290379, "precision": 0.21124399526501034, "f1": 0.29648935839398977 } } }, "our_minus_official_rescored": { "file": { "recall": 0.035774977826315224, "precision": -0.0585809645134725, "f1": -0.02317250672942528 }, "block": { "recall": 0.013415743165690164, "precision": 0.011058202612619084, "f1": 0.01580718273217191 }, "line": { "recall": -0.02432533477414678, "precision": 0.09628584250487401, "f1": 0.09349588859837793 } }, "different_backbone_and_agent_harness": true, "CodeNib_enhancement_gain_established": false, "context_only_our_run_vs_issue_resolution_competitor": true, "whole500_result_or_leaderboard_rank": false, "protocol_sha256": "77ec9ee3f1ee67feeb48ff5276f6d4768aaac4dcfc61adbf15089e0489e15ce2", "metrics_sha256": "959f6a4259108bc3bda21544cb8ded32af1cb20fe049346e7da1d442f8662cd9", "new_model_calls": 0, "new_API_usage_usd": 0, "new_Gold_cases_beyond_already_scored122": 0, "analysis_sha256": "0f89ba85d2d62fea9aa0bf184393503a3d5d42260f0bec9fbb217e5457bc5609", "official_submission_commit": "a40dfa6b16f14cafd2b1cc46faf0969b38ef03b9", "official_submission": "https://github.com/ContextBench/contextbench.github.io/tree/a40dfa6b16f14cafd2b1cc46faf0969b38ef03b9/submissions/2026-09-12_openjiuwen-jiuwenswarm", "same122_official_valid_both_arms": true, "mean_of_task_F1": true } }