{
  "scope": "Frozen-plan replay of production candidate generation, not fresh model or agent evaluation",
  "dataset_revision": "4eb84e2e8918474969ce68c5b06facf14d6be604",
  "cases": 100,
  "completed_cases": 100,
  "errors": [],
  "ordered_text_matches": 94,
  "corrected_span_order_matches": 94,
  "planner_input_difference_fields": {
    "repo": 100
  },
  "product_grep_recall_at_5": 0.5861666666666666,
  "frozen_score_sensitivity_recall_at_5": null,
  "score_sensitivity_note": "Reuse only exact candidate-text scores, with product stable tie ordering; no new Jev calls",
  "input_sha256": {
    "prepared_manifest": "e055006df271dddfeb18350897cf5e619d1061ea8c1e3efe4d9253d595230ddb",
    "label_report": "22bea146f5971f295f18a6b08437719ff06663ac51826272133fc3dd1bdf2180",
    "config.json": "515fb29333413901524e9d3a687d186ae4e0c663c446f29ab9e8beccdebe7c2f",
    "rows.json": "83af1014f9bc326d22717b75678b5e929f9157b7f8a37d22bb91eb0b1ecb0caf",
    "calls.json": "f33ac02a5217d7e64005e6757d06a1d63e94a754053e3ce811d16e31918bc22d"
  },
  "product_sha256": "c2a3b89f1a89f62ccc3034814874ff949d1b81fdcbf90a15e37da11ce3f4a69d",
  "script_sha256": "f738a3a94473e906e528c565cd7e15d6c3e4d09ec058d911be345b06162e90f6",
  "billed_api_calls": 0,
  "per_case": [
    {
      "instance_id": "astral-sh__ruff-15309",
      "language_group": "Rust",
      "frozen_candidate_count": 22,
      "product_candidate_count": 22,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "astral-sh__ruff-15330",
      "language_group": "Rust",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "astral-sh__ruff-15356",
      "language_group": "Rust",
      "frozen_candidate_count": 3,
      "product_candidate_count": 3,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "astral-sh__ruff-15443",
      "language_group": "Rust",
      "frozen_candidate_count": 2,
      "product_candidate_count": 2,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "astropy__astropy-12907",
      "language_group": "Python",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "astropy__astropy-13033",
      "language_group": "Python",
      "frozen_candidate_count": 9,
      "product_candidate_count": 9,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "astropy__astropy-13236",
      "language_group": "Python",
      "frozen_candidate_count": 3,
      "product_candidate_count": 3,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "astropy__astropy-13579",
      "language_group": "Python",
      "frozen_candidate_count": 8,
      "product_candidate_count": 8,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "axios__axios-4731",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 6,
      "product_candidate_count": 6,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "axios__axios-4738",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "axios__axios-5085",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "axios__axios-6539",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "babel__babel-13928",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 0,
      "product_candidate_count": 0,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "babel__babel-14532",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "babel__babel-15445",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "babel__babel-15649",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 7,
      "product_candidate_count": 7,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "caddyserver__caddy-4943",
      "language_group": "Go",
      "frozen_candidate_count": 18,
      "product_candidate_count": 18,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "caddyserver__caddy-5404",
      "language_group": "Go",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.5,
      "product_grep_recall_at_5": 0.5,
      "frozen_score_sensitivity_recall_at_5": 0.5
    },
    {
      "instance_id": "caddyserver__caddy-5761",
      "language_group": "Go",
      "frozen_candidate_count": 9,
      "product_candidate_count": 9,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "caddyserver__caddy-5870",
      "language_group": "Go",
      "frozen_candidate_count": 6,
      "product_candidate_count": 6,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "caddyserver__caddy-6115",
      "language_group": "Go",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "facebook__docusaurus-10130",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 27,
      "product_candidate_count": 27,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "facebook__docusaurus-10309",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 28,
      "product_candidate_count": 28,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "facebook__docusaurus-8927",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 11,
      "product_candidate_count": 11,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "facebook__docusaurus-9897",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 8,
      "product_candidate_count": 8,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "fmtlib__fmt-2310",
      "language_group": "C++/C",
      "frozen_candidate_count": 42,
      "product_candidate_count": 42,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.5
    },
    {
      "instance_id": "fmtlib__fmt-2317",
      "language_group": "C++/C",
      "frozen_candidate_count": 21,
      "product_candidate_count": 21,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "fmtlib__fmt-3248",
      "language_group": "C++/C",
      "frozen_candidate_count": 25,
      "product_candidate_count": 25,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "gin-gonic__gin-1805",
      "language_group": "Go",
      "frozen_candidate_count": 57,
      "product_candidate_count": 57,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "gin-gonic__gin-2121",
      "language_group": "Go",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "gin-gonic__gin-2755",
      "language_group": "Go",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "gin-gonic__gin-3741",
      "language_group": "Go",
      "frozen_candidate_count": 31,
      "product_candidate_count": 31,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "gin-gonic__gin-3820",
      "language_group": "Go",
      "frozen_candidate_count": 19,
      "product_candidate_count": 19,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "gohugoio__hugo-12171",
      "language_group": "Go",
      "frozen_candidate_count": 43,
      "product_candidate_count": 43,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "gohugoio__hugo-12343",
      "language_group": "Go",
      "frozen_candidate_count": 30,
      "product_candidate_count": 30,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "gohugoio__hugo-12448",
      "language_group": "Go",
      "frozen_candidate_count": 57,
      "product_candidate_count": 57,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "gohugoio__hugo-12562",
      "language_group": "Go",
      "frozen_candidate_count": 26,
      "product_candidate_count": 26,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "gohugoio__hugo-12579",
      "language_group": "Go",
      "frozen_candidate_count": 13,
      "product_candidate_count": 13,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "hashicorp__terraform-34814",
      "language_group": "Go",
      "frozen_candidate_count": 19,
      "product_candidate_count": 19,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "hashicorp__terraform-35611",
      "language_group": "Go",
      "frozen_candidate_count": 15,
      "product_candidate_count": 15,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "jqlang__jq-2235",
      "language_group": "C++/C",
      "frozen_candidate_count": 26,
      "product_candidate_count": 99,
      "ordered_text_match": false,
      "corrected_span_order_match": false,
      "all_product_texts_have_frozen_scores": false,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": null
    },
    {
      "instance_id": "jqlang__jq-2598",
      "language_group": "C++/C",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.25,
      "product_grep_recall_at_5": 0.25,
      "frozen_score_sensitivity_recall_at_5": 0.25
    },
    {
      "instance_id": "jqlang__jq-2658",
      "language_group": "C++/C",
      "frozen_candidate_count": 36,
      "product_candidate_count": 35,
      "ordered_text_match": false,
      "corrected_span_order_match": false,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "jqlang__jq-2728",
      "language_group": "C++/C",
      "frozen_candidate_count": 19,
      "product_candidate_count": 19,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "jqlang__jq-2750",
      "language_group": "C++/C",
      "frozen_candidate_count": 22,
      "product_candidate_count": 22,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.5,
      "product_grep_recall_at_5": 0.5,
      "frozen_score_sensitivity_recall_at_5": 0.75
    },
    {
      "instance_id": "matplotlib__matplotlib-13989",
      "language_group": "Python",
      "frozen_candidate_count": 7,
      "product_candidate_count": 7,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "matplotlib__matplotlib-14623",
      "language_group": "Python",
      "frozen_candidate_count": 25,
      "product_candidate_count": 25,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "matplotlib__matplotlib-20488",
      "language_group": "Python",
      "frozen_candidate_count": 13,
      "product_candidate_count": 13,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "matplotlib__matplotlib-20676",
      "language_group": "Python",
      "frozen_candidate_count": 26,
      "product_candidate_count": 26,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "micropython__micropython-10095",
      "language_group": "C++/C",
      "frozen_candidate_count": 100,
      "product_candidate_count": 100,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "micropython__micropython-13039",
      "language_group": "C++/C",
      "frozen_candidate_count": 8,
      "product_candidate_count": 8,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "micropython__micropython-13569",
      "language_group": "C++/C",
      "frozen_candidate_count": 18,
      "product_candidate_count": 18,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.5
    },
    {
      "instance_id": "micropython__micropython-15898",
      "language_group": "C++/C",
      "frozen_candidate_count": 2,
      "product_candidate_count": 2,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "nushell__nushell-12901",
      "language_group": "Rust",
      "frozen_candidate_count": 23,
      "product_candidate_count": 23,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "nushell__nushell-12950",
      "language_group": "Rust",
      "frozen_candidate_count": 58,
      "product_candidate_count": 57,
      "ordered_text_match": false,
      "corrected_span_order_match": false,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "nushell__nushell-13605",
      "language_group": "Rust",
      "frozen_candidate_count": 6,
      "product_candidate_count": 6,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "nushell__nushell-13831",
      "language_group": "Rust",
      "frozen_candidate_count": 15,
      "product_candidate_count": 15,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.3333333333333333,
      "product_grep_recall_at_5": 0.3333333333333333,
      "frozen_score_sensitivity_recall_at_5": 0.3333333333333333
    },
    {
      "instance_id": "preactjs__preact-2757",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 12,
      "product_candidate_count": 12,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "preactjs__preact-2896",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 14,
      "product_candidate_count": 14,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "preactjs__preact-2927",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "preactjs__preact-3567",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 22,
      "product_candidate_count": 22,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.2,
      "product_grep_recall_at_5": 0.2,
      "frozen_score_sensitivity_recall_at_5": 0.6
    },
    {
      "instance_id": "prometheus__prometheus-10633",
      "language_group": "Go",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "prometheus__prometheus-11859",
      "language_group": "Go",
      "frozen_candidate_count": 10,
      "product_candidate_count": 10,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "prometheus__prometheus-14861",
      "language_group": "Go",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "prometheus__prometheus-15142",
      "language_group": "Go",
      "frozen_candidate_count": 22,
      "product_candidate_count": 22,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.4,
      "product_grep_recall_at_5": 0.4,
      "frozen_score_sensitivity_recall_at_5": 0.2
    },
    {
      "instance_id": "pydata__xarray-2905",
      "language_group": "Python",
      "frozen_candidate_count": 23,
      "product_candidate_count": 23,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "pydata__xarray-3151",
      "language_group": "Python",
      "frozen_candidate_count": 16,
      "product_candidate_count": 16,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "pydata__xarray-3305",
      "language_group": "Python",
      "frozen_candidate_count": 10,
      "product_candidate_count": 10,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.5,
      "product_grep_recall_at_5": 0.5,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "pydata__xarray-6992",
      "language_group": "Python",
      "frozen_candidate_count": 19,
      "product_candidate_count": 19,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.6666666666666666,
      "product_grep_recall_at_5": 0.6666666666666666,
      "frozen_score_sensitivity_recall_at_5": 0.6666666666666666
    },
    {
      "instance_id": "redis__redis-10068",
      "language_group": "C++/C",
      "frozen_candidate_count": 22,
      "product_candidate_count": 22,
      "ordered_text_match": false,
      "corrected_span_order_match": false,
      "all_product_texts_have_frozen_scores": false,
      "frozen_grep_recall_at_5": 0.3333333333333333,
      "product_grep_recall_at_5": 0.3333333333333333,
      "frozen_score_sensitivity_recall_at_5": null
    },
    {
      "instance_id": "redis__redis-10095",
      "language_group": "C++/C",
      "frozen_candidate_count": 52,
      "product_candidate_count": 52,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "redis__redis-11631",
      "language_group": "C++/C",
      "frozen_candidate_count": 38,
      "product_candidate_count": 38,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "redis__redis-11734",
      "language_group": "C++/C",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "redis__redis-13338",
      "language_group": "C++/C",
      "frozen_candidate_count": 16,
      "product_candidate_count": 16,
      "ordered_text_match": false,
      "corrected_span_order_match": false,
      "all_product_texts_have_frozen_scores": false,
      "frozen_grep_recall_at_5": 0.3333333333333333,
      "product_grep_recall_at_5": 0.3333333333333333,
      "frozen_score_sensitivity_recall_at_5": null
    },
    {
      "instance_id": "scikit-learn__scikit-learn-10297",
      "language_group": "Python",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "scikit-learn__scikit-learn-10844",
      "language_group": "Python",
      "frozen_candidate_count": 1,
      "product_candidate_count": 1,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "scikit-learn__scikit-learn-10908",
      "language_group": "Python",
      "frozen_candidate_count": 7,
      "product_candidate_count": 7,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "scikit-learn__scikit-learn-13142",
      "language_group": "Python",
      "frozen_candidate_count": 8,
      "product_candidate_count": 8,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sharkdp__bat-2201",
      "language_group": "Rust",
      "frozen_candidate_count": 21,
      "product_candidate_count": 21,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sharkdp__bat-2260",
      "language_group": "Rust",
      "frozen_candidate_count": 19,
      "product_candidate_count": 19,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sharkdp__bat-2650",
      "language_group": "Rust",
      "frozen_candidate_count": 7,
      "product_candidate_count": 7,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sharkdp__bat-3108",
      "language_group": "Rust",
      "frozen_candidate_count": 40,
      "product_candidate_count": 40,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sympy__sympy-11618",
      "language_group": "Python",
      "frozen_candidate_count": 12,
      "product_candidate_count": 12,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sympy__sympy-12096",
      "language_group": "Python",
      "frozen_candidate_count": 7,
      "product_candidate_count": 7,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "sympy__sympy-12419",
      "language_group": "Python",
      "frozen_candidate_count": 13,
      "product_candidate_count": 13,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.5,
      "product_grep_recall_at_5": 0.5,
      "frozen_score_sensitivity_recall_at_5": 0.5
    },
    {
      "instance_id": "sympy__sympy-13031",
      "language_group": "Python",
      "frozen_candidate_count": 27,
      "product_candidate_count": 27,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.5
    },
    {
      "instance_id": "tokio-rs__tokio-4898",
      "language_group": "Rust",
      "frozen_candidate_count": 2,
      "product_candidate_count": 2,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "tokio-rs__tokio-6551",
      "language_group": "Rust",
      "frozen_candidate_count": 8,
      "product_candidate_count": 8,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.5,
      "product_grep_recall_at_5": 0.5,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "tokio-rs__tokio-6603",
      "language_group": "Rust",
      "frozen_candidate_count": 38,
      "product_candidate_count": 38,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.0
    },
    {
      "instance_id": "tokio-rs__tokio-6752",
      "language_group": "Rust",
      "frozen_candidate_count": 16,
      "product_candidate_count": 16,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "uutils__coreutils-6575",
      "language_group": "Rust",
      "frozen_candidate_count": 6,
      "product_candidate_count": 6,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.6,
      "product_grep_recall_at_5": 0.6,
      "frozen_score_sensitivity_recall_at_5": 0.6
    },
    {
      "instance_id": "uutils__coreutils-6682",
      "language_group": "Rust",
      "frozen_candidate_count": 5,
      "product_candidate_count": 5,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "uutils__coreutils-6690",
      "language_group": "Rust",
      "frozen_candidate_count": 4,
      "product_candidate_count": 4,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "uutils__coreutils-6731",
      "language_group": "Rust",
      "frozen_candidate_count": 1,
      "product_candidate_count": 1,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "valkey-io__valkey-1842",
      "language_group": "C++/C",
      "frozen_candidate_count": 68,
      "product_candidate_count": 66,
      "ordered_text_match": false,
      "corrected_span_order_match": false,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "valkey-io__valkey-790",
      "language_group": "C++/C",
      "frozen_candidate_count": 35,
      "product_candidate_count": 35,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.5
    },
    {
      "instance_id": "valkey-io__valkey-928",
      "language_group": "C++/C",
      "frozen_candidate_count": 9,
      "product_candidate_count": 9,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "vuejs__core-11589",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 16,
      "product_candidate_count": 16,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 0.6666666666666666
    },
    {
      "instance_id": "vuejs__core-11870",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 22,
      "product_candidate_count": 22,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 1.0,
      "product_grep_recall_at_5": 1.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    },
    {
      "instance_id": "vuejs__core-11915",
      "language_group": "TypeScript/JavaScript",
      "frozen_candidate_count": 34,
      "product_candidate_count": 34,
      "ordered_text_match": true,
      "corrected_span_order_match": true,
      "all_product_texts_have_frozen_scores": true,
      "frozen_grep_recall_at_5": 0.0,
      "product_grep_recall_at_5": 0.0,
      "frozen_score_sensitivity_recall_at_5": 1.0
    }
  ],
  "model_evaluation_complete": false
}