{
  "dataset": "isodata-samples-v1",
  "diagnostics_version": 1,
  "compiled_date": "2026-10-10",
  "title": "The same programs, different wording",
  "metric": "Held-out cases with correct outputs",
  "conditions": {
    "original": "Saved programs evaluated on the original held-out case text.",
    "reworded": "Those programs evaluated on the same underlying case events and expected outcomes, rendered with the phrasing used in public examples."
  },
  "method": "Derived from saved diagnostic outputs. Five saved programs per task; no fresh agent attempt or solver repair. Expense diagnostics adapt table paths for local execution. Pharmacy output comparisons require exact on-hand values and the expected discrepancy set; expense comparisons require correct monetary values and flag sets; stream diagnostics identify incorrect fieldbooks.",
  "records": [
    {
      "task": "stream-gauging-fieldbook",
      "label": "Stream fieldbooks",
      "case_unit": "fieldbooks",
      "cases_per_program": 60,
      "programs": [
        {
          "program": "sg-1",
          "original_correct": 0,
          "reworded_correct": 60,
          "source_artifacts": [
            "s01"
          ]
        },
        {
          "program": "sg-2",
          "original_correct": 5,
          "reworded_correct": 60,
          "source_artifacts": [
            "s01"
          ]
        },
        {
          "program": "sg-3",
          "original_correct": 5,
          "reworded_correct": 60,
          "source_artifacts": [
            "s01"
          ]
        },
        {
          "program": "sg-4",
          "original_correct": 0,
          "reworded_correct": 58,
          "source_artifacts": [
            "s01"
          ]
        },
        {
          "program": "sg-5",
          "original_correct": 0,
          "reworded_correct": 60,
          "source_artifacts": [
            "s01"
          ]
        }
      ],
      "totals": {
        "program_count": 5,
        "program_case_observations": 300,
        "original_correct": 10,
        "reworded_correct": 298,
        "original_percent_correct": 3.3333333333333335,
        "reworded_percent_correct": 99.33333333333333,
        "original_programs_correct_on_all_hidden": 0,
        "reworded_programs_correct_on_all_hidden": 4
      }
    },
    {
      "task": "expense-claim-audit",
      "label": "Expense claims",
      "case_unit": "claims",
      "cases_per_program": 56,
      "programs": [
        {
          "program": "ec-1",
          "original_correct": 42,
          "reworded_correct": 49,
          "source_artifacts": [
            "s03"
          ],
          "public_example_correct": 9,
          "public_example_total": 10
        },
        {
          "program": "ec-2",
          "original_correct": 42,
          "reworded_correct": 56,
          "source_artifacts": [
            "s04"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "ec-3",
          "original_correct": 27,
          "reworded_correct": 51,
          "source_artifacts": [
            "s05"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "ec-4",
          "original_correct": 23,
          "reworded_correct": 35,
          "source_artifacts": [
            "s06"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "ec-5",
          "original_correct": 36,
          "reworded_correct": 50,
          "source_artifacts": [
            "s07"
          ],
          "public_example_correct": 9,
          "public_example_total": 10
        }
      ],
      "totals": {
        "program_count": 5,
        "program_case_observations": 280,
        "original_correct": 170,
        "reworded_correct": 241,
        "original_percent_correct": 60.714285714285715,
        "reworded_percent_correct": 86.07142857142857,
        "original_programs_correct_on_all_hidden": 0,
        "reworded_programs_correct_on_all_hidden": 1
      }
    },
    {
      "task": "controlled-count-close/notes",
      "label": "Pharmacy notes",
      "case_unit": "shifts",
      "cases_per_program": 48,
      "programs": [
        {
          "program": "cn2-1",
          "original_correct": 0,
          "reworded_correct": 41,
          "source_artifacts": [
            "s11",
            "s12",
            "s10"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "cn2-2",
          "original_correct": 0,
          "reworded_correct": 39,
          "source_artifacts": [
            "s13",
            "s14",
            "s10"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "cn2-3",
          "original_correct": 5,
          "reworded_correct": 42,
          "source_artifacts": [
            "s15",
            "s16",
            "s10"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "cn2-4",
          "original_correct": 2,
          "reworded_correct": 41,
          "source_artifacts": [
            "s17",
            "s18",
            "s10"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        },
        {
          "program": "cn2-5",
          "original_correct": 0,
          "reworded_correct": 41,
          "source_artifacts": [
            "s19",
            "s20",
            "s10"
          ],
          "public_example_correct": 10,
          "public_example_total": 10
        }
      ],
      "totals": {
        "program_count": 5,
        "program_case_observations": 240,
        "original_correct": 7,
        "reworded_correct": 204,
        "original_percent_correct": 2.9166666666666665,
        "reworded_percent_correct": 85.0,
        "original_programs_correct_on_all_hidden": 0,
        "reworded_programs_correct_on_all_hidden": 0
      }
    }
  ],
  "disclosures": [
    "Totals count program-case observations: each held-out case is evaluated by five programs. They do not represent that many unique cases or independent trials.",
    "These are case-output diagnostics, not full task rewards. In particular, they do not establish that every file, format, and runtime requirement of the released task passes.",
    "Stream and expense programs were written against earlier drafts and regraded. Pharmacy diagnostics preceded an added policy clarification for removals expressed as drug amounts; the six reworded shifts missed by all five pharmacy programs involve that form.",
    "Expense public-example counts are 9, 10, 10, 10, and 9 of 10 after revisions. Pharmacy public-example counts are 10 of 10 for each program. Public cases differ in complexity from held-out cases, so that comparison is descriptive.",
    "The separate three-attempt pharmacy database control used fresh programs. It is not part of this paired rewording diagnostic.",
    "This is a selected difficulty-focused sample, not a representative benchmark or a human-versus-model study. Model identity and attempt conditions follow the supplied study record.",
    "Source hashes identify the saved artifacts used to compile these records. Raw diagnostic reports, solver programs, and internal notes are not included in this file. This compilation did not rerun the tasks."
  ],
  "source_artifacts": [
    {
      "id": "s01",
      "file": "build-notes/stream-gauging-fieldbook/blind_diag_family.json",
      "sha256": "bf2ca32d1da087c7cb6de746601147c456180649746f2d2e512d30554661617c"
    },
    {
      "id": "s03",
      "file": "build-notes/expense-claim-audit/blind-audit/results/ec-1.json",
      "sha256": "7aad3373560ec29e9908f294c70ef63875a5a12d69ff94f7f82372a4c6026faa"
    },
    {
      "id": "s04",
      "file": "build-notes/expense-claim-audit/blind-audit/results/ec-2.json",
      "sha256": "402bb564b73c1a74f6dfd734210603dc980639cba16cc0cd3d1032a07d1207dc"
    },
    {
      "id": "s05",
      "file": "build-notes/expense-claim-audit/blind-audit/results/ec-3.json",
      "sha256": "788ab00cf5487d16e5a7df84a2cc61c78c4070a69e39d1145ebb62cc9cceb56b"
    },
    {
      "id": "s06",
      "file": "build-notes/expense-claim-audit/blind-audit/results/ec-4.json",
      "sha256": "5aecd02a34a3244a377057f7f6786956becdce712acd038b97fbbc0e755a71a4"
    },
    {
      "id": "s07",
      "file": "build-notes/expense-claim-audit/blind-audit/results/ec-5.json",
      "sha256": "3cdd6b900b7d37e41b9e7801ffede7be7eb53c67a93e648db632833985530395"
    },
    {
      "id": "s10",
      "file": "released-tasks/controlled-count-close/notes/tests/expected.json",
      "sha256": "e1f65032651a8e08de5939a6e2acdc2791514c9bf0f16093888dced3a1d83da0"
    },
    {
      "id": "s11",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-1.reports.json",
      "sha256": "c672d2ba72f0f976d801d53067c81fb40db2833262321168e260eab0835dd08c"
    },
    {
      "id": "s12",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-1.reports_public_wording.json",
      "sha256": "9665cf30ee80f4f744bf04ed715cc9fa4047b5c38228833aca413378ddc7ef02"
    },
    {
      "id": "s13",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-2.reports.json",
      "sha256": "aacbabe8dd81882bd3917cc65df85ef3c62c04f3ff1ff3d2365141bbfd514c55"
    },
    {
      "id": "s14",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-2.reports_public_wording.json",
      "sha256": "4eba1920007a670ce46c34a6c0f452039bc5c204061b84ba11650cb9b9c04219"
    },
    {
      "id": "s15",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-3.reports.json",
      "sha256": "1e8ffa0c0a8dd3f1b7e83d229cde28dc5eb99b0fa8f91d6d25a1cbf8ff6a9dec"
    },
    {
      "id": "s16",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-3.reports_public_wording.json",
      "sha256": "00fe805a9008d186a90d157011c8453130d1565751adab156b6641fa43545230"
    },
    {
      "id": "s17",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-4.reports.json",
      "sha256": "2cb8460dda1e0b5728ee8964cd3823ce7b028b9190190e37507b9d45444488b5"
    },
    {
      "id": "s18",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-4.reports_public_wording.json",
      "sha256": "0f4bab8967abc1e51645822a06b4ae0e610e3fcbe3b1ea182b62dc1a229df5ea"
    },
    {
      "id": "s19",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-5.reports.json",
      "sha256": "0c97c3943fced3104d3816e03ec036f7cf3ffd000c6b99dcfd24643291ed4e5d"
    },
    {
      "id": "s20",
      "file": "build-notes/controlled-count-close/audit-v2/solvers/cn2-5.reports_public_wording.json",
      "sha256": "f1785d619b0a5775f2e5ecaa2af07c931a549c15013d7d12c1cd4c9e5c8334cc"
    }
  ]
}