{
  "schema": "faros-benchmark-publication-v3",
  "budgets": {
    "origination": 16,
    "refinement": 64
  },
  "repetitions": {
    "origination": 5,
    "refinement": {
      "astra": 3,
      "terra": 3,
      "luna": 3,
      "sonnet": 5
    }
  },
  "refinement_cohorts": {
    "astra": [
      2,
      3,
      7
    ],
    "terra": [
      2,
      3,
      7
    ],
    "luna": [
      2,
      3,
      7
    ],
    "sonnet": [
      2,
      3,
      7,
      5,
      6
    ]
  },
  "cohort_rule": "all completed runs per model",
  "models": [
    {
      "model": "astra",
      "name": "GPT-6 Astra / Codex",
      "color": "#b96b45",
      "origination": {
        "n": 5,
        "mean": 4.612952862819812,
        "ci95": [
          2.917549842237179,
          6.308355883402445
        ]
      },
      "refinement": {
        "n": 3,
        "mean": 26.323764330174626,
        "ci95": [
          23.498330608480554,
          29.149198051868698
        ]
      },
      "score": {
        "axes": [
          "origination",
          "refinement"
        ],
        "n": {
          "origination": 5,
          "refinement": 3
        },
        "axis_means": {
          "origination": 4.612952862819812,
          "refinement": 26.323764330174626
        },
        "mean": 156.14906519427723,
        "ci95": [
          107.3649709767908,
          204.93315941176365
        ],
        "definition": "fixed-reference-core-v1",
        "scales_pp": {
          "origination": 2.885429367211115,
          "refinement": 17.26969005796163
        },
        "normalized_axes": {
          "origination": 159.8705868609918,
          "refinement": 152.42754352756268
        }
      },
      "refinement_observations": [
        {
          "value": 25.116522500315206,
          "repetition": 2
        },
        {
          "value": 27.37524871449654,
          "repetition": 3
        },
        {
          "value": 26.479521775712136,
          "repetition": 7
        }
      ],
      "origination_curve": [
        {
          "k": 1,
          "n": 5,
          "mean": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        {
          "k": 2,
          "n": 5,
          "mean": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        {
          "k": 4,
          "n": 5,
          "mean": 0.391073950818161,
          "ci95": [
            -0.5355133186694381,
            1.31766122030576
          ]
        },
        {
          "k": 8,
          "n": 5,
          "mean": 2.440736140492819,
          "ci95": [
            1.6112075189576562,
            3.2702647620279817
          ]
        },
        {
          "k": 10,
          "n": 5,
          "mean": 2.723075402521656,
          "ci95": [
            1.3311787390765921,
            4.11497206596672
          ]
        },
        {
          "k": 16,
          "n": 5,
          "mean": 4.612952862819812,
          "ci95": [
            2.917549842237179,
            6.308355883402445
          ]
        }
      ],
      "refinement_curve": [
        {
          "k": 1,
          "n": 3,
          "mean": 0.7529825636033161,
          "ci95": [
            -2.4227300361638298,
            3.9286951633704614
          ]
        },
        {
          "k": 2,
          "n": 3,
          "mean": 6.272881835201329,
          "ci95": [
            -16.09376093488501,
            28.63952460528767
          ]
        },
        {
          "k": 4,
          "n": 3,
          "mean": 9.693552838980882,
          "ci95": [
            -6.43512950166323,
            25.822235179624993
          ]
        },
        {
          "k": 8,
          "n": 3,
          "mean": 18.527884446639753,
          "ci95": [
            11.499461201711437,
            25.55630769156807
          ]
        },
        {
          "k": 10,
          "n": 3,
          "mean": 18.881589761627612,
          "ci95": [
            12.312944047273326,
            25.4502354759819
          ]
        },
        {
          "k": 16,
          "n": 3,
          "mean": 23.198525118622012,
          "ci95": [
            19.96070078461096,
            26.436349452633063
          ]
        },
        {
          "k": 32,
          "n": 3,
          "mean": 24.23868923643645,
          "ci95": [
            22.671476886262724,
            25.805901586610176
          ]
        },
        {
          "k": 64,
          "n": 3,
          "mean": 26.323764330174626,
          "ci95": [
            23.498330608480554,
            29.149198051868698
          ]
        }
      ]
    },
    {
      "model": "terra",
      "name": "GPT-5.6 Terra / Codex",
      "color": "#647b9c",
      "origination": {
        "n": 5,
        "mean": 4.489629114853982,
        "ci95": [
          3.4071730768801203,
          5.572085152827844
        ]
      },
      "refinement": {
        "n": 3,
        "mean": 22.097607812579284,
        "ci95": [
          17.6964150216855,
          26.498800603473068
        ]
      },
      "score": {
        "axes": [
          "origination",
          "refinement"
        ],
        "n": {
          "origination": 5,
          "refinement": 3
        },
        "axis_means": {
          "origination": 4.489629114853982,
          "refinement": 22.097607812579284
        },
        "mean": 141.77629409214833,
        "ci95": [
          99.78435041288515,
          183.76823777141152
        ],
        "definition": "fixed-reference-core-v1",
        "scales_pp": {
          "origination": 2.885429367211115,
          "refinement": 17.26969005796163
        },
        "normalized_axes": {
          "origination": 155.59656964306117,
          "refinement": 127.9560185412355
        }
      },
      "refinement_observations": [
        {
          "value": 20.15312343276798,
          "repetition": 2
        },
        {
          "value": 22.51919413200796,
          "repetition": 3
        },
        {
          "value": 23.620505872961914,
          "repetition": 7
        }
      ],
      "origination_curve": [
        {
          "k": 1,
          "n": 5,
          "mean": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        {
          "k": 2,
          "n": 5,
          "mean": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        {
          "k": 4,
          "n": 5,
          "mean": 0.3425399279822956,
          "ci95": [
            -0.6085033783989536,
            1.293583234363545
          ]
        },
        {
          "k": 8,
          "n": 5,
          "mean": 2.174565612603439,
          "ci95": [
            -0.04037719947232743,
            4.389508424679205
          ]
        },
        {
          "k": 10,
          "n": 5,
          "mean": 2.174565612603439,
          "ci95": [
            -0.04037719947232743,
            4.389508424679205
          ]
        },
        {
          "k": 16,
          "n": 5,
          "mean": 4.489629114853982,
          "ci95": [
            3.4071730768801203,
            5.572085152827844
          ]
        }
      ],
      "refinement_curve": [
        {
          "k": 1,
          "n": 3,
          "mean": 3.522889233845681,
          "ci95": [
            -11.63487974476543,
            18.68065821245679
          ]
        },
        {
          "k": 2,
          "n": 3,
          "mean": 4.168909230524997,
          "ci95": [
            -13.768459450271113,
            22.106277911321104
          ]
        },
        {
          "k": 4,
          "n": 3,
          "mean": 9.386375034607441,
          "ci95": [
            -10.919684199643223,
            29.692434268858108
          ]
        },
        {
          "k": 8,
          "n": 3,
          "mean": 12.729019726117372,
          "ci95": [
            -9.998276256798459,
            35.456315709033206
          ]
        },
        {
          "k": 10,
          "n": 3,
          "mean": 12.745812311602464,
          "ci95": [
            -9.909453363691956,
            35.40107798689688
          ]
        },
        {
          "k": 16,
          "n": 3,
          "mean": 17.918093932878403,
          "ci95": [
            13.738669746541929,
            22.097518119214875
          ]
        },
        {
          "k": 32,
          "n": 3,
          "mean": 20.37303365542422,
          "ci95": [
            14.230609140645363,
            26.515458170203075
          ]
        },
        {
          "k": 64,
          "n": 3,
          "mean": 22.097607812579284,
          "ci95": [
            17.6964150216855,
            26.498800603473068
          ]
        }
      ]
    },
    {
      "model": "luna",
      "name": "GPT-5.6 Luna / Codex",
      "color": "#38694e",
      "origination": {
        "n": 5,
        "mean": 4.260375828763727,
        "ci95": [
          2.54110187081513,
          5.979649786712324
        ]
      },
      "refinement": {
        "n": 3,
        "mean": 20.38226129202354,
        "ci95": [
          17.716385833943914,
          23.048136750103165
        ]
      },
      "score": {
        "axes": [
          "origination",
          "refinement"
        ],
        "n": {
          "origination": 5,
          "refinement": 3
        },
        "axis_means": {
          "origination": 4.260375828763727,
          "refinement": 20.38226129202354
        },
        "mean": 132.83734141129852,
        "ci95": [
          84.19873313117873,
          181.47594969141832
        ],
        "definition": "fixed-reference-core-v1",
        "scales_pp": {
          "origination": 2.885429367211115,
          "refinement": 17.26969005796163
        },
        "normalized_axes": {
          "origination": 147.65136437498566,
          "refinement": 118.0233184476114
        }
      },
      "refinement_observations": [
        {
          "value": 19.287034272569397,
          "repetition": 2
        },
        {
          "value": 20.427849713381903,
          "repetition": 3
        },
        {
          "value": 21.431899890119322,
          "repetition": 7
        }
      ],
      "origination_curve": [
        {
          "k": 1,
          "n": 5,
          "mean": 0.3425399279822956,
          "ci95": [
            -0.6085033783989536,
            1.293583234363545
          ]
        },
        {
          "k": 2,
          "n": 5,
          "mean": 0.3425399279822956,
          "ci95": [
            -0.6085033783989536,
            1.293583234363545
          ]
        },
        {
          "k": 4,
          "n": 5,
          "mean": 1.253932220095043,
          "ci95": [
            -0.278752912698097,
            2.786617352888183
          ]
        },
        {
          "k": 8,
          "n": 5,
          "mean": 2.515700281534439,
          "ci95": [
            0.2701785446735139,
            4.761222018395364
          ]
        },
        {
          "k": 10,
          "n": 5,
          "mean": 2.5220513213807285,
          "ci95": [
            0.2901695600957095,
            4.753933082665748
          ]
        },
        {
          "k": 16,
          "n": 5,
          "mean": 4.260375828763727,
          "ci95": [
            2.54110187081513,
            5.979649786712324
          ]
        }
      ],
      "refinement_curve": [
        {
          "k": 1,
          "n": 3,
          "mean": 4.377016808262432,
          "ci95": [
            -9.318189376834834,
            18.0722229933597
          ]
        },
        {
          "k": 2,
          "n": 3,
          "mean": 7.723628591527369,
          "ci95": [
            -3.3993071302329945,
            18.846564313287733
          ]
        },
        {
          "k": 4,
          "n": 3,
          "mean": 11.913860194562536,
          "ci95": [
            -8.204219840003658,
            32.03194022912873
          ]
        },
        {
          "k": 8,
          "n": 3,
          "mean": 15.547633702004818,
          "ci95": [
            0.7415827578579002,
            30.353684646151734
          ]
        },
        {
          "k": 10,
          "n": 3,
          "mean": 16.762421776948937,
          "ci95": [
            6.697322530631286,
            26.827521023266588
          ]
        },
        {
          "k": 16,
          "n": 3,
          "mean": 17.904613450912823,
          "ci95": [
            12.74055613835461,
            23.068670763471037
          ]
        },
        {
          "k": 32,
          "n": 3,
          "mean": 19.611105526016953,
          "ci95": [
            14.77372769433379,
            24.448483357700116
          ]
        },
        {
          "k": 64,
          "n": 3,
          "mean": 20.38226129202354,
          "ci95": [
            17.716385833943914,
            23.048136750103165
          ]
        }
      ]
    },
    {
      "model": "sonnet",
      "name": "Sonnet 5 / Claude Code",
      "color": "#936aa0",
      "origination": {
        "n": 5,
        "mean": 3.7549378927182055,
        "ci95": [
          1.2022227753936034,
          6.307653010042808
        ]
      },
      "refinement": {
        "n": 5,
        "mean": 21.903224914998336,
        "ci95": [
          18.480635297398152,
          25.32581453259852
        ]
      },
      "score": {
        "axes": [
          "origination",
          "refinement"
        ],
        "n": {
          "origination": 5,
          "refinement": 5
        },
        "axis_means": {
          "origination": 3.7549378927182055,
          "refinement": 21.903224914998336
        },
        "mean": 128.48245240432573,
        "ci95": [
          60.3180848378682,
          196.64681997078327
        ],
        "definition": "fixed-reference-core-v1",
        "scales_pp": {
          "origination": 2.885429367211115,
          "refinement": 17.26969005796163
        },
        "normalized_axes": {
          "origination": 130.13445885689816,
          "refinement": 126.83044595175329
        }
      },
      "refinement_observations": [
        {
          "value": 24.3820334643836,
          "repetition": 2
        },
        {
          "value": 18.915039853156724,
          "repetition": 3
        },
        {
          "value": 24.317339584955697,
          "repetition": 7
        },
        {
          "value": 18.99171723748608,
          "repetition": 5
        },
        {
          "value": 22.909994435009583,
          "repetition": 6
        }
      ],
      "origination_curve": [
        {
          "k": 1,
          "n": 5,
          "mean": 0.0,
          "ci95": [
            0.0,
            0.0
          ]
        },
        {
          "k": 2,
          "n": 5,
          "mean": 0.03894497860213823,
          "ci95": [
            -0.06918361660980127,
            0.14707357381407773
          ]
        },
        {
          "k": 4,
          "n": 5,
          "mean": 0.9263628724183997,
          "ci95": [
            -0.19911792719876953,
            2.051843672035569
          ]
        },
        {
          "k": 8,
          "n": 5,
          "mean": 2.0946975481073875,
          "ci95": [
            -0.43227110450445094,
            4.6216662007192255
          ]
        },
        {
          "k": 10,
          "n": 5,
          "mean": 2.257294660648489,
          "ci95": [
            -0.38877709042150643,
            4.903366411718485
          ]
        },
        {
          "k": 16,
          "n": 5,
          "mean": 3.7549378927182055,
          "ci95": [
            1.2022227753936034,
            6.307653010042808
          ]
        }
      ],
      "refinement_curve": [
        {
          "k": 1,
          "n": 5,
          "mean": 5.49057796952406,
          "ci95": [
            1.139371338600963,
            9.841784600447157
          ]
        },
        {
          "k": 2,
          "n": 5,
          "mean": 8.76397299740307,
          "ci95": [
            2.3536459203906714,
            15.174300074415466
          ]
        },
        {
          "k": 4,
          "n": 5,
          "mean": 15.226263806551463,
          "ci95": [
            11.933716088695512,
            18.518811524407415
          ]
        },
        {
          "k": 8,
          "n": 5,
          "mean": 16.32902003392497,
          "ci95": [
            13.703785530912619,
            18.95425453693732
          ]
        },
        {
          "k": 10,
          "n": 5,
          "mean": 16.87580023651497,
          "ci95": [
            14.75554579119031,
            18.996054681839627
          ]
        },
        {
          "k": 16,
          "n": 5,
          "mean": 18.105291346413274,
          "ci95": [
            17.01368115918384,
            19.19690153364271
          ]
        },
        {
          "k": 32,
          "n": 5,
          "mean": 19.351976346990106,
          "ci95": [
            17.53722955099421,
            21.166723142986005
          ]
        },
        {
          "k": 64,
          "n": 5,
          "mean": 21.903224914998336,
          "ci95": [
            18.480635297398152,
            25.32581453259852
          ]
        }
      ]
    }
  ]
}