{
  "schemaVersion": 1,
  "title": "AI code-review model pilot",
  "preparedAt": "2026-10-02",
  "publicationStatus": "approved anonymized aggregate scope; draft, not published",
  "design": {
    "models": 7,
    "pullRequestSnapshots": 4,
    "attemptsPerModelPerSnapshot": 2,
    "attemptedReviews": 56,
    "effort": "medium",
    "distinctDiscoveredRuntimeBugs": 4,
    "sourceSet": "private historical snapshots",
    "fixedAcrossModels": [
      "source snapshots",
      "review instructions",
      "configured tool catalog",
      "output contract",
      "scoring rules"
    ]
  },
  "definitions": {
    "bugs": "Independent runtime root causes supported by AI-assisted source audits, counted once per attempted four-PR round.",
    "falsePositives": "Independent accusations contradicted by source evidence, counted once per round. Unconfirmed claims are not silently called false positives.",
    "qualityDisplayed": "Source-audited delivered findings averaged across two attempted four-PR rounds, not a claim that every attempt passed protocol or parsing gates.",
    "cost": "Mean Standard API-equivalent reviewer token estimate across eight attempts/model; includes recorded invalid attempts; excludes setup, grading, advisors and graphic generation.",
    "time": "Median reviewer elapsed seconds across eight attempts/model, with retained bounds where timing is uncertain.",
    "formatFailure": "Raw output failed the findings format contract; separate source grading may still evaluate delivered text.",
    "harnessFailure": "Runner-reported execution failures. Later discovered protocol-gate violations are separate fields and are not certified passed by a zero here."
  },
  "results": [
    {
      "model": "gpt-5.4",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 0.5,
      "averageFalsePositivesPerFourPrRound": 2,
      "rounds": [
        {
          "round": 1,
          "bugs": 1,
          "falsePositives": 3
        },
        {
          "round": 2,
          "bugs": 0,
          "falsePositives": 1
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.38342587499999997,
        "upper": 0.38342587499999997
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 3.0674069999999998,
        "upper": 3.0674069999999998
      },
      "medianReviewSeconds": {
        "lower": 112.89699999999999,
        "upper": 120.04650000000001
      },
      "formatFailures": 0,
      "harnessFailures": 1,
      "tokenTotals": {
        "input": 3679128,
        "cachedInput": 3127808,
        "output": 60477,
        "reasoningOutput": 48128
      },
      "largestRecordedRequestInputTokens": 81655,
      "largestRecordedFractionOfCliReportedContextWindow": 0.31600232198142414,
      "recordedCompactions": 0
    },
    {
      "model": "gpt-5.5",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 2.5,
      "averageFalsePositivesPerFourPrRound": 1,
      "rounds": [
        {
          "round": 1,
          "bugs": 2,
          "falsePositives": 1
        },
        {
          "round": 2,
          "bugs": 3,
          "falsePositives": 1
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.67549575,
        "upper": 0.67549575
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 5.403966,
        "upper": 5.403966
      },
      "medianReviewSeconds": {
        "lower": 52.774,
        "upper": 52.774
      },
      "formatFailures": 1,
      "harnessFailures": 0,
      "tokenTotals": {
        "input": 3291432,
        "cachedInput": 2657792,
        "output": 30229,
        "reasoningOutput": 20747
      },
      "largestRecordedRequestInputTokens": 115174,
      "largestRecordedFractionOfCliReportedContextWindow": 0.44571981424148605,
      "recordedCompactions": 0
    },
    {
      "model": "gpt-5.6-luna",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 0,
      "averageFalsePositivesPerFourPrRound": 1.5,
      "rounds": [
        {
          "round": 1,
          "bugs": 0,
          "falsePositives": 1
        },
        {
          "round": 2,
          "bugs": 0,
          "falsePositives": 2
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.027649072499999996,
        "upper": 0.027649072499999996
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 0.22119257999999997,
        "upper": 0.22119257999999997
      },
      "medianReviewSeconds": {
        "lower": 54.7435,
        "upper": 54.7435
      },
      "formatFailures": 8,
      "harnessFailures": 0,
      "tokenTotals": {
        "input": 4106810,
        "cachedInput": 3615964,
        "output": 21814,
        "reasoningOutput": 12777
      },
      "largestRecordedRequestInputTokens": 80685,
      "largestRecordedFractionOfCliReportedContextWindow": 0.3122484520123839,
      "recordedCompactions": 0
    },
    {
      "model": "gpt-5.6-sol",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 2,
      "averageFalsePositivesPerFourPrRound": 0.5,
      "rounds": [
        {
          "round": 1,
          "bugs": 2,
          "falsePositives": 1
        },
        {
          "round": 2,
          "bugs": 2,
          "falsePositives": 0
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.533179575,
        "upper": 0.533179575
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 4.2654366,
        "upper": 4.2654366
      },
      "medianReviewSeconds": {
        "lower": 65.42399999999999,
        "upper": 65.42399999999999
      },
      "formatFailures": 0,
      "harnessFailures": 0,
      "tokenTotals": {
        "input": 3230046,
        "cachedInput": 2729719,
        "output": 33608,
        "reasoningOutput": 23148
      },
      "largestRecordedRequestInputTokens": 89566,
      "largestRecordedFractionOfCliReportedContextWindow": 0.34661764705882353,
      "recordedCompactions": 0,
      "protocolDeviation": {
        "nativeToolCatalogFailures": 1,
        "attemptsAfterMissedStop": 1,
        "strictEligibleAttempts": 6,
        "mainTotalsIncludeDeliveredClaimsPostHoc": true,
        "extraSourceObserved": false,
        "writesObserved": false,
        "strictSensitivity": {
          "plannedRoundDenominator": 2,
          "completedCompliantRounds": 1,
          "perRound": [
            {
              "round": 1,
              "bugsFound": 2,
              "falsePositives": 1,
              "unconfirmed": 1,
              "requiredDocumentation": 1
            },
            {
              "round": 2,
              "bugsFound": 1,
              "falsePositives": 0,
              "unconfirmed": 0,
              "requiredDocumentation": 0
            }
          ],
          "averageBugsFoundPerPlannedRound": 1.5,
          "averageFalsePositivesPerPlannedRound": 0.5,
          "accounting": "Exclude both attempts from all claim categories; retain all eight costs/times. Partial second round is not a clean review and this is not an alternative estimate of a complete compliant second round."
        },
        "note": "One extra native discovery call returned an empty list. The next attempt ran beyond the missed stop. Neither is certified compliant; all costs/time retained."
      }
    },
    {
      "model": "gpt-6-luna",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 1.5,
      "averageFalsePositivesPerFourPrRound": 0.5,
      "rounds": [
        {
          "round": 1,
          "bugs": 1,
          "falsePositives": 0
        },
        {
          "round": 2,
          "bugs": 2,
          "falsePositives": 1
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.01324594,
        "upper": 0.01324594
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 0.10596752,
        "upper": 0.10596752
      },
      "medianReviewSeconds": {
        "lower": 65.9825,
        "upper": 65.9825
      },
      "formatFailures": 6,
      "harnessFailures": 0,
      "tokenTotals": {
        "input": 3166214,
        "cachedInput": 2710272,
        "output": 43757,
        "reasoningOutput": 32045
      },
      "largestRecordedRequestInputTokens": 67870,
      "largestRecordedFractionOfCliReportedContextWindow": 0.2626547987616099,
      "recordedCompactions": 0
    },
    {
      "model": "gpt-6-sol",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 3.5,
      "averageFalsePositivesPerFourPrRound": 1,
      "rounds": [
        {
          "round": 1,
          "bugs": 4,
          "falsePositives": 1
        },
        {
          "round": 2,
          "bugs": 3,
          "falsePositives": 1
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.30824301249999997,
        "upper": 0.30824301249999997
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 2.4659440999999998,
        "upper": 2.4659440999999998
      },
      "medianReviewSeconds": {
        "lower": 60.564,
        "upper": 60.564
      },
      "formatFailures": 0,
      "harnessFailures": 0,
      "tokenTotals": {
        "input": 5017543,
        "cachedInput": 4512883,
        "output": 30189,
        "reasoningOutput": 13776
      },
      "largestRecordedRequestInputTokens": 72523,
      "largestRecordedFractionOfCliReportedContextWindow": 0.28066176470588233,
      "recordedCompactions": 0
    },
    {
      "model": "gpt-6.1-sol",
      "attemptedReviews": 8,
      "averageBugsPerFourPrRound": 4,
      "averageFalsePositivesPerFourPrRound": 0.5,
      "rounds": [
        {
          "round": 1,
          "bugs": 4,
          "falsePositives": 0
        },
        {
          "round": 2,
          "bugs": 4,
          "falsePositives": 1
        }
      ],
      "averageEstimatedCostUsdPerPr": {
        "lower": 0.23464515000000002,
        "upper": 0.2481225875
      },
      "totalEstimatedReviewerCostUsd": {
        "lower": 1.8771612000000002,
        "upper": 1.9849807
      },
      "medianReviewSeconds": {
        "lower": 64.0125,
        "upper": 64.0125
      },
      "formatFailures": 0,
      "harnessFailures": 0,
      "tokenTotals": {
        "input": 4061162,
        "cachedInput": 3475002,
        "output": 17216,
        "reasoningOutput": 4453
      },
      "largestRecordedRequestInputTokens": 101518,
      "largestRecordedFractionOfCliReportedContextWindow": 0.39287151702786377,
      "recordedCompactions": 0
    }
  ],
  "pricingSources": [
    "https://developers.openai.com/api/docs/models/gpt-5.4",
    "https://developers.openai.com/api/docs/models/gpt-5.5",
    "https://developers.openai.com/api/docs/models/gpt-5.6-luna",
    "https://developers.openai.com/api/docs/models/gpt-6-luna",
    "https://developers.openai.com/api/docs/models/gpt-6.1-sol",
    "https://developers.openai.com/api/docs/models/gpt-6-sol",
    "https://developers.openai.com/api/docs/models/gpt-5.6-sol",
    "https://developers.openai.com/api/docs/pricing"
  ],
  "limitations": [
    "Four selected changes, not a representative corpus.",
    "The discovered bug set is nonexhaustive; recall is not measured.",
    "Source adjudication was AI-assisted, not an independent human gold set.",
    "Private inputs and raw reviews are withheld; this public bundle does not reproduce the original runs.",
    "Sequential date-separated extensions, not a simultaneous randomized tournament.",
    "API-equivalent estimates are not invoices or full experiment costs.",
    "Raw format failures and the original invalid attempt remain accounted for.",
    "The benchmark review scope differs from production blocking-only policy; local compatibility is a separate check.",
    "Offline PR discussion tools return no existing comments. The pilot does not measure handling replies, intent or duplicate feedback.",
    "GPT-5.6 Sol includes one tool-catalog violation and one attempt after the missed stop. Observed-claim totals include them post hoc; strict exclusion sensitivity is provided."
  ]
}