{
  "name": "Aevral receipts: pull-request review eval runs",
  "url": "https://aevral.com/receipts",
  "description": "Runs of the frozen PR review eval suite, published as measured: what was tested, what was found, what missed, and the configuration that produced each.",
  "note": "Each recall number is the result of that named run on that corpus. It is not a product accuracy rate, and results on your code depend on your code. New runs of the customer setup get appended to this table as they happen, worse results included.",
  "caption": "Fabricated counts a finding published on a benign plant. On the founding runs of 25 September 2026, all 18 benign look-alikes stayed silent. The internal gate behind a configuration change sets a recall bar of 83.3% on the general corpus and a bar of 10 of 12 on the frozen authorization slice, with zero fabricated; a published finding that does not match a planted line fails a run too.",
  "dateModified": "2026-10-01",
  "columns": [
    {
      "key": "run",
      "description": "Run name"
    },
    {
      "key": "tested",
      "description": "What was tested"
    },
    {
      "key": "corpus",
      "description": "Corpus"
    },
    {
      "key": "recall",
      "description": "Recall on the general corpus for that run"
    },
    {
      "key": "fabricated",
      "description": "Findings published on a benign plant"
    },
    {
      "key": "authz",
      "description": "Result on the frozen authorization slice"
    },
    {
      "key": "configuration",
      "description": "Configuration that produced the run"
    },
    {
      "key": "date",
      "description": "Run date"
    }
  ],
  "runs": [
    {
      "run": "Run 1",
      "tested": "PR review, general scope, three passes per planted pull request",
      "corpus": "57 planted pull requests: 27 flaw plants under the general class list, 12 frozen authorization plants, 18 benign look-alikes",
      "recall": "26 of 27 (96.3%)",
      "fabricated": "0",
      "authz": "10 of 12",
      "configuration": "The production PR review lane, reasoning effort low",
      "date": "25 September 2026"
    },
    {
      "run": "Run 2",
      "tested": "PR review, general scope, three passes per planted pull request",
      "corpus": "57 planted pull requests: 27 flaw plants under the general class list, 12 frozen authorization plants, 18 benign look-alikes",
      "recall": "25 of 27 (92.6%)",
      "fabricated": "0",
      "authz": "11 of 12",
      "configuration": "The production PR review lane, reasoning effort low",
      "date": "25 September 2026"
    },
    {
      "run": "Run 3",
      "tested": "PR review, general scope, engine v2, the paid-plan lane, three passes per planted pull request",
      "corpus": "53 planted pull requests: 24 multi-line flaw plants in real repository files, 3 unproven-sink plants (the wanted outcome is an unverified listing), 24 clean control pull requests including the adversarial set, 2 large-PR coverage plants",
      "recall": "Median 21 of 24 across 3 repeats (22, 21, 17); 60 of 72 plant runs published",
      "fabricated": "0 across 72 clean-control runs",
      "authz": "Not a separate slice of this corpus",
      "configuration": "The production PR review lane, reasoning effort low",
      "date": "1 October 2026"
    },
    {
      "run": "Run 4",
      "tested": "PR review, general scope, engine v2 release build, the free-tier lane, three passes per planted pull request",
      "corpus": "43 planted pull requests: 24 multi-line flaw plants in real repository files, 17 clean control pull requests including the adversarial set, 2 large-PR coverage plants",
      "recall": "Median 16 of 24 across 3 repeats (16, 15, 21); 52 of 72 plant runs published",
      "fabricated": "1 in the worst repeat (2 of 51 clean-control runs)",
      "authz": "Not a separate slice of this corpus",
      "configuration": "The production PR review lane, reasoning effort low",
      "date": "1 October 2026"
    }
  ]
}