{
  "dataset": "dixon.ai — The State of AI Reliability (a documented index)",
  "description": "A dated, versioned index of how five leading consumer AI assistants (ChatGPT, Claude, Gemini, Perplexity and Grok) answer real, checkable finance-and-regulation questions, each asked three times with memory off and web search on, every answer graded by hand against the primary source. A documented index, methodology fully disclosed — not a statistical benchmark.",
  "version": "v1",
  "run": {
    "date_iso": "2026-06-25",
    "date_human": "25–26 June 2026",
    "models": 5,
    "questions_core": 6,
    "rounds": 3,
    "runs_core": 90,
    "results_core": 30,
    "protocol": "Each assistant asked N=3 in temporary chats with memory off and web search on; every answer graded by hand against a primary source fixed before the run.",
    "grader": "Ben Dixon",
    "grader_note": "Grades applied by hand from real captured responses (N=3, memory-off temporary/incognito chats, web search on, graded same-day against the primary source) and signed off by Ben (the named grader) for publication, s118 / 2026-06-26."
  },
  "headline": {
    "confident_error": {
      "numerator": 3,
      "denominator": 30,
      "pct": 10,
      "definition": "A wrong or misleading value served as reliable (accuracy below full AND unhedged). Honest estimates and honest abstentions are excluded — they are good behaviour, not errors.",
      "note": "Result level: each of the 30 is one question-and-model, the verdict of three runs."
    },
    "fabrication": {
      "numerator": 0,
      "denominator": 30,
      "definition": "An outright fabrication is a figure invented with no source. Defined term, decoupled from the board's \"confidently wrong\" (honesty=0) shorthand: this run had zero inventions. Every error was a real figure served in the wrong field or gone stale."
    },
    "everyday_folded_in": {
      "numerator": 3,
      "denominator": 43,
      "pct": 7,
      "note": "Partial battery — the everyday (non-finance) cells are near-perfect but ChatGPT's are incomplete, so this is disclosed in-body, never the headline."
    }
  },
  "split_by_data_type": {
    "live_moving_data": {
      "results": 10,
      "confident_errors": 3,
      "pct": 30,
      "examples": "a stock's closing price, a live options quote"
    },
    "fixed_public_record": {
      "results": 20,
      "confident_errors": 0,
      "clean": 20,
      "examples": "a company's revenue, the Bank of England base rate, the S&P 500 dividend yield"
    }
  },
  "per_model": [
    {
      "model": "claude",
      "label": "Claude",
      "tier": "Max · paid",
      "correct": 6,
      "of": 6,
      "fully_honest": 6,
      "confident_errors": 0,
      "outright_fabrications": 0
    },
    {
      "model": "gemini",
      "label": "Gemini",
      "tier": "Pro · paid",
      "correct": 5,
      "of": 6,
      "fully_honest": 6,
      "confident_errors": 0,
      "outright_fabrications": 0
    },
    {
      "model": "grok",
      "label": "Grok",
      "tier": "Free · Grok 4.3 Fast",
      "correct": 5,
      "of": 6,
      "fully_honest": 6,
      "confident_errors": 0,
      "outright_fabrications": 0
    },
    {
      "model": "chatgpt",
      "label": "ChatGPT",
      "tier": "Free",
      "correct": 5,
      "of": 6,
      "fully_honest": 5,
      "confident_errors": 1,
      "outright_fabrications": 0
    },
    {
      "model": "perplexity",
      "label": "Perplexity",
      "tier": "Pro · paid",
      "correct": 4,
      "of": 6,
      "fully_honest": 4,
      "confident_errors": 2,
      "outright_fabrications": 0
    }
  ],
  "everyday_battery": {
    "clean": 12,
    "graded": 13,
    "note": "Same method on three everyday, non-finance questions (scaling a recipe, an Excel menu path, a UK faulty-goods refund). Near-perfect. The one blemish: Perplexity led a consumer-rights answer with a confident wrong \"three weeks is after the 30-day right\" (it is not) before self-correcting. ChatGPT partial (HTTP-431 cookie limit)."
  },
  "caveats": [
    "A documented index, methodology fully disclosed — not a statistical benchmark. Small N by design; every result links to a dated transcript.",
    "Results level, not response level: the 30 are question-and-model verdicts over 90 runs. No bare response-level percentage is published — the frozen cell grades do not decompose cleanly to one.",
    "Intermittency: two of the three confident errors appeared in a single run of three; one (Perplexity serving the intraday high as the close) reproduced across all three.",
    "Paid/free asymmetry is disclosed, never faked into parity — the worst performer (Perplexity) was a paid flagship; a free model (Grok) out-scored two paid ones.",
    "Point-in-time: models change between runs; this is a dated snapshot, re-cut as models ship."
  ],
  "source_reliability_teaser": {
    "is_a_separate_run": true,
    "run_date_iso": "2026-07-07",
    "run_date_human": "7 July 2026",
    "what_it_measures": "Not accuracy — sourcing. On six questions engineered to trap retrieval, does the page a model cites actually back the claim?",
    "sharpest_receipt": {
      "model": "Gemini",
      "tier_word": "Confident misattributor",
      "what_happened": "The right figures, but the £2,500 maximum was sourced to Police.uk, the crime-data portal, not to the gov.uk guide.",
      "why_it_fails": "Police.uk publishes recorded-crime statistics. It has no remit over the penalty. The fine is set out on gov.uk. The number was correct; the receipt did not back it.",
      "reproduced": "Held across all three rounds."
    },
    "full_axis": "https://dixon.ai/scoreboard/#source-reliability"
  },
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "cite": "The State of AI Reliability — DIXON.AI (Ben Dixon). Quote with attribution and a link to https://dixon.ai/state-of-ai-reliability/. Licence: CC BY 4.0.",
  "site": "https://dixon.ai",
  "human_readable": "https://dixon.ai/state-of-ai-reliability/",
  "sources": {
    "scoreboard": "https://dixon.ai/scoreboard/",
    "per_question_grades": "https://dixon.ai/scoreboard/rdri.json",
    "documented_log": "https://dixon.ai/lessons/"
  },
  "generated": "2026-07-09T18:07:57.871Z"
}