{
  "dataset": "dixon.ai — the AI Reliability Scoreboard (documented errors and catches per model)",
  "description": "A running league table of how many specific, observable, falsifiable AI failures — and genuine catches — Ben Dixon has documented per model (ChatGPT, Claude, Gemini, Perplexity) while using them on real investing decisions. Every count is derived at build time from the findings and catches recorded on published posts; each model links back to the individual /lessons and /catches entries behind its tally.",
  "methodology": "This is a record of what happened in real tests, not a benchmark. The same prompt was not run an equal number of times against every model, the cases were not sampled at random, and the totals reflect what got written up — so they measure documented evidence, not relative reliability. Raw counts only; no rates or percentages are published because the sample is far too small to support them.",
  "disclaimer": "Small sample (low tens of cases). Treat these as documented cases, not a failure rate. A model with more documented errors here was not necessarily tested more, nor is it necessarily less reliable — it may simply have been used more, or failed in more screenshot-worthy ways. Human-verified by the author before publication.",
  "cite": "Quote freely with attribution to dixon.ai and a link to an entry URL. Author: Ben Dixon. Licence for this dataset: CC BY 4.0.",
  "site": "https://dixon.ai",
  "human_readable": "https://dixon.ai/scoreboard/",
  "sources": {
    "errors": "https://dixon.ai/lessons.json",
    "catches": "https://dixon.ai/catches.json"
  },
  "generated": "2026-06-24T17:47:04.244Z",
  "totals": {
    "models_scored": 4,
    "documented_errors": 20,
    "documented_catches": 15
  },
  "models": [
    {
      "tool": "perplexity",
      "label": "Perplexity",
      "documented_errors": 7,
      "documented_catches": 2,
      "errors_by_type": {
        "outdated-rule-stated-as-current": 1,
        "ignored-constraint": 1,
        "unit-error": 2,
        "linear-scaling-of-non-linear-quantity": 1,
        "inconsistent-5yr-returns": 1,
        "low-authority-source-led": 1
      },
      "errors_by_hallucination_type": {
        "factuality": 1
      },
      "errors_by_consequence": {
        "decision-affecting": 4,
        "cosmetic-but-revealing": 2
      },
      "catches_by_type": {
        "correct-source-attribution": 1,
        "honest-substitution": 1
      },
      "catches_by_impact": {
        "decision-relevant": 1,
        "validation-only": 1
      },
      "error_entries": [
        {
          "id": "perplexity-low-authority-source-led-2026-06-20",
          "url": "https://dixon.ai/lessons/#perplexity-low-authority-source-led-2026-06-20",
          "date": "2026-06-20"
        },
        {
          "id": "perplexity-outdated-rule-stated-as-current-2026-06-19",
          "url": "https://dixon.ai/lessons/#perplexity-outdated-rule-stated-as-current-2026-06-19",
          "date": "2026-06-19"
        },
        {
          "id": "perplexity-linear-scaling-of-non-linear-quantity-2026-06-19",
          "url": "https://dixon.ai/lessons/#perplexity-linear-scaling-of-non-linear-quantity-2026-06-19",
          "date": "2026-06-19"
        },
        {
          "id": "perplexity-inconsistent-5yr-returns-2026-06-13",
          "url": "https://dixon.ai/lessons/#perplexity-inconsistent-5yr-returns-2026-06-13",
          "date": "2026-06-13"
        },
        {
          "id": "perplexity-ignored-constraint-2026-05-15",
          "url": "https://dixon.ai/lessons/#perplexity-ignored-constraint-2026-05-15",
          "date": "2026-05-15"
        },
        {
          "id": "perplexity-unit-error-2026-05-15",
          "url": "https://dixon.ai/lessons/#perplexity-unit-error-2026-05-15",
          "date": "2026-05-15"
        },
        {
          "id": "perplexity-unit-error-2026-05-15",
          "url": "https://dixon.ai/lessons/#perplexity-unit-error-2026-05-15",
          "date": "2026-05-15"
        }
      ],
      "catch_entries": [
        {
          "id": "perplexity-correct-source-attribution-2026-06-20",
          "url": "https://dixon.ai/catches/#perplexity-correct-source-attribution-2026-06-20",
          "date": "2026-06-20"
        },
        {
          "id": "perplexity-honest-substitution-2026-06-18",
          "url": "https://dixon.ai/catches/#perplexity-honest-substitution-2026-06-18",
          "date": "2026-06-18"
        }
      ]
    },
    {
      "tool": "gemini",
      "label": "Gemini",
      "documented_errors": 6,
      "documented_catches": 1,
      "errors_by_type": {
        "fabrication": 2,
        "partial-fabrication": 1,
        "wrong-entity-audit": 1,
        "stale-memory-as-current": 1,
        "unprompted-cross-conversation-memory": 1
      },
      "errors_by_hallucination_type": {
        "faithfulness": 1
      },
      "errors_by_consequence": {
        "decision-affecting": 4,
        "cosmetic-but-revealing": 1
      },
      "catches_by_type": {
        "entity-overlap-risk": 1
      },
      "catches_by_impact": {
        "decision-relevant": 1
      },
      "error_entries": [
        {
          "id": "gemini-unprompted-cross-conversation-memory-2026-06-13",
          "url": "https://dixon.ai/lessons/#gemini-unprompted-cross-conversation-memory-2026-06-13",
          "date": "2026-06-13"
        },
        {
          "id": "gemini-wrong-entity-audit-2026-06-10",
          "url": "https://dixon.ai/lessons/#gemini-wrong-entity-audit-2026-06-10",
          "date": "2026-06-10"
        },
        {
          "id": "gemini-stale-memory-as-current-2026-06-10",
          "url": "https://dixon.ai/lessons/#gemini-stale-memory-as-current-2026-06-10",
          "date": "2026-06-10"
        },
        {
          "id": "gemini-partial-fabrication-2026-05-22",
          "url": "https://dixon.ai/lessons/#gemini-partial-fabrication-2026-05-22",
          "date": "2026-05-22"
        },
        {
          "id": "gemini-fabrication-2026-05-16",
          "url": "https://dixon.ai/lessons/#gemini-fabrication-2026-05-16",
          "date": "2026-05-16"
        },
        {
          "id": "gemini-fabrication-2026-05-15",
          "url": "https://dixon.ai/lessons/#gemini-fabrication-2026-05-15",
          "date": "2026-05-15"
        }
      ],
      "catch_entries": [
        {
          "id": "gemini-entity-overlap-risk-2026-06-10",
          "url": "https://dixon.ai/catches/#gemini-entity-overlap-risk-2026-06-10",
          "date": "2026-06-10"
        }
      ]
    },
    {
      "tool": "chatgpt",
      "label": "ChatGPT",
      "documented_errors": 4,
      "documented_catches": 0,
      "errors_by_type": {
        "outdated-rule-stated-as-current": 1,
        "web-confabulation": 1,
        "fabricated-premium-table": 1,
        "misattributed-source": 1
      },
      "errors_by_hallucination_type": {
        "factuality": 1
      },
      "errors_by_consequence": {
        "decision-affecting": 4
      },
      "catches_by_type": {},
      "catches_by_impact": {},
      "error_entries": [
        {
          "id": "chatgpt-misattributed-source-2026-06-20",
          "url": "https://dixon.ai/lessons/#chatgpt-misattributed-source-2026-06-20",
          "date": "2026-06-20"
        },
        {
          "id": "chatgpt-outdated-rule-stated-as-current-2026-06-19",
          "url": "https://dixon.ai/lessons/#chatgpt-outdated-rule-stated-as-current-2026-06-19",
          "date": "2026-06-19"
        },
        {
          "id": "chatgpt-fabricated-premium-table-2026-06-18",
          "url": "https://dixon.ai/lessons/#chatgpt-fabricated-premium-table-2026-06-18",
          "date": "2026-06-18"
        },
        {
          "id": "chatgpt-web-confabulation-2026-05-16",
          "url": "https://dixon.ai/lessons/#chatgpt-web-confabulation-2026-05-16",
          "date": "2026-05-16"
        }
      ],
      "catch_entries": []
    },
    {
      "tool": "claude",
      "label": "Claude",
      "documented_errors": 3,
      "documented_catches": 12,
      "errors_by_type": {
        "inferred-input": 1,
        "stale-prompt-framing": 1,
        "stale-figure-with-web-search": 1
      },
      "errors_by_hallucination_type": {
        "factuality": 1
      },
      "errors_by_consequence": {
        "decision-affecting": 1,
        "cosmetic-but-revealing": 1
      },
      "catches_by_type": {
        "language-tell": 3,
        "flagged-uncertainty-and-verified": 1,
        "reframe": 1,
        "non-recurring-strip": 1,
        "stayed-in-lane": 1,
        "unit-error-flag": 1,
        "asymmetry-tell": 1,
        "stale-data-flag": 1,
        "non-linear-constraint-flag": 1,
        "searched-before-answering-changed-rule": 1
      },
      "catches_by_impact": {
        "decision-relevant": 9,
        "insight-relevant": 3
      },
      "error_entries": [
        {
          "id": "claude-stale-figure-with-web-search-2026-06-13",
          "url": "https://dixon.ai/lessons/#claude-stale-figure-with-web-search-2026-06-13",
          "date": "2026-06-13"
        },
        {
          "id": "claude-stale-prompt-framing-2026-05-20",
          "url": "https://dixon.ai/lessons/#claude-stale-prompt-framing-2026-05-20",
          "date": "2026-05-20"
        },
        {
          "id": "claude-inferred-input-2026-05-16",
          "url": "https://dixon.ai/lessons/#claude-inferred-input-2026-05-16",
          "date": "2026-05-16"
        }
      ],
      "catch_entries": [
        {
          "id": "claude-flagged-uncertainty-and-verified-2026-06-19",
          "url": "https://dixon.ai/catches/#claude-flagged-uncertainty-and-verified-2026-06-19",
          "date": "2026-06-19"
        },
        {
          "id": "claude-non-linear-constraint-flag-2026-06-19",
          "url": "https://dixon.ai/catches/#claude-non-linear-constraint-flag-2026-06-19",
          "date": "2026-06-19"
        },
        {
          "id": "claude-searched-before-answering-changed-rule-2026-06-19",
          "url": "https://dixon.ai/catches/#claude-searched-before-answering-changed-rule-2026-06-19",
          "date": "2026-06-19"
        },
        {
          "id": "claude-language-tell-2026-06-18",
          "url": "https://dixon.ai/catches/#claude-language-tell-2026-06-18",
          "date": "2026-06-18"
        },
        {
          "id": "claude-unit-error-flag-2026-06-18",
          "url": "https://dixon.ai/catches/#claude-unit-error-flag-2026-06-18",
          "date": "2026-06-18"
        },
        {
          "id": "claude-non-recurring-strip-2026-06-11",
          "url": "https://dixon.ai/catches/#claude-non-recurring-strip-2026-06-11",
          "date": "2026-06-11"
        },
        {
          "id": "claude-reframe-2026-05-22",
          "url": "https://dixon.ai/catches/#claude-reframe-2026-05-22",
          "date": "2026-05-22"
        },
        {
          "id": "claude-asymmetry-tell-2026-05-22",
          "url": "https://dixon.ai/catches/#claude-asymmetry-tell-2026-05-22",
          "date": "2026-05-22"
        },
        {
          "id": "claude-stale-data-flag-2026-05-20",
          "url": "https://dixon.ai/catches/#claude-stale-data-flag-2026-05-20",
          "date": "2026-05-20"
        },
        {
          "id": "claude-language-tell-2026-05-15",
          "url": "https://dixon.ai/catches/#claude-language-tell-2026-05-15",
          "date": "2026-05-15"
        },
        {
          "id": "claude-language-tell-2026-05-15",
          "url": "https://dixon.ai/catches/#claude-language-tell-2026-05-15",
          "date": "2026-05-15"
        },
        {
          "id": "claude-stayed-in-lane-2026-05-14",
          "url": "https://dixon.ai/catches/#claude-stayed-in-lane-2026-05-14",
          "date": "2026-05-14"
        }
      ]
    }
  ]
}