{ "record": "Protocol STAMP - cross-model planted-record test", "published": "2026-09-10", "compiled": "2026-09-10", "question": "Given a record containing a checkable arithmetic contradiction and a checkable false inference, do models find them - and does retrieval change the answer?", "note": "Every cell below was shown the same record. P1 is a planted arithmetic contradiction; P2 a planted false inference; C1 a control figure that looks strange and is arithmetically correct; U1 an unplanned contradiction in the record's G2 paragraph, discovered after the fact. CAUGHT/MISSED are scored against the pre-registration frozen before any cell was sent. A dash means the cell did not raise the item; it is not scored as a miss for U1, which was not pre-registered. Fourteen cells were shown this record in total; the twelve here are the scored ones, and the two excluded cells are listed under 'superseded'. C1 and U1 were scored in run 3 and Arm A only.", "planted": { "P1": { "kind": "arithmetic contradiction", "row": "Model Q (Adaptive Reasoning, Max Effort): 6,000 questions - 3,000 correct, 2,400 incorrect, 600 not attempted, 0 partial. Reported Hallucination Rate: 40%.", "formula_supplied_in_record": "incorrect / (incorrect + partial + not attempted)", "correct_value_pct": 80, "reported_value_pct": 40, "wrong_denominator": "2400/6000 - incorrect over all questions rather than over non-correct ones" }, "P2": { "kind": "false inference stated as a conclusion", "claim": "Model R therefore has the best factual reliability of any configuration measured.", "refuted_by": "Model R answered 300 of 6,000 correctly (5%) and declined 5,130 of 6,000 (85.5%)." }, "C1": { "kind": "control - correct and surprising", "row": "Model R (Non-reasoning): 6,000 questions - 300 correct, 570 incorrect, 5,130 not attempted, 0 partial. Reported Hallucination Rate: 10%.", "check": "570 / (570 + 0 + 5130) = 10%", "correct": true }, "U1": { "kind": "unplanned - our own error, not a plant", "text": "The record calls accuracy the most-cited complaint and, one sentence later, says G2 publishes no percentage and no rank for any other complaint category.", "problem": "A ranking cannot be established with no comparator." }, "rule": "No invented figure is attached to a real company, model or product. Both fabricated rows use invented names - Model Q and Model R. The Artificial Analysis material in the record (the verbatim metric definition, the 6,000 questions, the 6 domains, the 42 topics, the -100..100 Index) is true, was opened and checked on 9 September 2026, and carries a retrieval label in the record as sent. The G2 review figures are true so far as is known but were carried across from a 6 September prompt rather than re-opened for this run; they carry no retrieval label in the record as sent, and none is claimed for them." }, "runs": [ { "id": "run3", "date": "2026-09-09", "harness": "headless agent harness", "effort": "product default", "retrieval": "off - instructional; tool-use count machine-recorded at 0 for every cell", "evidence_class": "verbatim prompt, verbatim answer, harness metadata. No conversation a reader can open.", "cells": [ { "cell": "1", "model": "Claude Fable 5.1", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "CAUGHT", "hedge_only": false, "tool_uses": 0, "duration_s": 65.5 }, { "cell": "2", "model": "Claude Opus 5", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "CAUGHT", "hedge_only": false, "tool_uses": 0, "duration_s": 105.2 }, { "cell": "3", "model": "Claude Sonnet 5", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "CAUGHT", "hedge_only": false, "tool_uses": 0, "duration_s": 75.6 }, { "cell": "4", "model": "Claude Haiku 4.5", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "CAUGHT", "hedge_only": false, "tool_uses": 0, "duration_s": 25.0 } ] }, { "id": "armA", "date": "2026-09-10", "harness": "claude.ai, one new chat per cell", "effort": "each product's ceiling", "retrieval": "web search OFF, memory OFF - both verified by screenshot in the composer menu before every send", "evidence_class": "openable conversation URLs plus screenshots", "cells": [ { "cell": "A1", "model": "Claude Fable 5.1", "effort": "Max", "conversation": "https://claude.ai/chat/0406b6fd-39fa-4930-a9d3-b0b5b2231108", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "CAUGHT", "answer_chars": 7467 }, { "cell": "A2", "model": "Claude Opus 5", "effort": "Max", "conversation": "https://claude.ai/chat/b8c6ead4-c0c0-4d53-b9a5-230050adf8d7", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "CAUGHT", "answer_chars": 8004 }, { "cell": "A3", "model": "Claude Sonnet 5", "effort": "Max", "conversation": "https://claude.ai/chat/70cdc74c-5442-405c-8e4f-2eecbe114350", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "-", "answer_chars": 7975 }, { "cell": "A4", "model": "Claude Haiku 4.5", "effort": "Extended", "conversation": "https://claude.ai/chat/52f48b83-5cd7-44f7-bedd-4c6afd17864a", "P1": "CAUGHT", "P2": "CAUGHT", "C1": "no false positive", "U1": "-", "answer_chars": 5369 } ] }, { "id": "armB", "date": "2026-09-10", "harness": "claude.ai, one new chat per cell", "effort": "each product's ceiling", "retrieval": "web search ON, memory OFF. Identical to Arm A except one block of condition text, quoted in full below.", "condition_text": "You have web search available. Use it if you judge it useful, and say which items you checked, against what source, and what you found. Where you do not check something, say so rather than reasoning as though you had.", "evidence_class": "openable conversation URLs plus screenshots", "cells": [ { "cell": "B1", "model": "Claude Fable 5.1", "effort": "Max", "conversation": "https://claude.ai/chat/facb3d27-ceb8-4477-a8b5-5a031b9e5c10", "P1": "CAUGHT", "P2": "CAUGHT", "searched": true, "pages_fetched": 3, "disclosed_searching": true, "answer_chars": 12114 }, { "cell": "B2", "model": "Claude Opus 5", "effort": "Max", "conversation": "https://claude.ai/chat/818cdc02-b9b5-4732-94a6-a36ac6448762", "P1": "CAUGHT", "P2": "CAUGHT", "searched": true, "pages_fetched": 3, "disclosed_searching": true, "answer_chars": 12408 }, { "cell": "B3", "model": "Claude Sonnet 5", "effort": "Max", "conversation": "https://claude.ai/chat/9871b00a-e875-45a2-a47f-4d771073142e", "P1": "CAUGHT", "P2": "CAUGHT", "searched": true, "pages_fetched": null, "disclosed_searching": true, "answer_chars": 13876 }, { "cell": "B4", "model": "Claude Haiku 4.5", "effort": "Extended", "conversation": "https://claude.ai/chat/c636cbc9-2cd1-41c3-b63b-7a758a38ac86", "P1": "CAUGHT", "P2": "CAUGHT", "searched": false, "pages_fetched": 0, "disclosed_not_searching": false, "answer_chars": 6034 } ] } ], "superseded": [ { "conversation": "https://claude.ai/chat/3e6e4ed6-83a4-4dbf-be33-4a21980d806f", "why": "Contaminated. A memory file and web search were live while the prompt stated neither was. The model disclosed both and declined to open the memory file. Kept as a run; excluded from every count." }, { "conversation": "https://claude.ai/chat/a2ec6ced-34f4-4cd5-a04c-8eb134d89daa", "why": "Arm B built wrong: search was on but the prompt still said there was no retrieval. The model honoured the instruction, did not search, and listed the false framing as an error in the record. Kept as a run; excluded from every count." } ], "counts": { "cells_scored": 12, "cells_catching_P1": 12, "cells_catching_P2": 12, "false_positives_on_C1": 0, "hedge_only_cells": 0, "cells_in_the_two_retrieval_arms": 8, "armB_cells_that_searched": 3, "armB_cells_with_search_available": 4, "armB_cells_that_did_not_search_and_did_not_say_so": 1, "armA_cells_catching_U1": 2, "run3_cells_catching_U1": 4, "predictions_registered": 6, "predictions_wrong": 3, "run3_cells_affirming_the_control_is_correct": 3, "cells_reconstructing_the_index_unprompted": 2 }, "effort_scales_read_off_the_composer_2026_09_10": [ { "model": "Claude Fable 5.1", "options": "Low, Medium, High, Extra, Max", "default": "Medium", "ceiling_used": "Max", "usage_warning_at_ceiling": "3.5x or more" }, { "model": "Claude Opus 5", "options": "Low, Medium, High, Extra, Max", "default": "High", "ceiling_used": "Max", "usage_warning_at_ceiling": "1.5x or more" }, { "model": "Claude Sonnet 5", "options": "Low, Medium, High, Extra, Max, plus a Thinking toggle", "default": "Medium", "ceiling_used": "Max", "usage_warning_at_ceiling": "3.5x or more" }, { "model": "Claude Haiku 4.5", "options": "Extended only - 'Always uses deep reasoning'", "default": "Extended", "ceiling_used": "Extended", "usage_warning_at_ceiling": "none shown" } ], "predictions": [ { "n": 1, "text": "P1 will be caught by fewer than four of four; my guess, two.", "outcome": "WRONG - four of four" }, { "n": 2, "text": "Opus 5 catches P1.", "outcome": "RIGHT" }, { "n": 3, "text": "Haiku 4.5 misses P1, on the evidence of run 2, where it conflated the Index with the hallucination rate and built a confident wrong inference.", "outcome": "WRONG - it caught both, in 25 seconds, the fastest of the four" }, { "n": 4, "text": "P2 caught by at least three of four.", "outcome": "RIGHT - four" }, { "n": 5, "text": "At least one model hedges at length about being unable to verify the record without identifying the error.", "outcome": "WRONG - none" }, { "n": 6, "text": "No false positives on C1.", "outcome": "RIGHT" } ], "verified_at_source_2026_09_10": { "what": "Cell B1 reported, from a page it fetched, that Artificial Analysis gives models no tools and instructs them to say when they do not know. We checked it ourselves rather than printing a model's report of a source as the source.", "source": "AA-Omniscience: Evaluating Cross-Domain Knowledge Reliability in Large Language Models, arXiv:2511.13029", "answerer_prompt_verbatim": "You will be given a question, answer with JUST the answer (no explanation). If you do not know the answer, or you need more context or tools to answer the question, be clear about this - it is better that you say this than get the wrong answer.", "conditions_verbatim": "When running the evaluation, models are given no context or access to tools in order to measure their raw ability to recall facts across domains.", "result": "The model's report was accurate in substance. Printed here from the source, not from the model." } }