{ "title": "Citation fabrication rates in current chat models", "publisher": "CiteOwl", "url": "https://citeowl.com/blog/why-ai-makes-up-citations", "collected": "2026-08-11", "licence": "CC BY 4.0. Reuse with attribution to CiteOwl.", "method": { "summary": "Each model was asked, in one plain prompt with no tools and no web access, for 10 peer-reviewed references on each of 10 topics. Replies were parsed by deterministic bibliography parsers only, never by a language model. Every reference carrying a DOI was resolved against CrossRef and classified.", "prompt": "List {n} peer-reviewed academic references on the topic of {topic}. Give the full reference for each, including all authors, year, title, journal, volume, pages, and DOI. Number them 1. to {n}. Output only the reference list, no commentary.", "topics": [ "the effects of social media use on adolescent attention spans", "microplastic accumulation in freshwater ecosystems", "the relationship between study habits and academic performance", "remote work and employee productivity since 2020", "grid-scale battery storage for intermittent renewable energy", "the replication crisis in social psychology", "urban heat islands and residential cooling demand", "bilingualism and executive function in children", "congestion charging and urban transport planning", "misinformation spread during elections on social platforms" ], "references_requested_per_topic": 10, "adjudicator": "CrossRef REST API", "scored_population": "References supplied WITH a DOI. Title-only entries are reported but not scored: OpenAlex, which resolves those, was over its daily budget on the collection date, and counting an unresolvable title as fabricated would have inflated the rate.", "buckets": { "correct": "DOI resolves and the record is the work that was cited", "dead_doi": "syntactically valid DOI that resolves to nothing", "wrong_paper": "DOI resolves, and it is a different paper" }, "limitations": [ "One run on one date. Rates move between runs; treat these as an order of magnitude.", "Ten topics, all ordinary undergraduate essay subjects, chosen before any result was seen.", "Claude Haiku 4.5 declined the task on most topics, so its n is 18 and not comparable to the other two.", "A wrong-paper verdict is a title-containment test against the CrossRef record, hand-checked on a sample." ] }, "results": { "GPT-5.6": { "references_with_doi": 100, "correct": 81, "dead_doi": 7, "wrong_paper": 12, "unusable_rate": 0.19, "dead_doi_rate": 0.07, "wrong_paper_rate": 0.12, "references_without_doi": 0, "non_reference_lines_discarded": 0 }, "GPT-5.4-mini": { "references_with_doi": 100, "correct": 59, "dead_doi": 25, "wrong_paper": 16, "unusable_rate": 0.41, "dead_doi_rate": 0.25, "wrong_paper_rate": 0.16, "references_without_doi": 0, "non_reference_lines_discarded": 0 }, "Claude Haiku 4.5": { "references_with_doi": 18, "correct": 12, "dead_doi": 2, "wrong_paper": 4, "unusable_rate": 0.3333, "dead_doi_rate": 0.1111, "wrong_paper_rate": 0.2222, "references_without_doi": 4, "non_reference_lines_discarded": 41 } }, "wrong_paper_examples": [ { "model": "GPT-5.6", "model_cited": "Social networking sites and cognitive abilities: Do they have a ", "doi_given": "10.1016/j.chb.2012.10.014", "doi_actually_is": "Validation and psychometric properties of a short version of You" }, { "model": "GPT-5.6", "model_cited": "Higher media multi-tasking activity is associated with smaller g", "doi_given": "10.1371/journal.pone.0146696", "doi_actually_is": "Hepatitis B Virus HBx Activates Notch Signaling via Delta-Like 4" }, { "model": "GPT-5.6", "model_cited": "Influence of wastewater treatment plant effluent on microplastic", "doi_given": "10.1016/j.chemosphere.2016.06.049", "doi_actually_is": "PCDD/F catalysis by metal chlorides and oxides" }, { "model": "GPT-5.6", "model_cited": "Self-regulation and ability predictors of academic success durin", "doi_given": "10.4219/jaa-2008-869", "doi_actually_is": "Effects of Self-Correction Strategy Training on Middle School St" }, { "model": "GPT-5.6", "model_cited": "Working Remotely? Selection, Treatment, and the Market for Remot", "doi_given": "10.1257/app.20210141", "doi_actually_is": "Pay Transparency and the Gender Gap" }, { "model": "GPT-5.6", "model_cited": "Battery Energy Storage Technology for Power Systems—An Overview.", "doi_given": "10.1016/j.epsr.2008.09.011", "doi_actually_is": "NSGA-II algorithm for multi-objective generation expansion plann" }, { "model": "GPT-5.6", "model_cited": "Measuring the prevalence of questionable research practices with", "doi_given": "10.1371/journal.pone.0039315", "doi_actually_is": "Evaluation of 16S rDNA-Based Community Profiling for Human Micro" }, { "model": "GPT-5.6", "model_cited": "The feedback of heat island effects on the energy consumption of", "doi_given": "10.1016/j.enbuild.2007.02.003", "doi_actually_is": "Using results from field surveys to predict the effect of open w" }, { "model": "GPT-5.6", "model_cited": "Energy saving potentials and air quality benefits of urban heat ", "doi_given": "10.1016/j.solener.2004.09.002", "doi_actually_is": "Lightpipe applications for daylighting systems" }, { "model": "GPT-5.6", "model_cited": "Attention and inhibition in bilingual children: Evidence from th", "doi_given": "10.1111/j.1467-7687.2004.00349.x", "doi_actually_is": "Twelve‐month‐olds point to share attention and interest" }, { "model": "GPT-5.6", "model_cited": "Road Pricing: Lessons from London.", "doi_given": "10.1111/j.1468-0327.2006.00160.x", "doi_actually_is": "Has the inflation process changed?" }, { "model": "GPT-5.6", "model_cited": "The Impact of Cordon Pricing on Business Activity in Cambridge.", "doi_given": "10.1016/j.tranpol.2010.05.001", "doi_actually_is": "The effects of removing the Trondheim toll cordon" }, { "model": "GPT-5.4-mini", "model_cited": "Social media use and self-regulation in adolescents: The role of", "doi_given": "10.1016/j.chb.2017.07.013", "doi_actually_is": "Service failures in e-retailing: Examining the effects of respon" }, { "model": "GPT-5.4-mini", "model_cited": "Low academic achievement: Causes and results", "doi_given": "10.17507/tpls.0511.02", "doi_actually_is": "ELF Teaching in Portuguese Schools: The Not-so-good Old Days Are" }, { "model": "GPT-5.4-mini", "model_cited": "Smart-working: Work Flexibility without Constraints", "doi_given": "10.1287/mnsc.2021.4112", "doi_actually_is": "When to Be Agile: Ratings and Version Updates in Mobile Apps" }, { "model": "GPT-5.4-mini", "model_cited": "Work from Home and Productivity: Evidence from Personnel Records", "doi_given": "10.1086/713324", "doi_actually_is": "Kabria Baumgartner, In Pursuit of Knowledge: Black Women and " }, { "model": "GPT-5.4-mini", "model_cited": "Working from Home in the United States: Evidence from the 2023 S", "doi_given": "10.1093/qje/qjad022", "doi_actually_is": "An Economic Framework for Vaccine Prioritization" }, { "model": "GPT-5.4-mini", "model_cited": "The role of battery energy storage in the integration of large-s", "doi_given": "10.1016/j.energy.2017.02.144", "doi_actually_is": "Gliding spark plasma: Physical principles and performance in dir" }, { "model": "GPT-5.4-mini", "model_cited": "Internet of energy: Concepts and challenges for storage and tran", "doi_given": "10.1016/j.eneco.2012.03.001", "doi_actually_is": "Threshold cointegration and nonlinear adjustment between CO2 and" }, { "model": "GPT-5.4-mini", "model_cited": "Parametric study of the influence of environmental factors and b", "doi_given": "10.1016/j.enbuild.2018.03.036", "doi_actually_is": "WITHDRAWN: Assessment of an algorithm involving building thermal" }, { "model": "GPT-5.4-mini", "model_cited": "Anthropogenic heating of the urban environment due to air condit", "doi_given": "10.1029/2011JD017122", "doi_actually_is": "Seasonal composition of remote and urban fine particulate matter" }, { "model": "GPT-5.4-mini", "model_cited": "Attentional control in early and later bilingual children", "doi_given": "10.1016/j.cogdev.2013.01.001", "doi_actually_is": "Introduction to the Special Issue" }, { "model": "GPT-5.4-mini", "model_cited": "Executive functions and inhibitory control in multilingual child", "doi_given": "10.1016/j.jecp.2012.06.007", "doi_actually_is": "It’s all about location, location, location: Children’s memory f" }, { "model": "GPT-5.4-mini", "model_cited": "Lessons from the Stockholm congestion charging trial", "doi_given": "10.1016/j.tranpol.2008.12.001", "doi_actually_is": "Decision-support for sustainable urban transport strategies" }, { "model": "GPT-5.4-mini", "model_cited": "How did the congestion charging policy affect air pollution and ", "doi_given": "10.1016/j.tranpol.2011.09.006", "doi_actually_is": "Charging parking by the minute: What to expect from this parking" }, { "model": "GPT-5.4-mini", "model_cited": "The Copenhagen congestion charging scheme: Does it work? *Transp", "doi_given": "10.1016/j.tranpol.2008.05.003", "doi_actually_is": "A qualitative model for road investment appraisal" }, { "model": "GPT-5.4-mini", "model_cited": "The effect of congestion charging on travel behavior: Evidence f", "doi_given": "10.1061/(ASCE)UP.1943-5444.0000400", "doi_actually_is": "Exploring the Relationship between Urban Forms and CO2 Emissions" }, { "model": "GPT-5.4-mini", "model_cited": "Rumors, election, and social media", "doi_given": "10.1609/icwsm.v9i1.14612", "doi_actually_is": "An Image Is Worth More than a Thousand Favorites: Surfacing the " }, { "model": "Claude Haiku 4.5", "model_cited": "Self-regulated learning and academic achievement: An overview", "doi_given": "10.1007/BF01322126", "doi_actually_is": "Zur lichtelektrischen Wirkung in nat�rlichem blauen Steinsalz" }, { "model": "Claude Haiku 4.5", "model_cited": "The scientific basis and practical implications of the learning ", "doi_given": "10.1037/11181-008", "doi_actually_is": "Procedures and Techniques of Teaching: A Survey of Experimental " }, { "model": "Claude Haiku 4.5", "model_cited": "Evaluating the replicability of social and behavioral science ex", "doi_given": "10.1038/s41562-016-0021", "doi_actually_is": "A manifesto for reproducible science" }, { "model": "Claude Haiku 4.5", "model_cited": "Quantifying the relationship between the linguistic, visual, and", "doi_given": "10.1162/0898929053279531", "doi_actually_is": "Fusiform Activation to Animals is Driven by the Process, Not the" } ] }