@article{4794,
  author = {Pit Pichappan},
  title = {Comprehensive Evaluation of Fake News Detection Models: Calibration, Error Analysis, and Statistical Significance on the TruthSeeker Database},
  journal = {Journal of Information Security Research},
  year = {2026},
  volume = {17},
  number = {3},
  doi = {https://doi.org/10.6025/jisr/2026/17/3/155-183},
  url = {https://www.dline.info/jisr/fulltext/v17n3/jisrv17n3_3.pdf},
  abstract = {The proliferation of fake news on social media poses significant societal threats, necessitating automated
detection systems that are not only accurate but also trustworthy. While recent transformer based models
have achieved high classification performance, their probability calibration and prediction reliability remain
underexplored. This study presents a comprehensive evaluation framework for fake news detection using
the large scale TruthSeeker2023 dataset, comparing traditional machine learning models (Logistic
Regression, Random Forest, XGBoost) with transformer based architectures (BERT, RoBERTa). Beyond
standard accuracy metrics, we systematically assess model reliability through Expected Calibration Error
(ECE), Maximum Calibration Error (MCE), Brier Score, reliability diagrams, and confidence distribution
analysis. Results demonstrate that RoBERTa achieves the lowest calibration error (ECE = 0.017) and superior
discriminative performance (AUC  0.96), while traditional models exhibit significant miscalibration (ECE
up to 0.083). Qualitative error analysis identifies persistent linguistic challenges including sarcasm (28%),
missing context (24%), linguistic ambiguity (19%), and conflicting evidence (15%). Statistical significance
testing (McNemar, bootstrap, DeLong, paired t-test, Wilcoxon) confirms that observed performance
improvements are robust and not attributable to random variation. The findings establish that evaluating
confidence reliability alongside accuracy provides a more realistic assessment of model suitability for realworld
misinformation detection. This work offers evidence-based recommendations for deploying
trustworthy fake news detection systems and identifies critical directions for future research, including
knowledge enhanced reasoning, multimodal integration, and continual learning approaches.},
}