{
  "generated_at": "2026-09-07T17:55:01.555125+10:00",
  "result_status": "preliminary_ai_review_human_validation_pending",
  "calibration_only": false,
  "human_validated": false,
  "sampling_batch_id": "reforms-core-20260710-v1",
  "calibration_max_rank": 10,
  "screening": {
    "papers_screened": 456,
    "calibration_candidates": 60,
    "other_candidates": 396,
    "outcomes": [
      {
        "outcome": "eligible_full_ml_science",
        "papers": 133,
        "share": 0.2916666666666667
      },
      {
        "outcome": "not_ml_based_science",
        "papers": 317,
        "share": 0.6951754385964912
      },
      {
        "outcome": "ml_science_incomplete_body",
        "papers": 4,
        "share": 0.008771929824561403
      },
      {
        "outcome": "unresolved",
        "papers": 2,
        "share": 0.0043859649122807015
      }
    ],
    "classes": [
      {
        "class": "ml_based_science",
        "papers": 137,
        "share": 0.30043859649122806
      },
      {
        "class": "rhetorical_ml_only",
        "papers": 103,
        "share": 0.22587719298245615
      },
      {
        "class": "ml_methods_research",
        "papers": 86,
        "share": 0.18859649122807018
      },
      {
        "class": "predictive_analytics",
        "papers": 83,
        "share": 0.18201754385964913
      },
      {
        "class": "non_ml_quantitative",
        "papers": 45,
        "share": 0.09868421052631579
      },
      {
        "class": "uncertain",
        "papers": 2,
        "share": 0.0043859649122807015
      }
    ],
    "publication_forms": [
      {
        "publication_form": "original_research",
        "papers": 345,
        "share": 0.756578947368421
      },
      {
        "publication_form": "review",
        "papers": 47,
        "share": 0.10307017543859649
      },
      {
        "publication_form": "editorial_opinion",
        "papers": 17,
        "share": 0.03728070175438596
      },
      {
        "publication_form": "systematic_review",
        "papers": 15,
        "share": 0.03289473684210526
      },
      {
        "publication_form": "other",
        "papers": 10,
        "share": 0.021929824561403508
      },
      {
        "publication_form": "conference_abstract",
        "papers": 9,
        "share": 0.019736842105263157
      },
      {
        "publication_form": "commentary_response",
        "papers": 5,
        "share": 0.010964912280701754
      },
      {
        "publication_form": "bibliometric_review",
        "papers": 4,
        "share": 0.008771929824561403
      },
      {
        "publication_form": "meta_analysis",
        "papers": 2,
        "share": 0.0043859649122807015
      },
      {
        "publication_form": "protocol",
        "papers": 1,
        "share": 0.0021929824561403508
      },
      {
        "publication_form": "uncertain",
        "papers": 1,
        "share": 0.0021929824561403508
      }
    ]
  },
  "overall": {
    "scope": "overall",
    "papers": 133,
    "items": 4256,
    "present": 2153,
    "partial": 1028,
    "absent": 966,
    "unclear": 3,
    "not_applicable": 106,
    "applicable_items": 4150,
    "present_rate": 0.5187951807228915,
    "partial_credit_rate": 0.6426506024096386,
    "present_or_partial_rate": 0.7665060240963856
  },
  "periods": [
    {
      "scope": "period:pre_reforms",
      "papers": 63,
      "items": 2016,
      "present": 1022,
      "partial": 493,
      "absent": 448,
      "unclear": 2,
      "not_applicable": 51,
      "applicable_items": 1965,
      "present_rate": 0.5201017811704834,
      "partial_credit_rate": 0.6455470737913486,
      "present_or_partial_rate": 0.7709923664122137
    },
    {
      "scope": "period:post_reforms",
      "papers": 70,
      "items": 2240,
      "present": 1131,
      "partial": 535,
      "absent": 518,
      "unclear": 1,
      "not_applicable": 55,
      "applicable_items": 2185,
      "present_rate": 0.5176201372997712,
      "partial_credit_rate": 0.640045766590389,
      "present_or_partial_rate": 0.7624713958810069
    }
  ],
  "paper_distributions": [
    {
      "period": "pre_reforms",
      "papers": 63,
      "min_present": 6,
      "max_present": 30,
      "zero_present": 0,
      "zero_present_or_partial": 0,
      "mean_full_rate": 0.5195454847369674,
      "median_full_rate": 0.53125,
      "min_full_rate": 0.1935483870967742,
      "max_full_rate": 0.9375,
      "mean_partial_credit_rate": 0.6452531737437144
    },
    {
      "period": "post_reforms",
      "papers": 70,
      "min_present": 2,
      "max_present": 28,
      "zero_present": 0,
      "zero_present_or_partial": 0,
      "mean_full_rate": 0.5167533619217383,
      "median_full_rate": 0.5322916666666666,
      "min_full_rate": 0.0625,
      "max_full_rate": 0.9032258064516129,
      "mean_partial_credit_rate": 0.6393184637108701
    }
  ],
  "modules": [
    {
      "scope": "module:computational_reproducibility",
      "papers": 133,
      "items": 665,
      "present": 82,
      "partial": 195,
      "absent": 387,
      "unclear": 0,
      "not_applicable": 1,
      "applicable_items": 664,
      "present_rate": 0.12349397590361445,
      "partial_credit_rate": 0.2703313253012048,
      "present_or_partial_rate": 0.4171686746987952
    },
    {
      "scope": "module:data_leakage",
      "papers": 133,
      "items": 399,
      "present": 129,
      "partial": 113,
      "absent": 131,
      "unclear": 3,
      "not_applicable": 23,
      "applicable_items": 376,
      "present_rate": 0.34308510638297873,
      "partial_credit_rate": 0.4933510638297872,
      "present_or_partial_rate": 0.6436170212765957
    },
    {
      "scope": "module:metrics_uncertainty",
      "papers": 133,
      "items": 399,
      "present": 143,
      "partial": 116,
      "absent": 86,
      "unclear": 0,
      "not_applicable": 54,
      "applicable_items": 345,
      "present_rate": 0.4144927536231884,
      "partial_credit_rate": 0.5826086956521739,
      "present_or_partial_rate": 0.7507246376811594
    },
    {
      "scope": "module:generalizability_limitations",
      "papers": 133,
      "items": 266,
      "present": 130,
      "partial": 66,
      "absent": 70,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 266,
      "present_rate": 0.48872180451127817,
      "partial_credit_rate": 0.6127819548872181,
      "present_or_partial_rate": 0.7368421052631579
    },
    {
      "scope": "module:data_preprocessing",
      "papers": 133,
      "items": 399,
      "present": 209,
      "partial": 81,
      "absent": 104,
      "unclear": 0,
      "not_applicable": 5,
      "applicable_items": 394,
      "present_rate": 0.5304568527918782,
      "partial_credit_rate": 0.633248730964467,
      "present_or_partial_rate": 0.7360406091370558
    },
    {
      "scope": "module:data_quality",
      "papers": 133,
      "items": 931,
      "present": 561,
      "partial": 258,
      "absent": 102,
      "unclear": 0,
      "not_applicable": 10,
      "applicable_items": 921,
      "present_rate": 0.6091205211726385,
      "partial_credit_rate": 0.749185667752443,
      "present_or_partial_rate": 0.8892508143322475
    },
    {
      "scope": "module:modeling",
      "papers": 133,
      "items": 798,
      "present": 516,
      "partial": 183,
      "absent": 86,
      "unclear": 0,
      "not_applicable": 13,
      "applicable_items": 785,
      "present_rate": 0.6573248407643312,
      "partial_credit_rate": 0.7738853503184714,
      "present_or_partial_rate": 0.8904458598726115
    },
    {
      "scope": "module:study_goals",
      "papers": 133,
      "items": 399,
      "present": 383,
      "partial": 16,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 399,
      "present_rate": 0.9598997493734336,
      "partial_credit_rate": 0.9799498746867168,
      "present_or_partial_rate": 1.0
    }
  ],
  "items": [
    {
      "reforms_item_id": "R2d",
      "module": "computational_reproducibility",
      "project_label": "README or equivalent run instructions supplied",
      "present": 1,
      "partial": 5,
      "absent": 127,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.007518796992481203,
      "partial_credit_rate": 0.02631578947368421,
      "present_or_partial_rate": 0.045112781954887216
    },
    {
      "reforms_item_id": "R2e",
      "module": "computational_reproducibility",
      "project_label": "Reproduction script or workflow supplied",
      "present": 12,
      "partial": 9,
      "absent": 112,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.09022556390977443,
      "partial_credit_rate": 0.12406015037593984,
      "present_or_partial_rate": 0.15789473684210525
    },
    {
      "reforms_item_id": "R2b",
      "module": "computational_reproducibility",
      "project_label": "Code and code version identified",
      "present": 5,
      "partial": 24,
      "absent": 104,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.03759398496240601,
      "partial_credit_rate": 0.12781954887218044,
      "present_or_partial_rate": 0.21804511278195488
    },
    {
      "reforms_item_id": "R3f",
      "module": "data_quality",
      "project_label": "Missing data frequency reported",
      "present": 15,
      "partial": 30,
      "absent": 81,
      "unclear": 0,
      "not_applicable": 7,
      "applicable_items": 126,
      "present_rate": 0.11904761904761904,
      "partial_credit_rate": 0.23809523809523808,
      "present_or_partial_rate": 0.35714285714285715
    },
    {
      "reforms_item_id": "R7b",
      "module": "metrics_uncertainty",
      "project_label": "Uncertainty estimates reported with calculation method",
      "present": 30,
      "partial": 24,
      "absent": 79,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.22556390977443608,
      "partial_credit_rate": 0.3157894736842105,
      "present_or_partial_rate": 0.40601503759398494
    },
    {
      "reforms_item_id": "R6b",
      "module": "data_leakage",
      "project_label": "Dependencies or duplicates across train/test partitions addressed",
      "present": 15,
      "partial": 28,
      "absent": 78,
      "unclear": 1,
      "not_applicable": 11,
      "applicable_items": 122,
      "present_rate": 0.12295081967213115,
      "partial_credit_rate": 0.23770491803278687,
      "present_or_partial_rate": 0.3524590163934426
    },
    {
      "reforms_item_id": "R8a",
      "module": "generalizability_limitations",
      "project_label": "External validity evidence reported",
      "present": 30,
      "partial": 40,
      "absent": 63,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.22556390977443608,
      "partial_credit_rate": 0.37593984962406013,
      "present_or_partial_rate": 0.5263157894736842
    },
    {
      "reforms_item_id": "R4b",
      "module": "data_preprocessing",
      "project_label": "Impossible or corrupt sample handling described",
      "present": 39,
      "partial": 31,
      "absent": 60,
      "unclear": 0,
      "not_applicable": 3,
      "applicable_items": 130,
      "present_rate": 0.3,
      "partial_credit_rate": 0.41923076923076924,
      "present_or_partial_rate": 0.5384615384615384
    },
    {
      "reforms_item_id": "R6a",
      "module": "data_leakage",
      "project_label": "Preprocessing and modeling restricted to training information where required",
      "present": 30,
      "partial": 40,
      "absent": 49,
      "unclear": 2,
      "not_applicable": 12,
      "applicable_items": 121,
      "present_rate": 0.24793388429752067,
      "partial_credit_rate": 0.4132231404958678,
      "present_or_partial_rate": 0.5785123966942148
    },
    {
      "reforms_item_id": "R4a",
      "module": "data_preprocessing",
      "project_label": "Sample exclusions identified and justified",
      "present": 55,
      "partial": 34,
      "absent": 42,
      "unclear": 0,
      "not_applicable": 2,
      "applicable_items": 131,
      "present_rate": 0.4198473282442748,
      "partial_credit_rate": 0.549618320610687,
      "present_or_partial_rate": 0.6793893129770993
    },
    {
      "reforms_item_id": "R5f",
      "module": "modeling",
      "project_label": "Baseline comparisons justified",
      "present": 69,
      "partial": 28,
      "absent": 36,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.518796992481203,
      "partial_credit_rate": 0.6240601503759399,
      "present_or_partial_rate": 0.7293233082706767
    },
    {
      "reforms_item_id": "R5e",
      "module": "modeling",
      "project_label": "Hyperparameter tuning reported",
      "present": 44,
      "partial": 55,
      "absent": 28,
      "unclear": 0,
      "not_applicable": 6,
      "applicable_items": 127,
      "present_rate": 0.3464566929133858,
      "partial_credit_rate": 0.562992125984252,
      "present_or_partial_rate": 0.7795275590551181
    },
    {
      "reforms_item_id": "R2c",
      "module": "computational_reproducibility",
      "project_label": "Computing environment described",
      "present": 27,
      "partial": 79,
      "absent": 27,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.20300751879699247,
      "partial_credit_rate": 0.5,
      "present_or_partial_rate": 0.7969924812030075
    },
    {
      "reforms_item_id": "R3g",
      "module": "data_quality",
      "project_label": "Sample-to-claim representativeness justified",
      "present": 8,
      "partial": 106,
      "absent": 18,
      "unclear": 0,
      "not_applicable": 1,
      "applicable_items": 132,
      "present_rate": 0.06060606060606061,
      "partial_credit_rate": 0.4621212121212121,
      "present_or_partial_rate": 0.8636363636363636
    },
    {
      "reforms_item_id": "R2a",
      "module": "computational_reproducibility",
      "project_label": "Training and evaluation dataset identified with stable access information",
      "present": 37,
      "partial": 78,
      "absent": 17,
      "unclear": 0,
      "not_applicable": 1,
      "applicable_items": 132,
      "present_rate": 0.2803030303030303,
      "partial_credit_rate": 0.5757575757575758,
      "present_or_partial_rate": 0.8712121212121212
    },
    {
      "reforms_item_id": "R5d",
      "module": "modeling",
      "project_label": "Model selection method reported",
      "present": 76,
      "partial": 33,
      "absent": 17,
      "unclear": 0,
      "not_applicable": 7,
      "applicable_items": 126,
      "present_rate": 0.6031746031746031,
      "partial_credit_rate": 0.7341269841269841,
      "present_or_partial_rate": 0.8650793650793651
    },
    {
      "reforms_item_id": "R8b",
      "module": "generalizability_limitations",
      "project_label": "Limits or non-applicability contexts stated",
      "present": 100,
      "partial": 26,
      "absent": 7,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.7518796992481203,
      "partial_credit_rate": 0.849624060150376,
      "present_or_partial_rate": 0.9473684210526315
    },
    {
      "reforms_item_id": "R6c",
      "module": "data_leakage",
      "project_label": "Inputs justified as legitimate and non-leaking",
      "present": 84,
      "partial": 45,
      "absent": 4,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.631578947368421,
      "partial_credit_rate": 0.8007518796992481,
      "present_or_partial_rate": 0.9699248120300752
    },
    {
      "reforms_item_id": "R7a",
      "module": "metrics_uncertainty",
      "project_label": "Performance metrics reported and justified",
      "present": 92,
      "partial": 37,
      "absent": 4,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.6917293233082706,
      "partial_credit_rate": 0.8308270676691729,
      "present_or_partial_rate": 0.9699248120300752
    },
    {
      "reforms_item_id": "R5c",
      "module": "modeling",
      "project_label": "Model evaluation method reported",
      "present": 110,
      "partial": 19,
      "absent": 4,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.8270676691729323,
      "partial_credit_rate": 0.8984962406015038,
      "present_or_partial_rate": 0.9699248120300752
    },
    {
      "reforms_item_id": "R7c",
      "module": "metrics_uncertainty",
      "project_label": "Statistical tests justified and assumptions checked where used",
      "present": 21,
      "partial": 55,
      "absent": 3,
      "unclear": 0,
      "not_applicable": 54,
      "applicable_items": 79,
      "present_rate": 0.26582278481012656,
      "partial_credit_rate": 0.6139240506329114,
      "present_or_partial_rate": 0.9620253164556962
    },
    {
      "reforms_item_id": "R3e",
      "module": "data_quality",
      "project_label": "Sample size and outcome frequencies reported",
      "present": 104,
      "partial": 26,
      "absent": 3,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.7819548872180451,
      "partial_credit_rate": 0.8796992481203008,
      "present_or_partial_rate": 0.9774436090225563
    },
    {
      "reforms_item_id": "R4c",
      "module": "data_preprocessing",
      "project_label": "Raw-to-model data transformations described",
      "present": 115,
      "partial": 16,
      "absent": 2,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.8646616541353384,
      "partial_credit_rate": 0.924812030075188,
      "present_or_partial_rate": 0.9849624060150376
    },
    {
      "reforms_item_id": "R5b",
      "module": "modeling",
      "project_label": "Choice of model types justified",
      "present": 92,
      "partial": 40,
      "absent": 1,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.6917293233082706,
      "partial_credit_rate": 0.8421052631578947,
      "present_or_partial_rate": 0.9924812030075187
    },
    {
      "reforms_item_id": "R3b",
      "module": "data_quality",
      "project_label": "Sampling frame or source distribution reported",
      "present": 88,
      "partial": 44,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 1,
      "applicable_items": 132,
      "present_rate": 0.6666666666666666,
      "partial_credit_rate": 0.8333333333333334,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R3a",
      "module": "data_quality",
      "project_label": "Data sources, timing, annotation, and documentation reported",
      "present": 108,
      "partial": 25,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.8120300751879699,
      "partial_credit_rate": 0.9060150375939849,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R3d",
      "module": "data_quality",
      "project_label": "Outcome variable defined with descriptive statistics",
      "present": 112,
      "partial": 20,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 1,
      "applicable_items": 132,
      "present_rate": 0.8484848484848485,
      "partial_credit_rate": 0.9242424242424242,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R1b",
      "module": "study_goals",
      "project_label": "Population or distribution choice motivated",
      "present": 124,
      "partial": 9,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.9323308270676691,
      "partial_credit_rate": 0.9661654135338346,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R5a",
      "module": "modeling",
      "project_label": "Models trained described",
      "present": 125,
      "partial": 8,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.9398496240601504,
      "partial_credit_rate": 0.9699248120300752,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R3c",
      "module": "data_quality",
      "project_label": "Dataset usefulness for the modeling task justified",
      "present": 126,
      "partial": 7,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.9473684210526315,
      "partial_credit_rate": 0.9736842105263158,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R1c",
      "module": "study_goals",
      "project_label": "Use of ML methods motivated",
      "present": 129,
      "partial": 4,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.9699248120300752,
      "partial_credit_rate": 0.9849624060150376,
      "present_or_partial_rate": 1.0
    },
    {
      "reforms_item_id": "R1a",
      "module": "study_goals",
      "project_label": "Scientific claim population or distribution stated",
      "present": 130,
      "partial": 3,
      "absent": 0,
      "unclear": 0,
      "not_applicable": 0,
      "applicable_items": 133,
      "present_rate": 0.9774436090225563,
      "partial_credit_rate": 0.9887218045112782,
      "present_or_partial_rate": 1.0
    }
  ],
  "adoption_signals": {
    "papers": 133,
    "citing_papers": 0,
    "checklist_papers": 0,
    "human_validation_checks": 0,
    "validation_accuracy_estimates": 0
  },
  "ai_run_ids": [
    "codex-default-20260713-reforms-calibration-01",
    "daily-paper-retrieval-reforms-20260728T105347AEST",
    "daily-paper-retrieval-reforms-20260729T105209AEST",
    "daily-paper-retrieval-reforms-20260730T105816AEST",
    "daily-reforms-20260801T005436Z-v1",
    "daily-reforms-20260802T005603Z-v1",
    "daily-reforms-20260804T110000+1000",
    "daily-reforms-20260804T112000+1000",
    "daily-reforms-20260805T110500+1000",
    "daily-reforms-20260806T105501+1000",
    "daily-reforms-20260813T110405+1000",
    "daily-reforms-20260814T110546+1000",
    "daily-reforms-20260815T110254+1000",
    "daily-reforms-20260816T105650+1000",
    "daily-reforms-20260817T111500+1000",
    "daily-reforms-20260818T110500+1000",
    "daily-reforms-20260819T105306+1000",
    "daily-reforms-20260820T110500+1000",
    "daily-reforms-20260821T105220+1000",
    "daily-reforms-20260822T105129+1000",
    "daily-reforms-20260823T113155+1000",
    "daily-reforms-20260824T110108+1000",
    "daily-reforms-20260825T180500+1000",
    "daily-reforms-20260826T105919+1000",
    "daily-reforms-20260827T105631+1000",
    "daily-reforms-20260828T105402+1000",
    "daily-reforms-20260829T105835+1000",
    "daily-reforms-20260830T105000+1000",
    "daily-reforms-20260831T110546+1000",
    "daily-reforms-20260901T110645+1000",
    "daily-reforms-20260902T194500+1000",
    "daily-reforms-20260903T105904+1000"
  ]
}