{
  "schemaVersion": "1.6",
  "slug": "quantile-threshold-calibration-separate",
  "title": "Quantile and threshold calibration separate even for Gaussian forecasts",
  "shortTitle": "Quantile and threshold calibration checks can disagree",
  "url": "https://evidencepress.org/releases/quantile-threshold-calibration-separate/",
  "oneLine": "Passing every quantile calibration check does not mean passing every threshold check, even for smooth Gaussian forecasts: the two checks weight the same probability-zero set of forecasts differently, and errors reach about one percentage point.",
  "abstract": "A probabilistic forecast is calibrated when its stated probabilities match what happens. Two widely used checks condition on different summaries of the forecast: on a predicted quantile, or on the predicted probability of staying below a fixed threshold. Gneiting and Resin (2023) proved the two checks equivalent for continuous, strictly increasing forecast distributions that take only countably many values, and conjectured that countability could be dropped; Resin, Yang and Gneiting (2026) still list these implications as conjectures. This candidate shows that countability cannot be dropped. Gaussian forecasts with uniformly distributed location and scale, paired with explicit smooth outcome laws whose densities stay within a factor of two of the forecast densities, separate the checks in both directions: one law passes every conditional quantile check and has an exactly uniform PIT, yet fails threshold and marginal calibration at every threshold; the other passes every threshold check but fails quantile calibration, with a PIT distribution function strictly below the uniform one at every interior level. The errors reach about one percentage point. For outcomes with finitely many ordered values and positive forecast probabilities, conditional exceedance calibration implies threshold calibration when the forecast has countable support or there are at most three values, but not with four or more. For nominal outcomes, full probabilistic calibration implies marginal calibration with a sharp bound, and class-wise calibration does not imply average probabilistic calibration; two further nominal implications are settled for at most three classes or countable support and remain open beyond. The mechanism is the Borel–Kolmogorov phenomenon. Limits: population constructions, not a study of deployed forecasts; the amplitude rests on a computer-assisted interval bound; no finite-sample test theory. AI-assisted, unrefereed candidate; no external peer review, independent reproduction or formal verification is claimed.",
  "datePublished": "2026-11-15",
  "dateModified": "2026-11-15",
  "version": "0.1.0-candidate",
  "doi": "10.5281/zenodo.24259222",
  "doiUrl": "https://doi.org/10.5281/zenodo.24259222",
  "conceptDoi": "10.5281/zenodo.24259221",
  "pdfUrl": "https://github.com/ipitchford/quantile-threshold-calibration-separate/releases/download/v0.1.0-candidate/quantile-threshold-calibration-separate-0.1.0-candidate.pdf",
  "altPdfUrl": null,
  "altPdfLabel": null,
  "zenodoUrl": "https://zenodo.org/records/24259222",
  "repoUrl": "https://github.com/ipitchford/quantile-threshold-calibration-separate",
  "releaseUrl": "https://github.com/ipitchford/quantile-threshold-calibration-separate/releases/tag/v0.1.0-candidate",
  "markdownUrl": "https://evidencepress.org/releases/quantile-threshold-calibration-separate/index.md",
  "bibtexUrl": "https://evidencepress.org/releases/quantile-threshold-calibration-separate/cite.bib",
  "audioUrl": "https://evidencepress.org/assets/audio/quantile-threshold-calibration-separate.mp3",
  "imageUrl": "https://evidencepress.org/assets/og/quantile-threshold-calibration-separate.png?v=07c99680ea",
  "coverArtUrl": "https://evidencepress.org/assets/art/quantile-threshold-calibration-separate.svg?v=da12a9c960",
  "media": [
    {
      "type": "audio",
      "url": "https://evidencepress.org/assets/audio/quantile-threshold-calibration-separate.mp3",
      "name": "Plain-English audio briefing",
      "description": "Communication aid; not additional research evidence.",
      "transcriptUrl": "https://evidencepress.org/assets/audio/quantile-threshold-calibration-separate.txt"
    }
  ],
  "authors": [
    "Anonymous"
  ],
  "license": "CC0-1.0",
  "status": "unrefereed-candidate",
  "verification": {
    "peerReviewed": false,
    "independentlyReproduced": false,
    "formallyVerified": false,
    "internallyReplayed": true,
    "detail": "Unrefereed AI-assisted mathematics candidate with written proofs and one computer-assisted interval certificate. Internal AI-assisted review in five reviewer roles and an adversarial review by a second AI model, clean replays, one revision round with extensions (the certified amplitude and effect sizes, the boundary for finitely many ordered outcome values, the nominal results merged from a companion note, the status table), then a re-review, a proof audit and a second-model audit of the new proofs; no mathematical error was found, and summary wording was corrected. Except for the amplitude certificate the scripts are checks, not proofs. External specialist review, independent reproduction and formal verification have not been established."
  },
  "assurance": [
    {
      "dimension": "availability",
      "label": "Availability and archiving",
      "question": "Is the evidence package publicly retrievable from an archive under a persistent identifier?",
      "state": "passed",
      "evidenceUrl": "https://zenodo.org/records/24259222",
      "note": "The versioned research package and paper are publicly archived on GitHub and Zenodo. Provider asset parity was checked against frozen local files; availability does not establish correctness or peer review."
    },
    {
      "dimension": "internalReplay",
      "label": "Internal replay",
      "question": "Does the producer’s own pipeline reproduce the stated result from the archived package?",
      "state": "passed",
      "note": "All three scripts were rerun from fresh copies in normal mode and under python -O with pinned packages; every recorded output matched byte for byte. Lowering the certified bound to 4.7 and swapping two table entries of the four-value construction each make the checks fail under -O. GitHub CI reruns the exact discrete checks, the certificate and both negative controls (blocking) and the Gaussian checks (non-blocking) on Linux."
    },
    {
      "dimension": "independentRerun",
      "label": "Independent rerun",
      "question": "Has someone else run the supplied implementation and obtained the stated result?",
      "state": "not-assessed",
      "note": "No unaffiliated rerun has been established."
    },
    {
      "dimension": "independentReimplementation",
      "label": "Independent reimplementation",
      "question": "Has someone else reached the result from an independent implementation?",
      "state": "not-assessed",
      "note": "No unaffiliated reimplementation; the internal proof audit recomputed the numerical claims, and the release banner generator recomputes the two averages 0 and 1/9, c₀ and the sample size from the paper's formulas."
    },
    {
      "dimension": "formalVerification",
      "label": "Formal verification",
      "question": "Is a formalised statement machine-checked, and over which trusted base?",
      "state": "not-assessed",
      "note": "No proof-assistant formalisation; the amplitude certificate uses outward-rounded interval arithmetic."
    },
    {
      "dimension": "specialistReview",
      "label": "Specialist review",
      "question": "Has a domain specialist assessed the argument?",
      "state": "not-assessed",
      "note": "AI-assisted internal and adversarial review is not external specialist review."
    },
    {
      "dimension": "editorialPeerReview",
      "label": "Editorial peer review",
      "question": "Has a journal or venue run peer review to a decision?",
      "state": "not-assessed",
      "note": "No external editorial peer review."
    },
    {
      "dimension": "dataEnvironmentReproducibility",
      "label": "Data and environment reproducibility",
      "question": "Are data and computational environment pinned well enough to rebuild?",
      "state": "passed",
      "note": "Python 3.12 with pinned numpy 2.3.5, scipy 1.17.0, sympy 1.14.0 and mpmath 1.3.0; outputs reproduced byte for byte on the release platform and on Linux CI (the Gaussian checks within a relative tolerance of 1e-9). Not a locked container."
    }
  ],
  "provenance": {
    "aiGenerated": true,
    "aiAssisted": true,
    "generatedBy": [
      "AI-assisted mathematical research",
      "AI-assisted internal review (five reviewer roles), adversarial review and audit by a second AI model, re-review, proof audit, clean replays",
      "Claude Code: publication packaging and release"
    ],
    "humanRole": "Publication authority only; no human research contribution inferred.",
    "reviewStatus": "Internal AI-assisted and adversarial review, not external peer review."
  },
  "attributionBoundary": null,
  "problem": null,
  "orderedProductAverage": null,
  "corrections": [],
  "publicCorrections": [],
  "publicCorrectionIndexes": null,
  "pageStructureVersion": "2.0",
  "pageStructureVariant": "canonical-theorem",
  "pageStructureWaivers": [],
  "recordMaturity": "current",
  "metadataProvenance": "Prepared from the final paper (revision of 15 November 2026 with the DOI added to the title-page version line, 20 pages), its scripts and recorded outputs and the frozen release archive. The versioned GitHub and Zenodo research archives were checked against the frozen release assets on 15 November 2026.",
  "grandfatheredAtSchemaVersion": null,
  "keywords": [
    "probabilistic forecasting",
    "calibration",
    "quantile calibration",
    "threshold calibration",
    "probability integral transform",
    "reliability diagram",
    "Borel-Kolmogorov paradox",
    "forecast verification"
  ],
  "keyResults": [
    "Theorem 3.1 (proof): Gaussian forecasts N(M, S²) with M uniform on [−1, 1] and S uniform on [1, 2], with explicit smooth outcome laws (densities between half and one and a half times the forecast density, perturbation size ε ≤ 1/10): under one law the forecast passes every conditional quantile check with an exactly uniform PIT but fails threshold and marginal calibration at every threshold, P(Y ≤ x) − E F(x) = ε c₀ e^{−x²} with c₀ = 1 − (4/3) log 2; under the other it is threshold calibrated but fails quantile calibration, with PIT CDF strictly below the identity.",
    "Section 3.5 (computation): at ε = 1/10 outcomes fall below 0 with probability 0.50758 while forecasts average 1/2; on a common segment the threshold error is ε/9 ≈ 0.011; a two-sided binomial test at the 5% level with 80% power needs about 34,000 cases to detect the marginal error.",
    "Theorems 4.1–4.3 (proof): for k ordered outcome values with positive forecast probabilities, conditional exceedance calibration implies threshold calibration if the forecast has countable support or k ≤ 3; an explicit four-value forecast with one continuous parameter satisfies conditional exceedance, probabilistic and marginal calibration but not threshold calibration.",
    "Theorems 5.3 and 5.5, Proposition 5.6 (proof): for nominal outcomes, full probabilistic calibration implies marginal calibration, with the sharp bound TV ≤ 2d − d²; class-wise calibration does not imply average probabilistic calibration, with all probabilities positive; fCC ⇒ fTC and pCC ⇒ pTC hold for k ≤ 3 or countable support and remain open for four or more classes.",
    "Theorems 6.1 and 7.1 (proof): in location–scale families every compactly supported perturbation preserving either property is described by one differential operator; in any smooth family the two conditional laws on a common fibre differ by the forecast-density factor, so they agree for every outcome law only if that density is constant on the fibre."
  ],
  "reviews": [],
  "evidencePackage": "A 20-page paper with a plain introduction, written proofs, a status table of every conjectured implication and a limits paragraph; three Python scripts with recorded outputs: symbolic and numerical checks of the Gaussian examples and their effect sizes; exact rational checks of the discrete and nominal results; and an interval-arithmetic certificate of the amplitude bound, the one computer-assisted step of a proof. Clean replay from fresh copies in normal and python -O mode reproduced every recorded output byte for byte, and a lowered certificate bound or two swapped table entries make the checks fail.",
  "openProblems": [],
  "mathObjects": [],
  "relatedWorks": [
    {
      "citation": "Gneiting and Resin (2023), Regression diagnostics meets forecast evaluation: conditional calibration, reliability diagrams, and coefficient of determination. Electronic Journal of Statistics. Equivalence of quantile and threshold calibration for forecasts with countably many values (Theorem 2.16) and the conjecture, refuted here, that countability can be dropped; Appendix B.4 on testing.",
      "url": "https://doi.org/10.1214/23-ejs2180",
      "doi": "10.1214/23-ejs2180"
    },
    {
      "citation": "Resin, Yang and Gneiting (2026), Hierarchies of calibration: Classification meets regression. arXiv:2606.03245, version 1. The calibration hierarchies whose conjectured implications (Figures 2(a) and 4) this paper settles or partly settles.",
      "url": "https://arxiv.org/abs/2606.03245v1",
      "doi": "10.48550/arXiv.2606.03245"
    },
    {
      "citation": "Gneiting, Balabdaoui and Raftery (2007), Probabilistic Forecasts, Calibration and Sharpness. Journal of the Royal Statistical Society Series B: Statistical Methodology. Probabilistic and marginal calibration.",
      "url": "https://doi.org/10.1111/j.1467-9868.2007.00587.x",
      "doi": "10.1111/j.1467-9868.2007.00587.x"
    },
    {
      "citation": "Henzi, Ziegel and Gneiting (2021), Isotonic Distributional Regression. Journal of the Royal Statistical Society Series B: Statistical Methodology. Threshold calibration.",
      "url": "https://doi.org/10.1111/rssb.12450",
      "doi": "10.1111/rssb.12450"
    },
    {
      "citation": "Mason, Galpin, Goddard, Graham and Rajartnam (2007), Conditional Exceedance Probabilities. Monthly Weather Review. Conditional exceedance probability checks in forecast verification.",
      "url": "https://doi.org/10.1175/mwr3284.1",
      "doi": "10.1175/mwr3284.1"
    },
    {
      "citation": "Held, Rufibach and Balabdaoui (2010), A Score Regression Approach to Assess Calibration of Continuous Probabilistic Predictions. Biometrics. A test of exceedance-type calibration.",
      "url": "https://doi.org/10.1111/j.1541-0420.2010.01406.x",
      "doi": "10.1111/j.1541-0420.2010.01406.x"
    },
    {
      "citation": "Strähl and Ziegel (2017), Cross-calibration of probabilistic forecasts. Electronic Journal of Statistics. Conditional exceedance calibration tests.",
      "url": "https://doi.org/10.1214/17-ejs1244",
      "doi": "10.1214/17-ejs1244"
    },
    {
      "citation": "Dawid (1984), Present Position and Potential Developments: Some Personal Views: Statistical Theory: The Prequential Approach. Journal of the Royal Statistical Society. Series A (General). The probability integral transform as a calibration diagnostic.",
      "url": "https://doi.org/10.2307/2981683",
      "doi": "10.2307/2981683"
    },
    {
      "citation": "Diebold, Gunther and Tay (1998), Evaluating Density Forecasts with Applications to Financial Risk Management. International Economic Review. PIT histograms in practice.",
      "url": "https://doi.org/10.2307/2527342",
      "doi": "10.2307/2527342"
    },
    {
      "citation": "Arnold and Ziegel (2025), Isotonic conditional laws. Bernoulli. Isotonic calibration, one way to guarantee both properties.",
      "url": "https://doi.org/10.3150/24-bej1764",
      "doi": "10.3150/24-bej1764"
    },
    {
      "citation": "Allen, Gavrilopoulos, Henzi, Kleger and Ziegel (2025), In-sample calibration yields conformal calibration guarantees. arXiv:2503.03841, version 2. Statements on equivalence under continuity discussed in Section 8.",
      "url": "https://arxiv.org/abs/2503.03841v2",
      "doi": "10.48550/arXiv.2503.03841"
    },
    {
      "citation": "Silva Filho, Song, Perello-Nieto, Santos-Rodriguez, Kull and Flach (2023), Classifier calibration: a survey on how to assess and improve predicted class probabilities. Machine Learning. Source of the example (footnote 2) that Resin, Yang and Gneiting list for class-wise versus average probabilistic calibration.",
      "url": "https://doi.org/10.1007/s10994-023-06336-7",
      "doi": "10.1007/s10994-023-06336-7"
    },
    {
      "citation": "Kolmogoroff (1933), Grundbegriffe der Wahrscheinlichkeitsrechnung. Springer. The Borel–Kolmogorov phenomenon of conditioning on events of probability zero (Chapter V), the mechanism behind the separation.",
      "url": "https://doi.org/10.1007/978-3-642-49888-6",
      "doi": "10.1007/978-3-642-49888-6"
    },
    {
      "citation": "Diaconis, Holmes and Shahshahani (2013), Sampling from a Manifold. Institute of Mathematical Statistics Collections. The coarea formula used for the change-of-conditioning identity.",
      "url": "https://doi.org/10.1214/12-imscoll1006",
      "doi": "10.1214/12-imscoll1006"
    },
    {
      "citation": "Cohen and Fausti (2024), Hyperbolic contractivity and the Hilbert metric on probability measures. arXiv:2309.02413, version 2. The Hilbert-metric inequality behind the transfer bound (Proposition 7.2).",
      "url": "https://arxiv.org/abs/2309.02413v2",
      "doi": "10.48550/arXiv.2309.02413"
    }
  ],
  "claimAssurance": null,
  "operatingModel": {
    "version": "1.0",
    "workId": "ep-work:quantile-threshold-calibration-separate",
    "attemptIds": [
      "ep-attempt:quantile-threshold-calibration-separate-publication"
    ],
    "aims": [
      "science"
    ],
    "artifactRoles": [
      "research-output",
      "communication"
    ],
    "lineageId": null,
    "accelerationPrimitives": [
      "adversarial-controls",
      "assurance-vector",
      "agent-readable-research-object"
    ],
    "decisionObject": {
      "type": "other",
      "description": "Treat quantile-type (conditional exceedance) and threshold calibration checks as testing different hypotheses for continuously parameterised forecasts, and run both.",
      "scope": "Population properties; continuous forecast families; finitely many ordered values with the exact boundary; nominal implications fCC => fTC and pCC => pTC open for four or more classes."
    },
    "bottleneckTargeted": [
      "assurance",
      "publication"
    ],
    "semanticBridge": {
      "state": "explicit",
      "description": "Section 1 separates what was known (equivalence for countably many forecast values, Gneiting and Resin; the conjectured arrows of Gneiting and Resin and of Resin, Yang and Gneiting; the classical Borel–Kolmogorov phenomenon) from what is new (smooth Gaussian counterexamples in both directions, the discrete boundary, the nominal results and the structure theorems); Table 2 gives the status of every conjectured implication.",
      "remainingRisks": [
        "The bounded literature search is not exhaustive priority certification.",
        "Resin, Yang and Gneiting, whose conjectures are addressed, is a recent preprint."
      ]
    },
    "humanJudgmentGates": [
      "Publication is authorised; scientific endorsement is not inferred.",
      "External specialist review and independent reproduction remain open."
    ],
    "parentLinks": [],
    "assuranceTarget": {
      "dimensions": [
        "specialistReview",
        "independentRerun",
        "formalVerification",
        "priorityAssessment"
      ],
      "nextAction": "Have a specialist in forecast evaluation check Theorem 3.1 and the testing consequences in Section 8, and a probabilist check the discrete boundary (Section 4) and the interval certificate (Appendix B).",
      "claimCeiling": "Written proofs of the separation, the discrete boundary, the nominal results and the structure theorems; amplitude 1/10 via an interval certificate; population constructions only; internal verification is not external assurance."
    },
    "impactClaims": [
      {
        "id": "quantile-threshold-calibration-separate-no-impact",
        "aim": "science",
        "outcome": "An openly inspectable resolution of the Gneiting–Resin conjecture on quantile and threshold calibration, with the discrete boundary and the status of the nominal conjectures.",
        "setting": "AI-assisted mathematical statistics research",
        "status": "NO_IMPACT_EVIDENCE",
        "designClass": "none",
        "comparator": "None registered.",
        "estimand": "No research-speed or scientific-impact effect is estimated.",
        "evidenceRefs": [],
        "registeredDesignRef": null,
        "independentAssessment": null
      }
    ]
  },
  "researchMetrics": {
    "schemaVersion": "1.0",
    "policyUrl": "https://evidencepress.org/api/research-metrics-policy.json",
    "outcomePolicyUrl": "https://evidencepress.org/api/v2/research-metrics-policy.json",
    "workLedgerUrl": "https://evidencepress.org/api/work-ledger.json",
    "claimCeiling": "These records describe scoped workflow inputs and outcomes. They support calibration and reference-class construction, but do not by themselves establish research acceleration, theorem correctness, novelty, independent assurance or impact.",
    "attempts": [
      {
        "attemptId": "ep-attempt:quantile-threshold-calibration-separate-publication",
        "status": "active",
        "resultClass": "pending",
        "metrics": {
          "schemaVersion": "1.0",
          "measurementScope": "publication-only",
          "scopeBoundary": "Only archive packaging, site integration, guarded delivery and canonical readback from registration are prospective. Earlier research, review, revision and extension work are left-censored.",
          "forecast": {
            "frozenAt": "2026-11-15T19:54:36Z",
            "procedureClass": "full-candidate-publication",
            "targetOutcome": "Accurately scoped GitHub, Zenodo and canonical Evidence Press release with explanatory media and machine readback, if all publication gates pass.",
            "expectedActiveMinutes": 90,
            "plausibleLowMinutes": 60,
            "plausibleHighMinutes": 150,
            "expectedUnattendedWaitMinutes": 30,
            "referenceClass": {
              "label": "Full mathematical candidate publication",
              "sampleSize": 0,
              "basis": "Route benchmark planning prior, not empirical calibration."
            },
            "fermiComponents": [
              {
                "component": "Research archive packaging, replay and identities",
                "count": 1,
                "lowMinutesPerUnit": 25,
                "centralMinutesPerUnit": 35,
                "highMinutesPerUnit": 60,
                "basis": "One frozen package with a rating-free public export, clean replay and DOI insertion."
              },
              {
                "component": "Page, metadata and media integration",
                "count": 1,
                "lowMinutesPerUnit": 20,
                "centralMinutesPerUnit": 30,
                "highMinutesPerUnit": 45,
                "basis": "One reader-first release, drafted in parallel with verification."
              },
              {
                "component": "Guarded deployment and canonical readback",
                "count": 1,
                "lowMinutesPerUnit": 15,
                "centralMinutesPerUnit": 25,
                "highMinutesPerUnit": 45,
                "basis": "Mandatory site tests, one merge cycle with ledger carry-forward, zero-cost deployment and public checks."
              }
            ],
            "probabilityPositiveSignal": 0.9,
            "probabilityTargetClosure": 0.85,
            "probabilityHorizonMinutes": 300,
            "assumptions": [
              "The approved scientific package remains valid and configured providers remain available."
            ],
            "reforecastTriggers": [
              "A publication gate reveals a scientific or integrity defect, or a provider outage blocks delivery."
            ],
            "stopRule": "Do not weaken assurance, integrity or cost gates to meet the forecast."
          },
          "outcome": null
        },
        "missingnessReason": null
      }
    ]
  }
}