Probability Calibration Ontology

flow "ProbabilityCalibrationOntology" v4.0

# PROBABILITY CALIBRATION ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/probability-calibration-ontology.html"
  title: "Probability Calibration Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: PROBABILITY_CALIBRATION
  entity_type: STATISTICAL_MACHINE_LEARNING_ONTOLOGY

DEFINE:
  PROBABILITY_CALIBRATION :=
    AGREEMENT_STRUCTURE(
      PREDICTED_PROBABILITIES,
      OBSERVED_OUTCOME_FREQUENCIES,
      PREDICTIVE_MODEL,
      TARGET_DISTRIBUTION
    )

PURPOSE:
  QUANTIFY:
    RELIABILITY_OF_PREDICTED_PROBABILITIES
    AGREEMENT_BETWEEN_CONFIDENCE_AND_FREQUENCY
    PROBABILISTIC_PREDICTION_QUALITY

FOUNDATION:
  INPUT: X
  TARGET: Y
  PREDICTED_PROBABILITY:
    Q := q(Y=1|X)

  BINARY_CALIBRATION:
    P(Y=1 | Q=p) = p

  equivalent:
    E[Y|Q] = Q

ENTITIES:
  PROBABILITY_CALIBRATION: PREDICTIVE_RELIABILITY_PROPERTY
  PREDICTED_PROBABILITY: MODEL_PROBABILITY_OUTPUT
  OBSERVED_FREQUENCY: EMPIRICAL_OUTCOME_RATE
  CALIBRATION_CURVE: RELIABILITY_FUNCTION
  RELIABILITY_DIAGRAM: GRAPHICAL_CALIBRATION_SUMMARY
  CALIBRATION_ERROR: DISCREPANCY_MEASURE
  BRIER_SCORE: PROBABILISTIC_SCORING_RULE
  LOG_LOSS: LOGARITHMIC_SCORING_RULE
  PLATT_SCALING: PARAMETRIC_CALIBRATION_METHOD
  ISOTONIC_REGRESSION: NONPARAMETRIC_CALIBRATION_METHOD
  TEMPERATURE_SCALING: LOGIT_CALIBRATION_METHOD
  CALIBRATION_SET: DATA_USED_TO_FIT_CALIBRATOR

RELATIONSHIPS:
  MODEL -> OUTPUTS -> PREDICTED_PROBABILITY
  PREDICTED_PROBABILITY -> COMPARED_WITH -> OBSERVED_FREQUENCY
  COMPARISON -> DEFINES -> CALIBRATION
  CALIBRATION_CURVE -> SUMMARISES -> RELIABILITY
  CALIBRATION_METHOD -> TRANSFORMS -> MODEL_PROBABILITIES_OR_SCORES
  CALIBRATION_SET -> FITS -> CALIBRATOR
  DISTRIBUTION_SHIFT -> MAY_CHANGE -> CALIBRATION

BINARY_CALIBRATION:
  predictor:
    Q(X) IN [0,1]

  ideal_condition:
    E[Y|Q] = Q

  interpretation:
    PREDICTED_PROBABILITY_MATCHES_LONG_RUN_OUTCOME_FREQUENCY

MULTICLASS_CALIBRATION:
  predicted_vector:
    q(X) := (q_1,...,q_K)

  requirements:
    q_k >= 0
    SUM_k q_k = 1

  notions:
    - CLASSWISE_CALIBRATION
    - TOP_LABEL_CALIBRATION
    - CONFIDENCE_CALIBRATION
    - STRONG_MULTICLASS_CALIBRATION

  caution:
    MULTIPLE_NON_EQUIVALENT_CALIBRATION_DEFINITIONS_EXIST

PERFECT_CALIBRATION:
  condition:
    PREDICTED_PROBABILITIES_MATCH_CONDITIONAL_OUTCOME_FREQUENCIES

  caution:
    PERFECT_CALIBRATION_DOES_NOT_IMPLY:
      PERFECT_DISCRIMINATION
      PERFECT_ACCURACY

MISCALIBRATION:
  OVERCONFIDENCE:
    PREDICTED_CONFIDENCE_EXCEEDS_OBSERVED_FREQUENCY

  UNDERCONFIDENCE:
    PREDICTED_CONFIDENCE_BELOW_OBSERVED_FREQUENCY

RELIABILITY_DIAGRAM:
  flow:
    PARTITION_PREDICTIONS_INTO_BINS
      -> COMPUTE_MEAN_PREDICTED_PROBABILITY
      -> COMPUTE_EMPIRICAL_OUTCOME_FREQUENCY
      -> PLOT_FREQUENCY_VS_PROBABILITY

  ideal:
    POINTS_NEAR_DIAGONAL

CALIBRATION_CURVE:
  mapping:
    predicted_probability
      -> observed_frequency

  ideal:
    c(p) := p

BINNING:
  methods:
    - EQUAL_WIDTH
    - EQUAL_FREQUENCY
    - ADAPTIVE

  caution:
    BIN_CHOICE_AFFECTS_CALIBRATION_ESTIMATE

EXPECTED_CALIBRATION_ERROR:
  alias:
    ECE

  common_form:
    ECE :=
      SUM[m=1..M]
        (|B_m|/n)
        *
        |acc(B_m)-conf(B_m)|

  caution:
    ECE_DEPENDS_ON_BINNING
    ECE_IS_NOT_A_UNIQUE_UNIVERSAL_CALIBRATION_METRIC

MAXIMUM_CALIBRATION_ERROR:
  alias:
    MCE

  common_form:
    MCE :=
      max_m |acc(B_m)-conf(B_m)|

BRIER_SCORE:
  binary:
    BS :=
      (1/n)
      * SUM[i=1..n]
        (q_i-y_i)^2

  population:
    E[(Q-Y)^2]

  property:
    PROPER_SCORING_RULE

BRIER_DECOMPOSITION:
  components:
    - RELIABILITY
    - RESOLUTION
    - UNCERTAINTY

LOG_LOSS:
  binary:
    L :=
      -[
        y log q
        + (1-y) log(1-q)
      ]

  property:
    STRICTLY_PROPER_SCORING_RULE

CALIBRATION_VS_DISCRIMINATION:
  CALIBRATION:
    AGREEMENT_OF_PROBABILITY_WITH_FREQUENCY

  DISCRIMINATION:
    ABILITY_TO_RANK_OR_SEPARATE_OUTCOMES

  rule:
    CALIBRATION != DISCRIMINATION

CALIBRATION_VS_ACCURACY:
  ACCURACY:
    THRESHOLD_DEPENDENT_CLASSIFICATION_CORRECTNESS

  CALIBRATION:
    PROBABILITY_RELIABILITY

  rule:
    ACCURACY != CALIBRATION

BASE_RATE:
  prevalence:
    pi := P(Y=1)

  constant_predictor:
    q(x) := pi

  property:
    MAY_BE_CALIBRATED_BUT_HAVE_LOW_DISCRIMINATION

PLATT_SCALING:
  transformation:
    q_cal := sigmoid(a*s+b)

  fitting:
    ESTIMATE_a_AND_b_ON_CALIBRATION_DATA

  role:
    PARAMETRIC_POST_HOC_CALIBRATION

ISOTONIC_REGRESSION:
  mapping:
    MONOTONIC_NONDECREASING_FUNCTION

  role:
    NONPARAMETRIC_CALIBRATION

  caution:
    MAY_OVERFIT_WITH_SMALL_CALIBRATION_SET

TEMPERATURE_SCALING:
  logits:
    z_k

  temperature:
    T > 0

  calibrated_probability:
    q_k :=
      exp(z_k/T)
      /
      SUM_j exp(z_j/T)

  fitting:
    SELECT_T_TO_MINIMISE_NEGATIVE_LOG_LIKELIHOOD
    ON:
      CALIBRATION_SET

  property:
    PRESERVES_ARGMAX_CLASS
    FOR:
      POSITIVE_SCALAR_T

VECTOR_SCALING:
  relation:
    GENERALISES_TEMPERATURE_SCALING

MATRIX_SCALING:
  transformation:
    AFFINE_TRANSFORMATION_OF_LOGIT_VECTOR

  caution:
    MORE_PARAMETERS_INCREASE_OVERFITTING_RISK

BETA_CALIBRATION:
  role:
    PARAMETRIC_CALIBRATION_METHOD_FOR_PROBABILITIES

CALIBRATION_DATA:
  requirement:
    SEPARATE_FROM_BASE_MODEL_TRAINING_DATA
    OR:
      GENERATED_WITH_VALID_CROSS_VALIDATION_PROTOCOL

  purpose:
    AVOID_CALIBRATION_LEAKAGE_AND_OVERFITTING

CROSS_VALIDATED_CALIBRATION:
  flow:
    SPLIT_TRAINING_DATA
      -> FIT_BASE_MODEL_ON_TRAIN_FOLDS
      -> GENERATE_OUT_OF_FOLD_PREDICTIONS
      -> FIT_CALIBRATOR
      -> EVALUATE_ON_HELD_OUT_DATA

UNCERTAINTY_IN_CALIBRATION:
  methods:
    - BOOTSTRAP
    - BINOMIAL_INTERVALS
    - BAYESIAN_CALIBRATION_MODELS

CLASS_IMBALANCE:
  prevalence:
    MAY_AFFECT:
      CALIBRATION
      BRIER_SCORE
      RELIABILITY_DIAGRAM

DISTRIBUTION_SHIFT:
  IF:
    P_train(X,Y) != P_deploy(X,Y)

  THEN:
    CALIBRATION_MAY_DEGRADE

  forms:
    - COVARIATE_SHIFT
    - LABEL_SHIFT
    - CONCEPT_SHIFT

LABEL_SHIFT:
  condition:
    P_deploy(Y) != P_train(Y)

  consequence:
    CLASS_PRIORS_CHANGE

  response:
    MAY_REQUIRE_PRIOR_ADJUSTMENT_OR_RECALIBRATION

CONCEPT_SHIFT:
  condition:
    P_deploy(Y|X) != P_train(Y|X)

  consequence:
    CALIBRATION_MAPPING_MAY_NO_LONGER_BE_VALID

CALIBRATION_MONITORING:
  flow:
    COLLECT_PREDICTIONS
      -> COLLECT_OUTCOMES
      -> COMPUTE_CALIBRATION_METRICS
      -> COMPARE_WITH_REFERENCE
      -> INVESTIGATE_SHIFT
      -> RECALIBRATE_OR_RETRAIN_IF_REQUIRED

TIME_DEPENDENT_CALIBRATION:
  issue:
    CALIBRATION_MAY_CHANGE_OVER_TIME

  monitoring:
    - ROLLING_WINDOWS
    - TIME_STRATIFICATION
    - TEMPORAL_VALIDATION

GROUP_CALIBRATION:
  objective:
    ASSESS_CALIBRATION_WITHIN_RELEVANT_SUBGROUPS

  caution:
    OVERALL_CALIBRATION_DOES_NOT_GUARANTEE_SUBGROUP_CALIBRATION

CALIBRATION_IN_REGRESSION:
  predictive_distribution:
    F_hat(y|x)

  concepts:
    - PREDICTION_INTERVAL_COVERAGE
    - QUANTILE_CALIBRATION
    - DISTRIBUTIONAL_CALIBRATION
    - PIT_UNIFORMITY

PREDICTION_INTERVAL_CALIBRATION:
  nominal_coverage:
    1-alpha

  ideal:
    P(Y IN C_(1-alpha)(X))
      =
    1-alpha

  caution:
    MARGINAL_COVERAGE != CONDITIONAL_COVERAGE

QUANTILE_CALIBRATION:
  predicted_quantile:
    q_tau(X)

  ideal:
    P(Y <= q_tau(X)) = tau

PROBABILITY_INTEGRAL_TRANSFORM:
  alias:
    PIT

  value:
    U := F_hat(Y)

  ideal:
    U ~ Uniform(0,1)
    UNDER:
      CORRECT_CONTINUOUS_PREDICTIVE_DISTRIBUTION

CONFORMAL_PREDICTION_CONNECTION:
  output:
    PREDICTION_SET_OR_INTERVAL

  guarantee:
    MARGINAL_COVERAGE
    UNDER:
      EXCHANGEABILITY

  distinction:
    CONFORMAL_COVERAGE != PROBABILITY_CALIBRATION_BY_DEFAULT

DECISION_THEORY_CONNECTION:
  decision:
    DEPENDS_ON:
      PREDICTED_PROBABILITY
      COSTS
      BENEFITS
      THRESHOLD

  implication:
    CALIBRATION_IMPORTANT_WHEN_DECISIONS_USE_ABSOLUTE_PROBABILITIES

THRESHOLD_SELECTION:
  rule:
    OPTIMAL_THRESHOLD_DEPENDS_ON:
      COSTS
      PREVALENCE
      UTILITY
      CALIBRATION

  caution:
    DEFAULT_0_5_THRESHOLD_NOT_UNIVERSALLY_OPTIMAL

PROPER_SCORING_RULES:
  examples:
    - LOG_LOSS
    - BRIER_SCORE

  property:
    EXPECTED_SCORE_OPTIMISED_BY_TRUE_DISTRIBUTION
    UNDER:
      PROPER_SCORING_RULE_DEFINITION

MODEL_COMPARISON:
  compare:
    - LOG_LOSS
    - BRIER_SCORE
    - CALIBRATION_CURVE
    - DISCRIMINATION_METRIC
    - DECISION_UTILITY

  rule:
    NO_SINGLE_METRIC_CAPTURES_ALL_PREDICTIVE_QUALITY_DIMENSIONS

CALIBRATION_AFTER_CLASS_WEIGHTING:
  consequence:
    RAW_MODEL_OUTPUT_MAY_REQUIRE_RECALIBRATION

CALIBRATION_AFTER_RESAMPLING:
  consequence:
    PREDICTED_PROBABILITIES_MAY_NOT_MATCH_DEPLOYMENT_PREVALENCE

APPLICATIONS:
  PROBABILITY_CALIBRATION -> SUPPORTS -> CLASSIFICATION
  PROBABILITY_CALIBRATION -> SUPPORTS -> RISK_PREDICTION
  PROBABILITY_CALIBRATION -> SUPPORTS -> FORECASTING
  PROBABILITY_CALIBRATION -> SUPPORTS -> DECISION_SUPPORT
  PROBABILITY_CALIBRATION -> SUPPORTS -> UNCERTAINTY_QUANTIFICATION
  PROBABILITY_CALIBRATION -> SUPPORTS -> MODEL_MONITORING
  PROBABILITY_CALIBRATION -> SUPPORTS -> ENSEMBLE_MODELLING

MODEL_CONNECTIONS:
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> MACHINE_LEARNING
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> STATISTICS
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> PROBABILITY_THEORY
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> BRIER_SCORE
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> LOG_LOSS
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> DISTRIBUTION_SHIFT
  PROBABILITY_CALIBRATION -> CONNECTS_TO -> CONFORMAL_PREDICTION

SEMANTIC_EQUIVALENCE:
  "probability calibration"
    ~= "agreement between predicted probabilities and observed frequencies"

  "reliability diagram"
    ~= "graphical calibration assessment"

  "Platt scaling"
    ~= "logistic post-hoc probability calibration"

  EXCLUDE_EQUIVALENCE:
    CALIBRATION != ACCURACY
    CALIBRATION != DISCRIMINATION
    CALIBRATION != SHARPNESS
    BRIER_SCORE != CALIBRATION_ERROR_ONLY
    ECE != UNIQUE_CALIBRATION_MEASURE
    CONFORMAL_COVERAGE != PROBABILITY_CALIBRATION
    PERFECT_CALIBRATION != PERFECT_PREDICTION

VALIDATION_RULES:
  RULE_1:
    PREDICTED_VALUES_MUST_BE_VALID_PROBABILITIES_WHEN_PROBABILITY_INTERPRETATION_CLAIMED

  RULE_2:
    CALIBRATION_DEFINITION_MUST_BE_EXPLICIT

  RULE_3:
    CALIBRATION_DATA_MUST_BE_SEPARATE_FROM_BASE_MODEL_FITTING_OR_USE_VALID_CROSS_VALIDATION

  RULE_4:
    BINNED_CALIBRATION_METRICS_MUST_DOCUMENT_BINNING_METHOD

  RULE_5:
    CALIBRATION_MUST_BE_DISTINGUISHED_FROM_DISCRIMINATION_AND_ACCURACY

  RULE_6:
    DEPLOYMENT_CALIBRATION_MUST_CONSIDER_DISTRIBUTION_SHIFT

  RULE_7:
    MULTICLASS_CALIBRATION_MUST_STATE_WHICH_CALIBRATION_NOTION_IS_USED

  RULE_8:
    CALIBRATION_ESTIMATES_MUST_ACCOUNT_FOR_SAMPLE_SIZE_AND_UNCERTAINTY

CONFIDENCE:
  ontology_identity: 0.99
  binary_calibration_definition: 0.99
  multiclass_structure: 0.98
  scoring_rule_relationship: 0.99
  post_hoc_calibration_structure: 0.99
  distribution_shift_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_CALIBRATION_DATA_ARRIVES
  THEN:
    UPDATE:
      RELIABILITY_DIAGRAM
      CALIBRATION_CURVE
      ECE
      BRIER_SCORE
      LOG_LOSS

  IF:
    CALIBRATOR_IS_REFIT
  THEN:
    UPDATE:
      CALIBRATION_PARAMETERS
      CALIBRATED_PROBABILITIES
    REEVALUATE:
      CALIBRATION
      DISCRIMINATION
      PROPER_SCORING_RULES

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      CALIBRATION
      BASE_RATE
      LABEL_SHIFT
      CONCEPT_SHIFT
      DECISION_THRESHOLDS

  PRESERVE:
    BASE_MODEL_VERSION
    CALIBRATOR_VERSION
    CALIBRATION_DATASET_VERSION
    CALIBRATION_METHOD
    BINNING_METHOD
    TARGET_DEFINITION
    PROBABILITY_OUTPUT_DEFINITION
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_PREDICTED_PROBABILITIES
    - IDENTIFY_TARGET_OUTCOMES
    - VERIFY_PROBABILITY_VALIDITY
    - IDENTIFY_CALIBRATION_DEFINITION
    - COMPUTE_CALIBRATION_CURVE
    - COMPUTE_BRIER_SCORE_AND_LOG_LOSS
    - COMPUTE_BINNED_CALIBRATION_METRIC_IF_REQUIRED
    - QUANTIFY_CALIBRATION_UNCERTAINTY
    - APPLY_POST_HOC_CALIBRATION_IF_REQUIRED
    - ASSESS_DISTRIBUTION_SHIFT
    - EVALUATE_DECISION_RELEVANCE
    - RETURN_CALIBRATION_INTERPRETATION

  constraints:
    - PRESERVE_BASE_MODEL_AND_CALIBRATOR_IDENTITIES
    - PRESERVE_TRAINING_CALIBRATION_TEST_SEPARATION
    - DISTINGUISH_CALIBRATION_FROM_DISCRIMINATION
    - DISTINGUISH_CALIBRATION_FROM_ACCURACY
    - DOCUMENT_BINNING_FOR_BINNED_METRICS
    - CONSIDER_DISTRIBUTION_SHIFT
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "probability-calibration"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - PREDICTED_PROBABILITIES
    - OBSERVED_FREQUENCIES
    - CALIBRATION_MAPPING

END_FLOW