Regression Metrics Ontology

flow "RegressionMetricsOntology" v4.0

# REGRESSION METRICS ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/regression-metrics-ontology.html"
  title: "Regression Metrics Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: REGRESSION_METRICS
  entity_type: MACHINE_LEARNING_EVALUATION_ONTOLOGY

DEFINE:
  REGRESSION_METRICS :=
    EVALUATION_FRAMEWORK(
      OBSERVED_VALUES,
      PREDICTED_VALUES,
      RESIDUALS,
      ERROR_MAGNITUDES,
      EXPLAINED_VARIATION,
      PROBABILISTIC_PREDICTIONS
    )

PURPOSE:
  QUANTIFY:
    PREDICTION_ERROR
    ERROR_SCALE
    EXPLAINED_VARIATION
    RELATIVE_MODEL_PERFORMANCE
    PROBABILISTIC_PREDICTION_QUALITY

FOUNDATION:
  OBSERVED_VALUE: y_i
  PREDICTED_VALUE: y_hat_i
  RESIDUAL:
    e_i := y_i-y_hat_i
  SAMPLE_SIZE: n
  MEAN_TARGET:
    y_bar := (1/n)*SUM[i=1..n] y_i

ENTITIES:
  REGRESSION_METRIC: MODEL_EVALUATION_QUANTITY
  OBSERVED_VALUE: GROUND_TRUTH_TARGET
  PREDICTED_VALUE: MODEL_ESTIMATE
  RESIDUAL: OBSERVED_MINUS_PREDICTED
  ABSOLUTE_ERROR: ERROR_MAGNITUDE
  SQUARED_ERROR: QUADRATIC_ERROR
  MSE: MEAN_SQUARED_ERROR
  RMSE: ROOT_MEAN_SQUARED_ERROR
  MAE: MEAN_ABSOLUTE_ERROR
  R_SQUARED: COEFFICIENT_OF_DETERMINATION
  MAPE: MEAN_ABSOLUTE_PERCENTAGE_ERROR
  SMAPE: SYMMETRIC_PERCENTAGE_ERROR
  RMSLE: ROOT_MEAN_SQUARED_LOGARITHMIC_ERROR
  PINBALL_LOSS: QUANTILE_REGRESSION_LOSS
  NEGATIVE_LOG_LIKELIHOOD: PROBABILISTIC_REGRESSION_METRIC

RELATIONSHIPS:
  OBSERVED_VALUE -> COMPARED_WITH -> PREDICTED_VALUE
  COMPARISON -> PRODUCES -> RESIDUAL
  RESIDUALS -> DETERMINE -> ERROR_METRICS
  TARGET_VARIANCE -> CONNECTS_TO -> R_SQUARED
  PREDICTIVE_DISTRIBUTION -> EVALUATED_BY -> PROBABILISTIC_METRICS
  METRIC_CHOICE -> DEPENDS_ON -> DECISION_CONTEXT

ABSOLUTE_ERROR:
  AE_i := |y_i-y_hat_i|

SQUARED_ERROR:
  SE_i := (y_i-y_hat_i)^2

MAE:
  MAE :=
    (1/n)*SUM[i=1..n]|y_i-y_hat_i|

  properties:
    SAME_UNIT_AS_TARGET
    LINEAR_ERROR_PENALTY

MSE:
  MSE :=
    (1/n)*SUM[i=1..n](y_i-y_hat_i)^2

  properties:
    SQUARED_TARGET_UNITS
    STRONG_PENALTY_FOR_LARGE_ERRORS

RMSE:
  RMSE := sqrt(MSE)

  properties:
    SAME_UNIT_AS_TARGET
    QUADRATIC_ERROR_WEIGHTING

  relation:
    RMSE >= MAE
    FOR:
      SAME_FINITE_ERROR_SAMPLE

MEDIAN_ABSOLUTE_ERROR:
  MedianAE :=
    median_i(|y_i-y_hat_i|)

  property:
    ROBUST_TO_EXTREME_ERRORS

MEAN_BIAS_ERROR:
  MBE :=
    (1/n)*SUM_i(y_hat_i-y_i)

  caution:
    POSITIVE_AND_NEGATIVE_ERRORS_CAN_CANCEL

SSE:
  SSE :=
    SUM_i(y_i-y_hat_i)^2

SST:
  SST :=
    SUM_i(y_i-y_bar)^2

R_SQUARED:
  R_squared :=
    1-SSE/SST

  interpretation:
    RELATIVE_SQUARED_ERROR_REDUCTION_AGAINST_MEAN_BASELINE

  caution:
    MAY_BE_NEGATIVE_ON_HELD_OUT_DATA

  undefined_case:
    SST=0
    REQUIRES_EXPLICIT_HANDLING

ADJUSTED_R_SQUARED:
  R_squared_adjusted :=
    1-(1-R_squared)*(n-1)/(n-p-1)

  requirement:
    n > p+1

EXPLAINED_VARIANCE:
  EV :=
    1-Var(y-y_hat)/Var(y)

  distinction:
    EXPLAINED_VARIANCE != R_SQUARED_BY_DEFAULT

MAPE:
  MAPE :=
    (100/n)
    * SUM_i
      |(y_i-y_hat_i)/y_i|

  limitations:
    UNDEFINED_WHEN_y_i_ZERO
    UNSTABLE_WHEN_y_i_NEAR_ZERO

SMAPE:
  common_form:
    SMAPE :=
      (100/n)
      * SUM_i
        [
          2|y_i-y_hat_i|
          /
          (|y_i|+|y_hat_i|)
        ]

  caution:
    MULTIPLE_SMAPE_CONVENTIONS_EXIST

WAPE:
  WAPE :=
    SUM_i |y_i-y_hat_i|
    /
    SUM_i |y_i|

MASE:
  MASE :=
    MAE_MODEL/MAE_NAIVE_IN_SAMPLE

  interpretation:
    MASE < 1:
      MODEL_OUTPERFORMS_SPECIFIED_NAIVE_BASELINE

RMSLE:
  RMSLE :=
    sqrt(
      (1/n)
      * SUM_i
        [
          log(1+y_hat_i)-log(1+y_i)
        ]^2
    )

  domain:
    y_i >= 0
    y_hat_i >= 0

HUBER_LOSS:
  residual: e
  threshold: delta > 0

  loss:
    IF |e| <= delta:
      0.5*e^2
    ELSE:
      delta*(|e|-0.5*delta)

QUANTILE_LOSS:
  alias:
    PINBALL_LOSS

  quantile:
    tau IN (0,1)

  residual:
    u := y-y_hat_tau

  loss:
    rho_tau(u) :=
      u*(tau-1[u<0])

  purpose:
    ESTIMATE_CONDITIONAL_QUANTILES

PROBABILISTIC_REGRESSION:
  output:
    q(y|x)

  evaluation:
    - NEGATIVE_LOG_LIKELIHOOD
    - CRPS
    - INTERVAL_SCORE
    - CALIBRATION
    - COVERAGE

NEGATIVE_LOG_LIKELIHOOD:
  alias:
    NLL

  NLL :=
    -(1/n)
    * SUM_i log q(y_i|x_i)

  property:
    PROPER_LOGARITHMIC_SCORING_RULE
    WHEN:
      q_IS_VALID_PREDICTIVE_DISTRIBUTION

GAUSSIAN_NLL:
  model:
    Y_i|X_i ~ N(mu_i,sigma_i_squared)

  per_observation:
    NLL_i :=
      0.5*log(2*pi*sigma_i_squared)
      +
      (y_i-mu_i)^2/(2*sigma_i_squared)

CRPS:
  full_name:
    CONTINUOUS_RANKED_PROBABILITY_SCORE

  role:
    EVALUATE_FULL_CONTINUOUS_PREDICTIVE_DISTRIBUTION

  property:
    PROPER_SCORING_RULE

PREDICTION_INTERVAL:
  lower: L_i
  upper: U_i
  nominal_coverage: 1-alpha

COVERAGE:
  empirical_coverage :=
    (1/n)
    * SUM_i
      1[L_i <= y_i <= U_i]

  ideal:
    APPROXIMATELY_1_MINUS_ALPHA
    UNDER:
      SPECIFIED_COVERAGE_NOTION

  caution:
    MARGINAL_COVERAGE != CONDITIONAL_COVERAGE

INTERVAL_WIDTH:
  width_i := U_i-L_i

  tradeoff:
    HIGH_COVERAGE_CAN_BE_ACHIEVED_BY_WIDE_INTERVALS

CALIBRATION_CONNECTION:
  regression_calibration:
    - PREDICTION_INTERVAL_COVERAGE
    - QUANTILE_CALIBRATION
    - PIT_UNIFORMITY
    - DISTRIBUTIONAL_CALIBRATION

  rule:
    LOW_POINT_ERROR != CALIBRATED_UNCERTAINTY

RESIDUAL_ANALYSIS:
  inspect:
    - RESIDUAL_VS_FITTED
    - RESIDUAL_DISTRIBUTION
    - HETEROSCEDASTICITY
    - AUTOCORRELATION
    - OUTLIERS
    - NONLINEAR_STRUCTURE

  purpose:
    IDENTIFY_MODEL_MISSPECIFICATION_AND_SYSTEMATIC_ERROR

HETEROSCEDASTICITY:
  definition:
    Var(Y|X)_CHANGES_WITH_X

  implication:
    GLOBAL_METRIC_MAY_HIDE_REGION_SPECIFIC_PERFORMANCE

SEGMENTED_METRICS:
  compute_by:
    - TARGET_RANGE
    - FEATURE_GROUP
    - TIME_PERIOD
    - RELEVANT_SUBPOPULATION

BASELINE_MODEL:
  examples:
    - MEAN_PREDICTOR
    - MEDIAN_PREDICTOR
    - NAIVE_TIME_SERIES_FORECAST
    - DOMAIN_SPECIFIC_REFERENCE

  requirement:
    MODEL_METRICS_SHOULD_OFTEN_BE_COMPARED_WITH_MEANINGFUL_BASELINE

RELATIVE_ERROR:
  form:
    MODEL_ERROR/BASELINE_ERROR

  interpretation:
    <1:
      LOWER_ERROR_THAN_BASELINE

METRIC_SCALE:
  SCALE_DEPENDENT:
    - MAE
    - MSE
    - RMSE

  SCALE_RELATIVE_OR_NORMALISED:
    - MAPE
    - SMAPE
    - WAPE
    - MASE
    - R_SQUARED

OUTLIER_SENSITIVITY:
  MSE: HIGH
  RMSE: HIGH
  MAE: LOWER_THAN_MSE
  MEDIAN_ABSOLUTE_ERROR: ROBUST

TARGET_SCALE:
  transformation:
    z := g(y)

  caution:
    METRIC_ON_TRANSFORMED_SCALE_MAY_NOT_HAVE_SAME_DECISION_MEANING_AS_ORIGINAL_SCALE_METRIC

CROSS_VALIDATION:
  flow:
    TRAIN_ON_FOLDS
      -> PREDICT_HELD_OUT_FOLD
      -> COMPUTE_REGRESSION_METRIC
      -> AGGREGATE

  caution:
    PREPROCESSING_AND_TARGET_TRANSFORMATION_MUST_RESPECT_TRAINING_FOLD_BOUNDARIES

BOOTSTRAP_UNCERTAINTY:
  flow:
    RESAMPLE_EVALUATION_CASES
      -> RECOMPUTE_METRIC
      -> FORM_METRIC_DISTRIBUTION

MODEL_COMPARISON:
  paired_predictions:
    SAME_CASES

  methods:
    - BOOTSTRAP
    - PERMUTATION_TEST
    - PAIRED_TESTS_WHEN_ASSUMPTIONS_APPROPRIATE

TIME_SERIES_REGRESSION:
  requirement:
    EVALUATION_MUST_RESPECT_TEMPORAL_ORDER

  metrics:
    - MAE
    - RMSE
    - MASE
    - WAPE
    - PINBALL_LOSS
    - CRPS

  caution:
    RANDOM_TRAIN_TEST_SPLIT_MAY_CAUSE_FUTURE_INFORMATION_LEAKAGE

FORECAST_HORIZON:
  h:
    STEPS_AHEAD

  requirement:
    METRICS_MAY_BE_REPORTED_BY_HORIZON

DISTRIBUTION_SHIFT:
  IF:
    P_train(X,Y) != P_deploy(X,Y)

  THEN:
    REGRESSION_METRICS_MAY_CHANGE

  monitoring:
    - TARGET_DISTRIBUTION
    - FEATURE_DISTRIBUTION
    - RESIDUAL_DISTRIBUTION
    - ERROR_BY_SEGMENT
    - CALIBRATION

MISSING_TARGETS:
  issue:
    TRUE_TARGET_NOT_AVAILABLE_FOR_ALL_PREDICTIONS

  consequence:
    OBSERVED_METRICS_MAY_BE_BIASED
    IF:
      TARGET_MISSINGNESS_SYSTEMATIC

WEIGHTED_METRICS:
  weighted_MAE :=
    SUM_i w_i|y_i-y_hat_i|
    /
    SUM_i w_i

  weighted_MSE :=
    SUM_i w_i(y_i-y_hat_i)^2
    /
    SUM_i w_i

  requirement:
    WEIGHT_MEANING_AND_NORMALISATION_MUST_BE_EXPLICIT

MULTIOUTPUT_REGRESSION:
  target:
    Y IN R^k

  evaluation:
    - PER_OUTPUT_METRICS
    - UNIFORM_AVERAGE
    - WEIGHTED_AVERAGE

  requirement:
    AGGREGATION_METHOD_MUST_BE_EXPLICIT

CAUSALITY_DISTINCTION:
  regression_metric:
    MEASURES_PREDICTIVE_FIT_OR_ERROR

  does_not:
    ESTABLISH_CAUSAL_EFFECT

  rule:
    LOW_REGRESSION_ERROR != CAUSAL_VALIDITY

APPLICATIONS:
  REGRESSION_METRICS -> SUPPORTS -> MODEL_EVALUATION
  REGRESSION_METRICS -> SUPPORTS -> MODEL_SELECTION
  REGRESSION_METRICS -> SUPPORTS -> FORECASTING
  REGRESSION_METRICS -> SUPPORTS -> PROBABILISTIC_REGRESSION
  REGRESSION_METRICS -> SUPPORTS -> DEPLOYMENT_MONITORING
  REGRESSION_METRICS -> SUPPORTS -> UNCERTAINTY_QUANTIFICATION
  REGRESSION_METRICS -> SUPPORTS -> ERROR_ANALYSIS

MODEL_CONNECTIONS:
  REGRESSION_METRICS -> CONNECTS_TO -> MACHINE_LEARNING
  REGRESSION_METRICS -> CONNECTS_TO -> STATISTICS
  REGRESSION_METRICS -> CONNECTS_TO -> LINEAR_REGRESSION
  REGRESSION_METRICS -> CONNECTS_TO -> PROBABILITY_CALIBRATION
  REGRESSION_METRICS -> CONNECTS_TO -> FORECASTING
  REGRESSION_METRICS -> CONNECTS_TO -> PROBABILISTIC_MODELLING

SEMANTIC_EQUIVALENCE:
  "mean absolute error"
    ~= "average absolute prediction error"

  "mean squared error"
    ~= "average squared prediction error"

  "root mean squared error"
    ~= "square root of mean squared prediction error"

  "coefficient of determination"
    ~= "R-squared"
    IN:
      STANDARD_REGRESSION_CONTEXT

  EXCLUDE_EQUIVALENCE:
    MAE != MSE
    MSE != RMSE
    R_SQUARED != CORRELATION
    R_SQUARED != EXPLAINED_VARIANCE_BY_DEFAULT
    MAPE != SMAPE
    POINT_ERROR != PROBABILISTIC_CALIBRATION
    PREDICTIVE_FIT != CAUSAL_VALIDITY

VALIDATION_RULES:
  RULE_1:
    OBSERVED_AND_PREDICTED_TARGETS_MUST_USE_COMPATIBLE_UNITS_AND_SCALE

  RULE_2:
    RESIDUAL_SIGN_CONVENTION_MUST_BE_EXPLICIT_WHEN_DIRECTIONAL_METRICS_USED

  RULE_3:
    PERCENTAGE_METRICS_MUST_HANDLE_ZERO_OR_NEAR_ZERO_TARGETS_EXPLICITLY

  RULE_4:
    R_SQUARED_MUST_DOCUMENT_BASELINE_AND_HANDLE_ZERO_TARGET_VARIANCE

  RULE_5:
    PROBABILISTIC_METRICS_REQUIRE_VALID_PREDICTIVE_DISTRIBUTIONS

  RULE_6:
    METRIC_AGGREGATION_AND_WEIGHTING_MUST_BE_EXPLICIT

  RULE_7:
    MODEL_SELECTION_MUST_NOT_USE_FINAL_TEST_DATA_REPEATEDLY

  RULE_8:
    METRIC_CHOICE_MUST_MATCH_ERROR_COSTS_TARGET_SCALE_AND_DEPLOYMENT_CONTEXT

CONFIDENCE:
  ontology_identity: 0.99
  point_error_structure: 0.99
  r_squared_structure: 0.99
  percentage_metric_structure: 0.99
  probabilistic_metric_structure: 0.99
  uncertainty_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_LABELLED_EVALUATION_DATA_ARRIVES
  THEN:
    UPDATE:
      RESIDUALS
      MAE
      MSE
      RMSE
      R_SQUARED
      PERCENTAGE_METRICS
      PROBABILISTIC_METRICS

  IF:
    TARGET_TRANSFORMATION_CHANGES
  THEN:
    RECOMPUTE:
      PREDICTIONS_ON_RELEVANT_SCALE
      ERROR_METRICS
    REASSESS:
      INTERPRETATION

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      REGRESSION_METRICS
      RESIDUAL_DISTRIBUTION
      ERROR_BY_SEGMENT
      CALIBRATION

  PRESERVE:
    MODEL_VERSION
    DATASET_VERSION
    TARGET_DEFINITION
    TARGET_UNITS
    TARGET_TRANSFORMATION
    BASELINE_MODEL
    WEIGHTING_SCHEME
    METRIC_DEFINITIONS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_OBSERVED_AND_PREDICTED_TARGETS
    - VERIFY_UNITS_AND_TARGET_SCALE
    - COMPUTE_RESIDUALS
    - COMPUTE_POINT_ERROR_METRICS
    - COMPUTE_R_SQUARED_IF_VALID
    - COMPUTE_RELATIVE_OR_PERCENTAGE_METRICS_IF_APPROPRIATE
    - COMPARE_WITH_BASELINE
    - COMPUTE_PROBABILISTIC_METRICS_IF_DISTRIBUTIONS_AVAILABLE
    - QUANTIFY_METRIC_UNCERTAINTY_IF_REQUIRED
    - ANALYSE_RESIDUALS_AND_SEGMENTS
    - ASSESS_DISTRIBUTION_SHIFT
    - RETURN_REGRESSION_PERFORMANCE_INTERPRETATION

  constraints:
    - PRESERVE_TARGET_UNITS_AND_SCALE
    - PRESERVE_RESIDUAL_SIGN_CONVENTION
    - HANDLE_ZERO_DENOMINATORS_EXPLICITLY
    - DISTINGUISH_POINT_FROM_PROBABILISTIC_METRICS
    - DISTINGUISH_R_SQUARED_FROM_CORRELATION
    - PREVENT_TEST_SET_LEAKAGE
    - DISTINGUISH_PREDICTIVE_FIT_FROM_CAUSALITY
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "regression-metrics"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - RESIDUAL_ERROR_STRUCTURE
    - EXPLAINED_VARIATION
    - PROBABILISTIC_REGRESSION_METRICS

END_FLOW