Classification Metrics Ontology

flow "ClassificationMetricsOntology" v4.0

# CLASSIFICATION METRICS ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/classification-metrics-ontology.html"
  title: "Classification Metrics Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: CLASSIFICATION_METRICS
  entity_type: MACHINE_LEARNING_EVALUATION_ONTOLOGY

DEFINE:
  CLASSIFICATION_METRICS :=
    EVALUATION_FRAMEWORK(
      TRUE_LABELS,
      PREDICTED_LABELS,
      PREDICTED_PROBABILITIES,
      CONFUSION_MATRIX,
      THRESHOLDS,
      PERFORMANCE_METRICS
    )

PURPOSE:
  QUANTIFY:
    CLASSIFICATION_CORRECTNESS
    CLASS_SPECIFIC_PERFORMANCE
    RANKING_PERFORMANCE
    PROBABILISTIC_PERFORMANCE
    ERROR_TRADEOFFS

FOUNDATION:
  TRUE_LABEL: Y
  PREDICTED_LABEL: Y_hat
  PREDICTED_PROBABILITY: q(y|x)

  CONFUSION_MATRIX:
    TP
    TN
    FP
    FN

ENTITIES:
  CLASSIFICATION_METRIC: MODEL_EVALUATION_QUANTITY
  CONFUSION_MATRIX: CLASSIFICATION_OUTCOME_TABLE
  ACCURACY: OVERALL_CORRECTNESS
  PRECISION: POSITIVE_PREDICTIVE_VALUE
  RECALL: TRUE_POSITIVE_RATE
  SPECIFICITY: TRUE_NEGATIVE_RATE
  F1_SCORE: HARMONIC_PRECISION_RECALL_SUMMARY
  ROC_AUC: RANKING_DISCRIMINATION_METRIC
  PR_AUC: PRECISION_RECALL_SUMMARY
  LOG_LOSS: PROBABILISTIC_SCORING_METRIC
  BRIER_SCORE: PROBABILITY_ERROR_METRIC

RELATIONSHIPS:
  TRUE_LABELS -> COMPARED_WITH -> PREDICTED_LABELS
  COMPARISON -> PRODUCES -> CONFUSION_MATRIX
  CONFUSION_MATRIX -> DETERMINES -> THRESHOLD_METRICS
  PREDICTED_PROBABILITIES -> DETERMINE -> RANKING_METRICS
  PREDICTED_PROBABILITIES -> DETERMINE -> PROBABILISTIC_METRICS
  DECISION_THRESHOLD -> CHANGES -> CONFUSION_MATRIX
  CLASS_PREVALENCE -> INFLUENCES -> CLASSIFICATION_METRICS

CONFUSION_MATRIX:
  TRUE_POSITIVE: TP
  TRUE_NEGATIVE: TN
  FALSE_POSITIVE: FP
  FALSE_NEGATIVE: FN
  total:
    N := TP+TN+FP+FN

ACCURACY:
  Accuracy :=
    (TP+TN)/(TP+TN+FP+FN)

  caution:
    MAY_BE_MISLEADING_WITH_CLASS_IMBALANCE_OR_UNEQUAL_ERROR_COSTS

ERROR_RATE:
  ErrorRate :=
    (FP+FN)/N

  relation:
    ErrorRate := 1-Accuracy

PRECISION:
  Precision :=
    TP/(TP+FP)

  alias:
    POSITIVE_PREDICTIVE_VALUE

RECALL:
  Recall :=
    TP/(TP+FN)

  aliases:
    SENSITIVITY
    TRUE_POSITIVE_RATE

SPECIFICITY:
  Specificity :=
    TN/(TN+FP)

  alias:
    TRUE_NEGATIVE_RATE

FALSE_POSITIVE_RATE:
  FPR :=
    FP/(FP+TN)

  relation:
    FPR := 1-Specificity

FALSE_NEGATIVE_RATE:
  FNR :=
    FN/(FN+TP)

  relation:
    FNR := 1-Recall

NEGATIVE_PREDICTIVE_VALUE:
  NPV :=
    TN/(TN+FN)

F1_SCORE:
  F1 :=
    2*Precision*Recall
    /
    (Precision+Recall)

  equivalent:
    F1 :=
      2TP/(2TP+FP+FN)

F_BETA_SCORE:
  F_beta :=
    (1+beta^2)*Precision*Recall
    /
    (beta^2*Precision+Recall)

  interpretation:
    beta > 1:
      EMPHASISE_RECALL
    beta < 1:
      EMPHASISE_PRECISION

BALANCED_ACCURACY:
  binary:
    BalancedAccuracy :=
      (Recall+Specificity)/2

YOUDEN_J:
  J :=
    Sensitivity+Specificity-1

MATTHEWS_CORRELATION_COEFFICIENT:
  alias:
    MCC

  MCC :=
    (TP*TN-FP*FN)
    /
    sqrt(
      (TP+FP)
      (TP+FN)
      (TN+FP)
      (TN+FN)
    )

  range:
    -1 <= MCC <= 1
    WHEN_DENOMINATOR_NONZERO

PREVALENCE:
  Prevalence :=
    (TP+FN)/N

THRESHOLD:
  rule:
    IF q >= t:
      PREDICT_POSITIVE
    ELSE:
      PREDICT_NEGATIVE

  effect:
    CHANGES:
      TP
      TN
      FP
      FN
      PRECISION
      RECALL
      SPECIFICITY

ROC_CURVE:
  x_axis:
    FALSE_POSITIVE_RATE
  y_axis:
    TRUE_POSITIVE_RATE
  generated_by:
    VARYING_THRESHOLD

ROC_AUC:
  role:
    RANKING_DISCRIMINATION

  range:
    0 <= ROC_AUC <= 1

  caution:
    ROC_AUC_DOES_NOT_MEASURE_CALIBRATION_OR_THRESHOLD_SPECIFIC_UTILITY

PRECISION_RECALL_CURVE:
  x_axis:
    RECALL
  y_axis:
    PRECISION
  generated_by:
    VARYING_THRESHOLD

PR_AUC:
  role:
    SUMMARISE_PRECISION_RECALL_TRADEOFF

  relevance:
    OFTEN_USEFUL_FOR_RARE_POSITIVE_CLASS

  caution:
    PR_AUC_DEPENDS_ON_CLASS_PREVALENCE

LOG_LOSS:
  binary:
    L :=
      -[
        y log q
        + (1-y)log(1-q)
      ]

  multiclass:
    L :=
      -SUM_k y_k log q_k

  property:
    STRICTLY_PROPER_SCORING_RULE

BRIER_SCORE:
  binary:
    BS :=
      (1/n)
      * SUM_i (q_i-y_i)^2

  property:
    PROPER_SCORING_RULE

CALIBRATION_CONNECTION:
  ideal:
    P(Y=1 | Q=p)
      approximately_equals
    p

  rule:
    DISCRIMINATION != CALIBRATION

CLASS_IMBALANCE:
  caution:
    ACCURACY_MAY_BE_DOMINATED_BY_MAJORITY_CLASS

  useful_metrics:
    - PRECISION
    - RECALL
    - F1
    - BALANCED_ACCURACY
    - MCC
    - PR_AUC

COST_SENSITIVE_EVALUATION:
  expected_cost:
    DEPENDS_ON:
      FALSE_POSITIVE_COST
      FALSE_NEGATIVE_COST
      PREVALENCE
      DECISION_THRESHOLD

  rule:
    METRIC_SELECTION_SHOULD_MATCH_DECISION_COSTS

THRESHOLD_SELECTION:
  criteria:
    - MAXIMISE_F1
    - MAXIMISE_YOUDEN_J
    - MINIMISE_EXPECTED_COST
    - SATISFY_MINIMUM_RECALL
    - SATISFY_MINIMUM_PRECISION

  caution:
    THRESHOLD_MUST_BE_SELECTED_ON_VALIDATION_DATA
    NOT:
      FINAL_TEST_DATA

MULTICLASS_CLASSIFICATION:
  classes:
    1,...,K

  confusion_matrix:
    K_BY_K

  evaluation:
    PER_CLASS_METRICS
    +
    AGGREGATED_METRICS

MACRO_AVERAGING:
  procedure:
    COMPUTE_METRIC_PER_CLASS
      -> UNWEIGHTED_MEAN

  property:
    TREATS_CLASSES_EQUALLY

MICRO_AVERAGING:
  procedure:
    AGGREGATE_COUNTS_ACROSS_CLASSES
      -> COMPUTE_METRIC

  property:
    WEIGHTS_BY_INSTANCE_FREQUENCY

WEIGHTED_AVERAGING:
  procedure:
    COMPUTE_CLASS_METRIC
      -> WEIGHT_BY_CLASS_SUPPORT
      -> SUM

  caution:
    MAY_BE_DOMINATED_BY_COMMON_CLASSES

ONE_VS_REST:
  flow:
    EACH_CLASS_AS_POSITIVE
      -> OTHER_CLASSES_AS_NEGATIVE
      -> COMPUTE_BINARY_STYLE_METRICS

MULTILABEL_CLASSIFICATION:
  property:
    OBSERVATION_MAY_HAVE_MULTIPLE_TRUE_LABELS

  metrics:
    - HAMMING_LOSS
    - SUBSET_ACCURACY
    - MICRO_F1
    - MACRO_F1
    - LABELWISE_PRECISION
    - LABELWISE_RECALL

  caution:
    MULTILABEL != MULTICLASS

HAMMING_LOSS:
  definition:
    FRACTION_OF_LABEL_ASSIGNMENTS_INCORRECT

SUBSET_ACCURACY:
  definition:
    FRACTION_OF_SAMPLES_WITH_EXACT_LABEL_SET_MATCH

TOP_K_ACCURACY:
  definition:
    TRUE_CLASS_APPEARS_WITHIN_TOP_k_PREDICTED_CLASSES

CONFUSION_MATRIX_NORMALISATION:
  options:
    - RAW_COUNTS
    - NORMALISE_BY_TRUE_CLASS
    - NORMALISE_BY_PREDICTED_CLASS
    - NORMALISE_GLOBALLY

  requirement:
    NORMALISATION_METHOD_MUST_BE_EXPLICIT

BOOTSTRAP_UNCERTAINTY:
  flow:
    RESAMPLE_EVALUATION_CASES
      -> RECOMPUTE_METRIC
      -> FORM_EMPIRICAL_METRIC_DISTRIBUTION

  uses:
    - CONFIDENCE_INTERVAL
    - MODEL_COMPARISON
    - METRIC_STABILITY

CROSS_VALIDATION_METRICS:
  flow:
    TRAIN_ON_FOLDS
      -> PREDICT_HELD_OUT_FOLD
      -> COMPUTE_METRIC
      -> AGGREGATE

MODEL_COMPARISON:
  paired_methods:
    - BOOTSTRAP
    - PERMUTATION_TEST
    - MCNEMAR_TEST
    - DELONG_METHOD_FOR_ROC_AUC

  caution:
    TEST_ASSUMPTIONS_MUST_MATCH_DATA_STRUCTURE

MCNEMAR_TEST:
  use:
    COMPARE_TWO_CLASSIFIERS_ON_PAIRED_BINARY_OUTCOMES

  focuses_on:
    DISCORDANT_PREDICTIONS

DELONG_CONNECTION:
  use:
    ESTIMATE_VARIANCE_OR_COMPARE_CORRELATED_ROC_AUC_VALUES

TEMPORAL_EVALUATION:
  requirement:
    TRAINING_DATA_PRECEDE_VALIDATION_OR_TEST_DATA
    WHEN:
      FUTURE_PREDICTION_INTENDED

  caution:
    RANDOM_SPLIT_MAY_CAUSE_TEMPORAL_LEAKAGE

GROUPED_DATA:
  examples:
    - MULTIPLE_RECORDS_PER_PERSON
    - MULTIPLE_IMAGES_PER_SUBJECT
    - CLUSTERED_OBSERVATIONS

  requirement:
    SPLITTING_AND_UNCERTAINTY_SHOULD_RESPECT_GROUP_STRUCTURE

DISTRIBUTION_SHIFT:
  IF:
    P_train(X,Y) != P_deploy(X,Y)

  THEN:
    CLASSIFICATION_METRICS_MAY_CHANGE

  monitoring:
    - PERFORMANCE_OVER_TIME
    - CLASS_PREVALENCE
    - CALIBRATION
    - FEATURE_DISTRIBUTIONS

PREVALENCE_SHIFT:
  change:
    P(Y)

  effects:
    PRECISION:
      MAY_CHANGE
    NPV:
      MAY_CHANGE

  caution:
    DEPLOYMENT_METRICS_REQUIRE_CURRENT_BASE_RATE_CONTEXT

MISSING_LABELS:
  issue:
    OUTCOMES_NOT_OBSERVED_FOR_ALL_PREDICTIONS

  consequence:
    METRIC_ESTIMATE_MAY_BE_BIASED
    IF:
      LABEL_MISSINGNESS_NOT_RANDOM

SELECTIVE_CLASSIFICATION:
  model:
    MAY_ABSTAIN_ON_UNCERTAIN_CASES

  metrics:
    - COVERAGE
    - SELECTIVE_RISK
    - ACCURACY_AT_COVERAGE

COVERAGE:
  definition:
    FRACTION_OF_CASES_RECEIVING_AUTOMATIC_PREDICTION

SELECTIVE_RISK:
  definition:
    ERROR_RATE_AMONG_NON_ABSTAINED_CASES

FAIRNESS_EVALUATION_CONNECTION:
  group_metrics:
    - TRUE_POSITIVE_RATE
    - FALSE_POSITIVE_RATE
    - PRECISION
    - CALIBRATION

  caution:
    FAIRNESS_DEFINITIONS_ARE_CONTEXT_DEPENDENT

CAUSALITY_DISTINCTION:
  classification_metric:
    MEASURES_PREDICTIVE_PERFORMANCE

  does_not:
    ESTABLISH_CAUSAL_RELATIONSHIP

  rule:
    PREDICTIVE_METRIC != CAUSAL_EFFECT

APPLICATIONS:
  CLASSIFICATION_METRICS -> SUPPORTS -> MODEL_EVALUATION
  CLASSIFICATION_METRICS -> SUPPORTS -> MODEL_SELECTION
  CLASSIFICATION_METRICS -> SUPPORTS -> THRESHOLD_SELECTION
  CLASSIFICATION_METRICS -> SUPPORTS -> DEPLOYMENT_MONITORING
  CLASSIFICATION_METRICS -> SUPPORTS -> CLASS_IMBALANCE_ANALYSIS
  CLASSIFICATION_METRICS -> SUPPORTS -> PROBABILISTIC_CLASSIFICATION
  CLASSIFICATION_METRICS -> SUPPORTS -> MULTICLASS_EVALUATION

MODEL_CONNECTIONS:
  CLASSIFICATION_METRICS -> CONNECTS_TO -> MACHINE_LEARNING
  CLASSIFICATION_METRICS -> CONNECTS_TO -> STATISTICS
  CLASSIFICATION_METRICS -> CONNECTS_TO -> PROBABILITY_CALIBRATION
  CLASSIFICATION_METRICS -> CONNECTS_TO -> CONFUSION_MATRIX
  CLASSIFICATION_METRICS -> CONNECTS_TO -> ROC_ANALYSIS
  CLASSIFICATION_METRICS -> CONNECTS_TO -> PRECISION_RECALL_ANALYSIS
  CLASSIFICATION_METRICS -> CONNECTS_TO -> DECISION_THEORY

SEMANTIC_EQUIVALENCE:
  "recall"
    ~= "sensitivity"
    ~= "true positive rate"

  "specificity"
    ~= "true negative rate"

  "precision"
    ~= "positive predictive value"

  "classification error"
    ~= "misclassification rate"
    IN:
      STANDARD_HARD_CLASSIFICATION_CONTEXT

  EXCLUDE_EQUIVALENCE:
    ACCURACY != PRECISION
    PRECISION != RECALL
    ROC_AUC != ACCURACY
    PR_AUC != ROC_AUC
    CALIBRATION != DISCRIMINATION
    F1 != BALANCED_ACCURACY
    MULTICLASS != MULTILABEL
    PREDICTIVE_PERFORMANCE != CAUSAL_VALIDITY

VALIDATION_RULES:
  RULE_1:
    POSITIVE_CLASS_AND_LABEL_ENCODING_MUST_BE_EXPLICIT

  RULE_2:
    METRIC_DENOMINATORS_MUST_BE_CHECKED_FOR_ZERO

  RULE_3:
    CLASS_PREVALENCE_MUST_BE_CONSIDERED_WHEN_INTERPRETING_PRECISION_PR_AUC_AND_ACCURACY

  RULE_4:
    THRESHOLD_DEPENDENT_METRICS_MUST_STATE_OR_PRESERVE_DECISION_THRESHOLD

  RULE_5:
    PROBABILISTIC_METRICS_REQUIRE_VALID_PROBABILITIES

  RULE_6:
    MULTICLASS_AVERAGING_METHOD_MUST_BE_EXPLICIT

  RULE_7:
    MODEL_SELECTION_AND_THRESHOLD_SELECTION_MUST_NOT_USE_FINAL_TEST_DATA

  RULE_8:
    METRIC_INTERPRETATION_MUST_MATCH_DECISION_COSTS_AND_DEPLOYMENT_DISTRIBUTION

CONFIDENCE:
  ontology_identity: 0.99
  confusion_matrix_structure: 0.99
  precision_recall_structure: 0.99
  roc_pr_structure: 0.99
  probabilistic_metric_structure: 0.99
  multiclass_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_LABELLED_EVALUATION_DATA_ARRIVES
  THEN:
    UPDATE:
      CONFUSION_MATRIX
      ACCURACY
      PRECISION
      RECALL
      SPECIFICITY
      F1
      ROC_AUC
      PR_AUC
      LOG_LOSS
      BRIER_SCORE

  IF:
    DECISION_THRESHOLD_CHANGES
  THEN:
    RECOMPUTE:
      TP
      TN
      FP
      FN
      THRESHOLD_DEPENDENT_METRICS

  IF:
    CLASS_PREVALENCE_CHANGES
  THEN:
    REASSESS:
      PRECISION
      NPV
      PR_METRICS
      EXPECTED_COST
      THRESHOLD

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      ALL_DEPLOYMENT_METRICS
      CALIBRATION
      CLASS_PREVALENCE
      DISTRIBUTION_SHIFT

  PRESERVE:
    MODEL_VERSION
    DATASET_VERSION
    LABEL_DEFINITION
    POSITIVE_CLASS
    DECISION_THRESHOLD
    AVERAGING_METHOD
    METRIC_DEFINITIONS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_CLASSIFICATION_TYPE
    - IDENTIFY_TRUE_AND_PREDICTED_LABELS
    - IDENTIFY_POSITIVE_CLASS_IF_BINARY
    - BUILD_CONFUSION_MATRIX
    - COMPUTE_THRESHOLD_DEPENDENT_METRICS
    - COMPUTE_RANKING_METRICS_IF_SCORES_AVAILABLE
    - COMPUTE_PROBABILISTIC_METRICS_IF_PROBABILITIES_AVAILABLE
    - ACCOUNT_FOR_CLASS_IMBALANCE
    - QUANTIFY_METRIC_UNCERTAINTY_IF_REQUIRED
    - ASSESS_CALIBRATION_AND_DISTRIBUTION_SHIFT
    - MATCH_METRICS_TO_DECISION_COSTS
    - RETURN_CLASSIFICATION_PERFORMANCE_INTERPRETATION

  constraints:
    - PRESERVE_LABEL_SEMANTICS
    - PRESERVE_POSITIVE_CLASS_IDENTITY
    - PRESERVE_DECISION_THRESHOLD
    - DISTINGUISH_THRESHOLD_RANKING_AND_PROBABILISTIC_METRICS
    - DISTINGUISH_CALIBRATION_FROM_DISCRIMINATION
    - DOCUMENT_MULTICLASS_AVERAGING
    - PREVENT_TEST_SET_LEAKAGE
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "classification-metrics"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - CONFUSION_MATRIX
    - THRESHOLD_METRICS
    - RANKING_AND_PROBABILISTIC_METRICS

END_FLOW