Confusion Matrix Ontology

flow "ConfusionMatrixOntology" v4.0

# CONFUSION MATRIX ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/confusion-matrix-ontology.html"
  title: "Confusion Matrix Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: CONFUSION_MATRIX
  entity_type: MACHINE_LEARNING_EVALUATION_ONTOLOGY

DEFINE:
  CONFUSION_MATRIX :=
    CLASSIFICATION_OUTCOME_STRUCTURE(
      TRUE_LABELS,
      PREDICTED_LABELS,
      TRUE_POSITIVES,
      TRUE_NEGATIVES,
      FALSE_POSITIVES,
      FALSE_NEGATIVES
    )

PURPOSE:
  REPRESENT:
    CLASSIFICATION_OUTCOMES
    CORRECT_PREDICTIONS
    CLASSIFICATION_ERRORS
    CLASS_SPECIFIC_PERFORMANCE
    THRESHOLD_DEPENDENT_BEHAVIOUR

FOUNDATION:
  TRUE_LABEL: Y
  PREDICTED_LABEL: Y_hat

  BINARY_MATRIX:
    ACTUAL_POSITIVE:
      PREDICTED_POSITIVE: TP
      PREDICTED_NEGATIVE: FN
    ACTUAL_NEGATIVE:
      PREDICTED_POSITIVE: FP
      PREDICTED_NEGATIVE: TN

ENTITIES:
  CONFUSION_MATRIX: CLASSIFICATION_OUTCOME_TABLE
  TRUE_POSITIVE: CORRECT_POSITIVE_PREDICTION
  TRUE_NEGATIVE: CORRECT_NEGATIVE_PREDICTION
  FALSE_POSITIVE: INCORRECT_POSITIVE_PREDICTION
  FALSE_NEGATIVE: MISSED_POSITIVE_PREDICTION
  ACTUAL_CLASS: GROUND_TRUTH_CLASS
  PREDICTED_CLASS: MODEL_ASSIGNED_CLASS
  POSITIVE_CLASS: TARGET_EVENT_CLASS
  NEGATIVE_CLASS: NON_EVENT_CLASS
  SUPPORT: NUMBER_OF_TRUE_INSTANCES_OF_CLASS
  DECISION_THRESHOLD: SCORE_CUTOFF

RELATIONSHIPS:
  TRUE_LABEL -> COMPARED_WITH -> PREDICTED_LABEL
  LABEL_COMPARISON -> INCREMENTS -> CONFUSION_MATRIX_CELL
  CONFUSION_MATRIX -> DETERMINES -> ACCURACY
  CONFUSION_MATRIX -> DETERMINES -> PRECISION
  CONFUSION_MATRIX -> DETERMINES -> RECALL
  CONFUSION_MATRIX -> DETERMINES -> SPECIFICITY
  CONFUSION_MATRIX -> DETERMINES -> F1_SCORE
  DECISION_THRESHOLD -> CHANGES -> CONFUSION_MATRIX

TRUE_POSITIVE:
  condition:
    Y = POSITIVE
    AND:
      Y_hat = POSITIVE

TRUE_NEGATIVE:
  condition:
    Y = NEGATIVE
    AND:
      Y_hat = NEGATIVE

FALSE_POSITIVE:
  condition:
    Y = NEGATIVE
    AND:
      Y_hat = POSITIVE

FALSE_NEGATIVE:
  condition:
    Y = POSITIVE
    AND:
      Y_hat = NEGATIVE

TOTAL:
  N := TP + TN + FP + FN

ACTUAL_POSITIVES:
  P := TP + FN

ACTUAL_NEGATIVES:
  N_negative := TN + FP

PREDICTED_POSITIVES:
  PP := TP + FP

PREDICTED_NEGATIVES:
  PN := TN + FN

ACCURACY:
  Accuracy :=
    (TP+TN)/(TP+TN+FP+FN)

ERROR_RATE:
  ErrorRate :=
    (FP+FN)/(TP+TN+FP+FN)

  relation:
    ErrorRate := 1-Accuracy

PRECISION:
  Precision :=
    TP/(TP+FP)

  alias:
    POSITIVE_PREDICTIVE_VALUE

RECALL:
  Recall :=
    TP/(TP+FN)

  aliases:
    SENSITIVITY
    TRUE_POSITIVE_RATE

SPECIFICITY:
  Specificity :=
    TN/(TN+FP)

  alias:
    TRUE_NEGATIVE_RATE

FALSE_POSITIVE_RATE:
  FPR :=
    FP/(FP+TN)

  relation:
    FPR := 1-Specificity

FALSE_NEGATIVE_RATE:
  FNR :=
    FN/(FN+TP)

  relation:
    FNR := 1-Recall

NEGATIVE_PREDICTIVE_VALUE:
  NPV :=
    TN/(TN+FN)

F1_SCORE:
  F1 :=
    2TP/(2TP+FP+FN)

  equivalent:
    2*Precision*Recall/(Precision+Recall)

BALANCED_ACCURACY:
  BalancedAccuracy :=
    (Recall+Specificity)/2

MATTHEWS_CORRELATION_COEFFICIENT:
  alias:
    MCC

  MCC :=
    (TP*TN-FP*FN)
    /
    sqrt(
      (TP+FP)
      (TP+FN)
      (TN+FP)
      (TN+FN)
    )

  range:
    -1 <= MCC <= 1
    WHEN:
      DENOMINATOR_NONZERO

PREVALENCE:
  Prevalence :=
    (TP+FN)/N

BINARY_CONSTRUCTION:
  flow:
    FOR_EACH_OBSERVATION
      -> READ_TRUE_LABEL
      -> READ_PREDICTED_LABEL
      -> IDENTIFY_OUTCOME_TYPE
      -> INCREMENT_TP_TN_FP_OR_FN
      -> RETURN_MATRIX

SCORE_TO_MATRIX:
  score:
    S(X)

  threshold:
    t

  rule:
    IF:
      S(X) >= t
    THEN:
      PREDICT_POSITIVE
    ELSE:
      PREDICT_NEGATIVE

  consequence:
    CHANGING_THRESHOLD_CHANGES_TP_TN_FP_FN

ROC_CONNECTION:
  TPR:
    DERIVED_FROM_TP_AND_FN

  FPR:
    DERIVED_FROM_FP_AND_TN

  flow:
    CONFUSION_MATRIX_ACROSS_THRESHOLDS
      -> TPR_FPR_POINTS
      -> ROC_CURVE

PRECISION_RECALL_CONNECTION:
  precision:
    DERIVED_FROM_TP_AND_FP

  recall:
    DERIVED_FROM_TP_AND_FN

  flow:
    CONFUSION_MATRIX_ACROSS_THRESHOLDS
      -> PRECISION_RECALL_POINTS
      -> PRECISION_RECALL_CURVE

CLASS_IMBALANCE:
  condition:
    ACTUAL_CLASS_COUNTS_HIGHLY_UNEQUAL

  caution:
    RAW_ACCURACY_MAY_HIDE_POOR_MINORITY_CLASS_PERFORMANCE

  useful_metrics:
    - PRECISION
    - RECALL
    - SPECIFICITY
    - F1
    - BALANCED_ACCURACY
    - MCC

COST_MATRIX_CONNECTION:
  total_cost :=
    C_TP*TP
    + C_TN*TN
    + C_FP*FP
    + C_FN*FN

  caution:
    COUNTS_ALONE_DO_NOT_DEFINE_APPLICATION_COSTS

NORMALISATION:
  RAW:
    DISPLAY_COUNTS

  TRUE_CLASS_NORMALISED:
    NORMALISE_WITHIN_TRUE_CLASS

  PREDICTED_CLASS_NORMALISED:
    NORMALISE_WITHIN_PREDICTED_CLASS

  GLOBAL_NORMALISED:
    ALL_CELLS_SUM_TO_1

  requirement:
    MATRIX_ORIENTATION_AND_NORMALISATION_MUST_BE_EXPLICIT

MATRIX_ORIENTATION:
  common_conventions:
    - TRUE_LABELS_AS_ROWS_PREDICTED_LABELS_AS_COLUMNS
    - PREDICTED_LABELS_AS_ROWS_TRUE_LABELS_AS_COLUMNS

  caution:
    NO_SINGLE_DISPLAY_ORIENTATION_IS_UNIVERSAL

  requirement:
    AXIS_LABELS_MUST_BE_EXPLICIT

MULTICLASS_CONFUSION_MATRIX:
  classes:
    C_1,...,C_K

  matrix:
    M IN N^(KxK)

  entry:
    M_ij :=
      COUNT(
        TRUE_CLASS=C_i
        AND
        PREDICTED_CLASS=C_j
      )
    UNDER:
      TRUE_ROWS_PREDICTED_COLUMNS_CONVENTION

  diagonal:
    CORRECT_PREDICTIONS

  off_diagonal:
    MISCLASSIFICATIONS

PER_CLASS_BINARY_REDUCTION:
  for_class:
    C_k

  TP_k:
    M_kk

  FN_k:
    SUM_{j != k} M_kj

  FP_k:
    SUM_{i != k} M_ik

  TN_k:
    TOTAL-TP_k-FN_k-FP_k

MACRO_AVERAGING:
  flow:
    COMPUTE_METRIC_PER_CLASS
      -> UNWEIGHTED_MEAN

MICRO_AVERAGING:
  flow:
    AGGREGATE_PER_CLASS_COUNTS
      -> COMPUTE_GLOBAL_METRIC

WEIGHTED_AVERAGING:
  flow:
    COMPUTE_METRIC_PER_CLASS
      -> WEIGHT_BY_CLASS_SUPPORT
      -> SUM

MULTILABEL_CONFUSION_MATRIX:
  structure:
    ONE_BINARY_CONFUSION_MATRIX_PER_LABEL

  caution:
    MULTILABEL_CONFUSION_STRUCTURE != MULTICLASS_K_BY_K_MATRIX

ABSTENTION:
  classifier:
    MAY_RETURN_NO_DECISION

  extended_matrix:
    MAY_INCLUDE_ABSTAIN_CATEGORY

  metrics:
    - COVERAGE
    - SELECTIVE_RISK
    - ACCURACY_AT_COVERAGE

SAMPLE_WEIGHTS:
  observation_weight:
    w_i >= 0

  weighted_cell:
    SUM_WEIGHTS_OF_OBSERVATIONS_IN_CELL

  requirement:
    WEIGHT_DEFINITION_AND_NORMALISATION_MUST_BE_EXPLICIT

BOOTSTRAP_UNCERTAINTY:
  flow:
    RESAMPLE_EVALUATION_OBSERVATIONS
      -> RECOMPUTE_CONFUSION_MATRIX
      -> RECOMPUTE_DERIVED_METRICS
      -> FORM_EMPIRICAL_DISTRIBUTIONS

MODEL_COMPARISON:
  same_cases:
    ENABLE_PAIRED_ERROR_ANALYSIS

  methods:
    - MCNEMAR_TEST
    - BOOTSTRAP
    - PERMUTATION_TEST

MCNEMAR_TEST:
  context:
    TWO_CLASSIFIERS
    SAME_BINARY_LABELLED_CASES

  focuses_on:
    DISCORDANT_CORRECTNESS_OUTCOMES

ERROR_ANALYSIS:
  inspect:
    FALSE_POSITIVES
    FALSE_NEGATIVES
    OFF_DIAGONAL_MULTICLASS_CELLS

  purpose:
    IDENTIFY:
      SYSTEMATIC_FAILURE_MODES
      CLASS_CONFUSIONS
      DATA_QUALITY_PROBLEMS
      THRESHOLD_TRADEOFFS

TEMPORAL_MONITORING:
  flow:
    COMPUTE_MATRIX_BY_TIME_WINDOW
      -> COMPARE_CELL_RATES
      -> COMPARE_DERIVED_METRICS
      -> INVESTIGATE_CHANGES

DISTRIBUTION_SHIFT:
  IF:
    P_train(X,Y) != P_deploy(X,Y)

  THEN:
    CONFUSION_MATRIX_MAY_CHANGE

  monitor:
    - CLASS_PREVALENCE
    - ERROR_RATES
    - PRECISION
    - RECALL
    - SPECIFICITY
    - CALIBRATION

PREVALENCE_SHIFT:
  change:
    P(Y)

  effect:
    RAW_CELL_COUNTS_CHANGE
    PRECISION_AND_NPV_MAY_CHANGE

MISSING_LABELS:
  issue:
    TRUE_LABEL_NOT_AVAILABLE_FOR_ALL_PREDICTIONS

  consequence:
    OBSERVED_CONFUSION_MATRIX_MAY_NOT_REPRESENT_FULL_DEPLOYMENT_POPULATION

NOISY_LABELS:
  issue:
    OBSERVED_TRUE_LABEL_MAY_CONTAIN_ERRORS

  consequence:
    CONFUSION_MATRIX_MIXES_MODEL_ERROR_AND_LABEL_ERROR

FAIRNESS_CONNECTION:
  group_specific_confusion_matrices:
    ENABLE_COMPARISON_OF:
      TPR
      FPR
      PRECISION
      SPECIFICITY

  caution:
    GROUP_METRIC_INTERPRETATION_DEPENDS_ON_BASE_RATES_AND_CONTEXT

CAUSALITY_DISTINCTION:
  confusion_matrix:
    MEASURES_PREDICTIVE_CLASSIFICATION_OUTCOMES

  does_not:
    ESTABLISH_CAUSAL_RELATIONSHIPS

  rule:
    CLASSIFICATION_PERFORMANCE != CAUSAL_VALIDITY

APPLICATIONS:
  CONFUSION_MATRIX -> SUPPORTS -> CLASSIFIER_EVALUATION
  CONFUSION_MATRIX -> SUPPORTS -> ERROR_ANALYSIS
  CONFUSION_MATRIX -> SUPPORTS -> THRESHOLD_SELECTION
  CONFUSION_MATRIX -> SUPPORTS -> CLASS_IMBALANCE_ANALYSIS
  CONFUSION_MATRIX -> SUPPORTS -> MULTICLASS_ANALYSIS
  CONFUSION_MATRIX -> SUPPORTS -> DEPLOYMENT_MONITORING
  CONFUSION_MATRIX -> SUPPORTS -> COST_SENSITIVE_DECISION_ANALYSIS

MODEL_CONNECTIONS:
  CONFUSION_MATRIX -> CONNECTS_TO -> CLASSIFICATION_METRICS
  CONFUSION_MATRIX -> CONNECTS_TO -> PRECISION_RECALL
  CONFUSION_MATRIX -> CONNECTS_TO -> ROC_CURVE
  CONFUSION_MATRIX -> CONNECTS_TO -> PROBABILITY_CALIBRATION
  CONFUSION_MATRIX -> CONNECTS_TO -> DECISION_THEORY
  CONFUSION_MATRIX -> CONNECTS_TO -> CLASS_IMBALANCE

SEMANTIC_EQUIVALENCE:
  "confusion matrix"
    ~= "classification contingency table"

  "true positive"
    ~= "correct positive classification"

  "false positive"
    ~= "incorrect positive classification"

  "false negative"
    ~= "missed positive classification"

  "true negative"
    ~= "correct negative classification"

  EXCLUDE_EQUIVALENCE:
    FALSE_POSITIVE != FALSE_NEGATIVE
    PRECISION != RECALL
    SENSITIVITY != SPECIFICITY
    CONFUSION_MATRIX != ROC_CURVE
    CONFUSION_MATRIX != CALIBRATION_MATRIX
    MULTICLASS != MULTILABEL
    PREDICTIVE_ERROR != CAUSAL_EFFECT

VALIDATION_RULES:
  RULE_1:
    TRUE_AND_PREDICTED_LABEL_SEMANTICS_MUST_BE_EXPLICIT

  RULE_2:
    POSITIVE_CLASS_MUST_BE_EXPLICIT_FOR_BINARY_METRICS

  RULE_3:
    MATRIX_ORIENTATION_MUST_BE_EXPLICIT

  RULE_4:
    NORMALISATION_METHOD_MUST_BE_EXPLICIT

  RULE_5:
    ZERO_DENOMINATORS_IN_DERIVED_METRICS_MUST_BE_HANDLED_EXPLICITLY

  RULE_6:
    MULTICLASS_CLASS_ORDER_MUST_BE_PRESERVED

  RULE_7:
    WEIGHTED_CONFUSION_MATRICES_MUST_DOCUMENT_OBSERVATION_WEIGHTS

  RULE_8:
    FINAL_TEST_CONFUSION_MATRIX_MUST_NOT_BE_USED_REPEATEDLY_FOR_MODEL_OR_THRESHOLD_SELECTION

CONFIDENCE:
  ontology_identity: 0.99
  binary_matrix_structure: 0.99
  derived_metric_structure: 0.99
  multiclass_structure: 0.99
  threshold_relationship: 0.99
  error_analysis_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_LABELLED_EVALUATION_DATA_ARRIVES
  THEN:
    UPDATE:
      TP
      TN
      FP
      FN
      CONFUSION_MATRIX
      DERIVED_METRICS

  IF:
    DECISION_THRESHOLD_CHANGES
  THEN:
    RECOMPUTE:
      PREDICTED_LABELS
      CONFUSION_MATRIX
      PRECISION
      RECALL
      SPECIFICITY
      F1
      ERROR_RATES

  IF:
    CLASS_PREVALENCE_CHANGES
  THEN:
    REASSESS:
      RAW_CELL_COUNTS
      PRECISION
      NPV
      COSTS
      THRESHOLD

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      CONFUSION_MATRIX
      ERROR_PATTERNS
      CLASS_PREVALENCE
      CALIBRATION

  PRESERVE:
    MODEL_VERSION
    DATASET_VERSION
    LABEL_DEFINITION
    POSITIVE_CLASS
    CLASS_ORDER
    MATRIX_ORIENTATION
    NORMALISATION_METHOD
    DECISION_THRESHOLD
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_TRUE_AND_PREDICTED_LABELS
    - IDENTIFY_CLASSIFICATION_TYPE
    - IDENTIFY_POSITIVE_CLASS_IF_BINARY
    - VERIFY_CLASS_ORDER_AND_MATRIX_ORIENTATION
    - COUNT_CLASSIFICATION_OUTCOMES
    - BUILD_CONFUSION_MATRIX
    - COMPUTE_DERIVED_METRICS
    - IDENTIFY_DOMINANT_ERROR_PATTERNS
    - ASSESS_CLASS_IMBALANCE
    - ASSESS_THRESHOLD_EFFECTS
    - QUANTIFY_UNCERTAINTY_IF_REQUIRED
    - ASSESS_DISTRIBUTION_SHIFT
    - RETURN_CONFUSION_MATRIX_INTERPRETATION

  constraints:
    - PRESERVE_LABEL_SEMANTICS
    - PRESERVE_POSITIVE_CLASS_IDENTITY
    - PRESERVE_CLASS_ORDER
    - PRESERVE_MATRIX_ORIENTATION
    - DOCUMENT_NORMALISATION
    - HANDLE_ZERO_DENOMINATORS
    - DISTINGUISH_MULTICLASS_FROM_MULTILABEL
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "confusion-matrix"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - TRUE_LABELS
    - PREDICTED_LABELS
    - TP_TN_FP_FN
    - DERIVED_CLASSIFICATION_METRICS

END_FLOW