Precision-Recall Ontology

flow "PrecisionRecallOntology" v4.0

# PRECISION-RECALL ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/precision-recall-ontology.html"
  title: "Precision-Recall Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: PRECISION_RECALL
  entity_type: MACHINE_LEARNING_EVALUATION_ONTOLOGY

DEFINE:
  PRECISION_RECALL :=
    CLASSIFICATION_EVALUATION_FRAMEWORK(
      TRUE_POSITIVES,
      FALSE_POSITIVES,
      FALSE_NEGATIVES,
      PRECISION,
      RECALL,
      DECISION_THRESHOLD,
      PRECISION_RECALL_CURVE
    )

PURPOSE:
  QUANTIFY:
    POSITIVE_PREDICTION_RELIABILITY
    POSITIVE_CLASS_DETECTION
    PRECISION_RECALL_TRADEOFF
    PERFORMANCE_UNDER_CLASS_IMBALANCE

FOUNDATION:
  TRUE_LABEL: Y IN {0,1}
  CLASSIFIER_SCORE: S(X)
  THRESHOLD: t
  PREDICTION:
    Y_hat_t := 1[S(X) >= t]
  PRECISION:
    Precision(t) := TP(t)/(TP(t)+FP(t))
  RECALL:
    Recall(t) := TP(t)/(TP(t)+FN(t))

ENTITIES:
  PRECISION: POSITIVE_PREDICTIVE_VALUE
  RECALL: TRUE_POSITIVE_RATE
  SENSITIVITY: TRUE_POSITIVE_RATE
  TRUE_POSITIVE: CORRECT_POSITIVE_PREDICTION
  FALSE_POSITIVE: INCORRECT_POSITIVE_PREDICTION
  FALSE_NEGATIVE: MISSED_POSITIVE
  PRECISION_RECALL_CURVE: THRESHOLD_PERFORMANCE_CURVE
  PR_AUC: AREA_UNDER_PRECISION_RECALL_CURVE
  AVERAGE_PRECISION: RANKING_SUMMARY
  F1_SCORE: HARMONIC_PRECISION_RECALL_SUMMARY
  DECISION_THRESHOLD: SCORE_CUTOFF
  CLASS_PREVALENCE: POSITIVE_BASE_RATE

RELATIONSHIPS:
  TRUE_LABELS -> COMPARED_WITH -> PREDICTED_LABELS
  COMPARISON -> PRODUCES -> CONFUSION_MATRIX
  CONFUSION_MATRIX -> DETERMINES -> PRECISION
  CONFUSION_MATRIX -> DETERMINES -> RECALL
  DECISION_THRESHOLD -> CHANGES -> PRECISION
  DECISION_THRESHOLD -> CHANGES -> RECALL
  THRESHOLD_SEQUENCE -> GENERATES -> PRECISION_RECALL_CURVE
  CLASS_PREVALENCE -> INFLUENCES -> PRECISION

CONFUSION_MATRIX:
  TP: TRUE_POSITIVE
  TN: TRUE_NEGATIVE
  FP: FALSE_POSITIVE
  FN: FALSE_NEGATIVE

PRECISION:
  Precision := TP/(TP+FP)
  aliases:
    POSITIVE_PREDICTIVE_VALUE
  interpretation:
    FRACTION_OF_PREDICTED_POSITIVES_THAT_ARE_TRUE_POSITIVES
  undefined_case:
    TP+FP = 0
    REQUIRES_EXPLICIT_HANDLING

RECALL:
  Recall := TP/(TP+FN)
  aliases:
    SENSITIVITY
    TRUE_POSITIVE_RATE
  interpretation:
    FRACTION_OF_ACTUAL_POSITIVES_CORRECTLY_IDENTIFIED
  undefined_case:
    TP+FN = 0
    REQUIRES_EXPLICIT_HANDLING

PRECISION_RECALL_CURVE:
  x_axis: RECALL
  y_axis: PRECISION
  generated_by:
    VARYING_DECISION_THRESHOLD

THRESHOLD_CONSTRUCTION:
  flow:
    OBTAIN_CLASSIFIER_SCORES
      -> ENUMERATE_THRESHOLDS
      -> COMPUTE_PREDICTED_LABELS
      -> COMPUTE_TP_FP_FN
      -> COMPUTE_PRECISION_AND_RECALL
      -> PLOT_PRECISION_AGAINST_RECALL

PREVALENCE:
  pi := P(Y=1)
  empirical:
    (TP+FN)/N
  role:
    POSITIVE_CLASS_BASE_RATE

PR_BASELINE:
  random_ranking:
    EXPECTED_PRECISION_APPROXIMATELY_PREVALENCE
    UNDER:
      STANDARD_RANDOM_RANKING_CONTEXT
  implication:
    PR_CURVE_BASELINE_DEPENDS_ON_CLASS_PREVALENCE

CLASS_IMBALANCE:
  rare_positive_class:
    LOW_PREVALENCE
  consequence:
    FALSE_POSITIVES_CAN_STRONGLY_REDUCE_PRECISION
  relevance:
    PRECISION_RECALL_ANALYSIS_OFTEN_INFORMATIVE_FOR_RARE_EVENTS

F1_SCORE:
  F1 :=
    2*Precision*Recall/(Precision+Recall)
  equivalent:
    F1 := 2TP/(2TP+FP+FN)
  caution:
    F1_IGNORES_TRUE_NEGATIVES
    F1_DOES_NOT_ENCODE_ALL_DECISION_COSTS

F_BETA_SCORE:
  F_beta :=
    (1+beta^2)*Precision*Recall
    /
    (beta^2*Precision+Recall)
  beta > 1:
    EMPHASISE_RECALL
  beta < 1:
    EMPHASISE_PRECISION
  beta = 1:
    F1_SCORE

PR_AUC:
  full_name:
    AREA_UNDER_PRECISION_RECALL_CURVE
  role:
    SUMMARISE_PRECISION_RECALL_PERFORMANCE_ACROSS_THRESHOLDS
  caution:
    VALUE_DEPENDS_ON_INTERPOLATION_IMPLEMENTATION_AND_PREVALENCE

AVERAGE_PRECISION:
  alias:
    AP
  common_form:
    AP :=
      SUM_n
        (Recall_n-Recall_(n-1))
        * Precision_n
  caution:
    AVERAGE_PRECISION != TRAPEZOIDAL_PR_AUC_BY_DEFAULT

ROC_CONNECTION:
  ROC_CURVE:
    TPR_VS_FPR
  PR_CURVE:
    PRECISION_VS_RECALL
  relation:
    RECALL := TRUE_POSITIVE_RATE
  rule:
    PR_CURVE != ROC_CURVE
    PR_AUC != ROC_AUC

ROC_TO_PR_RELATION:
  precision :=
    (prevalence*TPR)
    /
    (
      prevalence*TPR
      + (1-prevalence)*FPR
    )
  recall := TPR

CALIBRATION_CONNECTION:
  precision_recall:
    THRESHOLD_AND_RANKING_EVALUATION
  calibration:
    PROBABILITY_RELIABILITY
  rule:
    HIGH_PR_PERFORMANCE != GOOD_CALIBRATION

THRESHOLD_SELECTION:
  criteria:
    - MINIMUM_PRECISION
    - MINIMUM_RECALL
    - MAXIMUM_F1
    - MAXIMUM_F_BETA
    - MINIMUM_EXPECTED_COST
    - APPLICATION_SPECIFIC_UTILITY
  requirement:
    SELECT_THRESHOLD_ON_VALIDATION_DATA
  caution:
    DO_NOT_SELECT_THRESHOLD_ON_FINAL_TEST_DATA

PRECISION_AT_K:
  PrecisionAtK :=
    RELEVANT_POSITIVES_IN_TOP_k/k
  applications:
    - INFORMATION_RETRIEVAL
    - RECOMMENDER_SYSTEMS
    - RANKED_CLASSIFICATION

RECALL_AT_K:
  RecallAtK :=
    RELEVANT_POSITIVES_IN_TOP_k
    /
    TOTAL_RELEVANT_POSITIVES

MULTICLASS_PRECISION_RECALL:
  strategy:
    ONE_VS_REST
  outputs:
    CLASSWISE_PRECISION
    CLASSWISE_RECALL
    CLASSWISE_PR_CURVE

MACRO_AVERAGING:
  procedure:
    COMPUTE_CLASSWISE_METRIC
      -> UNWEIGHTED_MEAN
  property:
    TREATS_CLASSES_EQUALLY

MICRO_AVERAGING:
  procedure:
    SUM_TP_FP_FN_ACROSS_CLASSES
      -> COMPUTE_PRECISION_RECALL
  property:
    WEIGHTS_BY_INSTANCE_FREQUENCY

WEIGHTED_AVERAGING:
  procedure:
    COMPUTE_CLASSWISE_METRIC
      -> WEIGHT_BY_CLASS_SUPPORT
      -> SUM
  caution:
    MAY_BE_DOMINATED_BY_COMMON_CLASSES

MULTILABEL_PRECISION_RECALL:
  property:
    OBSERVATION_MAY_HAVE_MULTIPLE_TRUE_LABELS
  evaluation:
    - MICRO_PRECISION
    - MICRO_RECALL
    - MACRO_PRECISION
    - MACRO_RECALL
    - SAMPLE_AVERAGED_PRECISION
    - SAMPLE_AVERAGED_RECALL
  caution:
    MULTILABEL != MULTICLASS

BOOTSTRAP_UNCERTAINTY:
  flow:
    RESAMPLE_EVALUATION_CASES
      -> RECOMPUTE_PRECISION_RECALL_METRICS
      -> FORM_EMPIRICAL_DISTRIBUTION
  supports:
    - CONFIDENCE_INTERVALS
    - MODEL_COMPARISON
    - METRIC_STABILITY

CROSS_VALIDATION:
  flow:
    TRAIN_ON_FOLDS
      -> GENERATE_HELD_OUT_SCORES
      -> COMPUTE_PR_METRICS
      -> AGGREGATE
  caution:
    POOLED_OUT_OF_FOLD_PR_CURVE_AND_AVERAGED_FOLD_METRICS_MAY_DIFFER

MODEL_COMPARISON:
  compare:
    - PRECISION_AT_OPERATIONAL_THRESHOLD
    - RECALL_AT_OPERATIONAL_THRESHOLD
    - F1
    - PR_AUC
    - AVERAGE_PRECISION
  caution:
    METRIC_DIFFERENCE_SHOULD_INCLUDE_UNCERTAINTY_WHEN_INFERENTIAL_CLAIMS_MADE

DISTRIBUTION_SHIFT:
  IF:
    P_train(X,Y) != P_deploy(X,Y)
  THEN:
    PRECISION_AND_RECALL_MAY_CHANGE
  monitoring:
    - CLASS_PREVALENCE
    - SCORE_DISTRIBUTION
    - PRECISION
    - RECALL
    - PR_AUC
    - CALIBRATION

PREVALENCE_SHIFT:
  IF:
    P_deploy(Y=1) != P_train(Y=1)
  THEN:
    PRECISION_MAY_CHANGE_SUBSTANTIALLY
  caution:
    RECALL_STABILITY_DEPENDS_ON_SHIFT_ASSUMPTIONS

SCORE_DISTRIBUTIONS:
  positive:
    p(S|Y=1)
  negative:
    p(S|Y=0)
  threshold:
    t
  recall:
    P(S>=t|Y=1)
  precision:
    P(Y=1|S>=t)

DECISION_THEORY_CONNECTION:
  decision:
    SHOULD_CONSIDER:
      COST_OF_FALSE_POSITIVE
      COST_OF_FALSE_NEGATIVE
      CAPACITY_CONSTRAINTS
      CLASS_PREVALENCE

CAPACITY_CONSTRAINED_SELECTION:
  relevant_metrics:
    - PRECISION_AT_K
    - RECALL_AT_K
    - LIFT
    - GAIN

LIFT:
  LiftAtK :=
    PrecisionAtK/Prevalence

GAIN:
  cumulative_gain:
    FRACTION_OF_ALL_POSITIVES_CAPTURED_BY_TOP_FRACTION_OF_RANKED_POPULATION

MISSING_LABELS:
  issue:
    TRUE_LABEL_NOT_OBSERVED_FOR_ALL_PREDICTIONS
  consequence:
    PRECISION_RECALL_ESTIMATES_MAY_BE_BIASED_IF_LABEL_OBSERVATION_SYSTEMATIC

TEMPORAL_EVALUATION:
  requirement:
    TRAINING_PERIOD_PRECEDES_EVALUATION_PERIOD
    WHEN:
      FUTURE_PREDICTION_INTENDED
  caution:
    RANDOM_SPLIT_MAY_CAUSE_TEMPORAL_LEAKAGE

GROUPED_DATA:
  requirement:
    DATA_SPLITTING_AND_UNCERTAINTY_SHOULD_RESPECT_GROUP_STRUCTURE

SELECTIVE_CLASSIFICATION:
  abstention:
    MODEL_MAY_DECLINE_LOW_CONFIDENCE_CASES
  evaluate:
    - PRECISION_AT_COVERAGE
    - RECALL_AT_COVERAGE
    - SELECTIVE_RISK

FAIRNESS_CONNECTION:
  groupwise_metrics:
    - PRECISION
    - RECALL
    - FALSE_POSITIVE_RATE
    - CALIBRATION
  caution:
    FAIRNESS_CONSTRAINTS_ARE_CONTEXT_DEPENDENT

CAUSALITY_DISTINCTION:
  PRECISION_RECALL:
    MEASURES_PREDICTIVE_CLASSIFICATION_PERFORMANCE
  does_not:
    ESTABLISH_CAUSAL_RELATIONSHIP
  rule:
    HIGH_PRECISION_OR_RECALL != CAUSAL_VALIDITY

APPLICATIONS:
  PRECISION_RECALL -> SUPPORTS -> CLASSIFIER_EVALUATION
  PRECISION_RECALL -> SUPPORTS -> RARE_EVENT_DETECTION
  PRECISION_RECALL -> SUPPORTS -> INFORMATION_RETRIEVAL
  PRECISION_RECALL -> SUPPORTS -> RECOMMENDER_SYSTEMS
  PRECISION_RECALL -> SUPPORTS -> THRESHOLD_SELECTION
  PRECISION_RECALL -> SUPPORTS -> MODEL_COMPARISON
  PRECISION_RECALL -> SUPPORTS -> DEPLOYMENT_MONITORING

MODEL_CONNECTIONS:
  PRECISION_RECALL -> CONNECTS_TO -> CLASSIFICATION_METRICS
  PRECISION_RECALL -> CONNECTS_TO -> CONFUSION_MATRIX
  PRECISION_RECALL -> CONNECTS_TO -> ROC_CURVE
  PRECISION_RECALL -> CONNECTS_TO -> PROBABILITY_CALIBRATION
  PRECISION_RECALL -> CONNECTS_TO -> CLASS_IMBALANCE
  PRECISION_RECALL -> CONNECTS_TO -> DECISION_THEORY

SEMANTIC_EQUIVALENCE:
  "precision"
    ~= "positive predictive value"
  "recall"
    ~= "sensitivity"
    ~= "true positive rate"
  "precision-recall curve"
    ~= "precision versus recall across decision thresholds"

  EXCLUDE_EQUIVALENCE:
    PRECISION != RECALL
    PRECISION != ACCURACY
    RECALL != SPECIFICITY
    PR_AUC != ROC_AUC
    AVERAGE_PRECISION != TRAPEZOIDAL_PR_AUC_BY_DEFAULT
    PRECISION_RECALL != CALIBRATION
    HIGH_PRECISION_RECALL != CAUSAL_VALIDITY

VALIDATION_RULES:
  RULE_1:
    POSITIVE_CLASS_MUST_BE_EXPLICIT
  RULE_2:
    PRECISION_AND_RECALL_DENOMINATORS_MUST_BE_CHECKED_FOR_ZERO
  RULE_3:
    CLASS_PREVALENCE_MUST_BE_RECORDED_WHEN_COMPARING_PR_METRICS_ACROSS_DATASETS
  RULE_4:
    DECISION_THRESHOLD_MUST_BE_PRESERVED_FOR_THRESHOLD_SPECIFIC_METRICS
  RULE_5:
    PR_AUC_OR_AVERAGE_PRECISION_IMPLEMENTATION_MUST_BE_DOCUMENTED
  RULE_6:
    MULTICLASS_OR_MULTILABEL_AVERAGING_METHOD_MUST_BE_EXPLICIT
  RULE_7:
    THRESHOLD_SELECTION_MUST_NOT_USE_FINAL_TEST_DATA_IF_UNBIASED_FINAL_EVALUATION_DESIRED
  RULE_8:
    PRECISION_RECALL_INTERPRETATION_MUST_CONSIDER_DEPLOYMENT_PREVALENCE_AND_DECISION_COSTS

CONFIDENCE:
  ontology_identity: 0.99
  precision_definition: 0.99
  recall_definition: 0.99
  pr_curve_structure: 0.99
  f_score_structure: 0.99
  prevalence_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_LABELLED_EVALUATION_DATA_ARRIVES
  THEN:
    UPDATE:
      CONFUSION_MATRIX
      PRECISION
      RECALL
      F1
      PR_CURVE
      PR_AUC
      AVERAGE_PRECISION

  IF:
    DECISION_THRESHOLD_CHANGES
  THEN:
    RECOMPUTE:
      TP
      FP
      FN
      PRECISION
      RECALL
      F_SCORE

  IF:
    CLASS_PREVALENCE_CHANGES
  THEN:
    REASSESS:
      PRECISION
      PR_BASELINE
      PR_AUC_INTERPRETATION
      DECISION_THRESHOLD

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      PRECISION
      RECALL
      PR_CURVE
      PR_AUC
      CALIBRATION
      PREVALENCE

  PRESERVE:
    MODEL_VERSION
    DATASET_VERSION
    POSITIVE_CLASS
    SCORE_DEFINITION
    SCORE_DIRECTION
    DECISION_THRESHOLD
    CLASS_PREVALENCE
    AVERAGING_METHOD
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_POSITIVE_CLASS
    - IDENTIFY_TRUE_LABELS_AND_CLASSIFIER_SCORES
    - IDENTIFY_SCORE_DIRECTION
    - ENUMERATE_DECISION_THRESHOLDS
    - COMPUTE_TP_FP_FN
    - COMPUTE_PRECISION_AND_RECALL
    - CONSTRUCT_PRECISION_RECALL_CURVE
    - COMPUTE_F1_F_BETA_PR_AUC_OR_AVERAGE_PRECISION_AS_REQUIRED
    - ACCOUNT_FOR_CLASS_PREVALENCE
    - QUANTIFY_METRIC_UNCERTAINTY_IF_REQUIRED
    - ASSESS_OPERATIONAL_THRESHOLD_AND_DECISION_COSTS
    - ASSESS_DISTRIBUTION_SHIFT
    - RETURN_PRECISION_RECALL_INTERPRETATION

  constraints:
    - PRESERVE_POSITIVE_CLASS_IDENTITY
    - PRESERVE_SCORE_DIRECTION
    - PRESERVE_DECISION_THRESHOLD
    - DISTINGUISH_PRECISION_FROM_RECALL
    - DISTINGUISH_PR_AUC_FROM_ROC_AUC
    - DISTINGUISH_RANKING_FROM_CALIBRATION
    - PREVENT_TEST_SET_LEAKAGE_IN_THRESHOLD_SELECTION
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "precision-recall"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - PRECISION
    - RECALL
    - THRESHOLD_SEQUENCE
    - PRECISION_RECALL_CURVE

END_FLOW