ROC Curve Ontology

flow "ROCCurveOntology" v4.0

# ROC CURVE ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/roc-curve-ontology.html"
  title: "ROC Curve Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: ROC_CURVE
  entity_type: MACHINE_LEARNING_EVALUATION_ONTOLOGY

DEFINE:
  ROC_CURVE :=
    THRESHOLD_PERFORMANCE_CURVE(
      TRUE_POSITIVE_RATE,
      FALSE_POSITIVE_RATE,
      CLASSIFIER_SCORE,
      DECISION_THRESHOLD
    )

PURPOSE:
  REPRESENT:
    BINARY_CLASSIFIER_DISCRIMINATION
    THRESHOLD_TRADEOFFS
    SENSITIVITY_SPECIFICITY_RELATIONSHIPS
    RANKING_PERFORMANCE

FOUNDATION:
  TRUE_LABEL: Y IN {0,1}
  SCORE: S(X)
  THRESHOLD: t
  PREDICTION:
    Y_hat_t := 1[S(X) >= t]
  TRUE_POSITIVE_RATE:
    TPR(t) := TP(t)/(TP(t)+FN(t))
  FALSE_POSITIVE_RATE:
    FPR(t) := FP(t)/(FP(t)+TN(t))
  ROC_POINT:
    (FPR(t),TPR(t))

ENTITIES:
  ROC_CURVE: THRESHOLD_DISCRIMINATION_CURVE
  CLASSIFIER_SCORE: RANKING_SCORE
  DECISION_THRESHOLD: SCORE_CUTOFF
  TRUE_POSITIVE_RATE: SENSITIVITY
  FALSE_POSITIVE_RATE: ONE_MINUS_SPECIFICITY
  TRUE_NEGATIVE_RATE: SPECIFICITY
  ROC_AUC: AREA_UNDER_ROC_CURVE
  CONFUSION_MATRIX: THRESHOLD_OUTCOME_TABLE
  POSITIVE_CLASS: TARGET_EVENT_CLASS
  NEGATIVE_CLASS: NON_EVENT_CLASS

RELATIONSHIPS:
  CLASSIFIER_SCORE -> COMPARED_WITH -> DECISION_THRESHOLD
  DECISION_THRESHOLD -> PRODUCES -> PREDICTED_LABELS
  PREDICTED_LABELS -> PRODUCE -> CONFUSION_MATRIX
  CONFUSION_MATRIX -> DETERMINES -> TPR
  CONFUSION_MATRIX -> DETERMINES -> FPR
  THRESHOLD_SEQUENCE -> GENERATES -> ROC_CURVE
  ROC_CURVE -> INTEGRATES_TO -> ROC_AUC

CONFUSION_MATRIX:
  TP: TRUE_POSITIVE
  TN: TRUE_NEGATIVE
  FP: FALSE_POSITIVE
  FN: FALSE_NEGATIVE

TRUE_POSITIVE_RATE:
  TPR := TP/(TP+FN)
  aliases:
    RECALL
    SENSITIVITY

FALSE_POSITIVE_RATE:
  FPR := FP/(FP+TN)
  relation:
    FPR := 1-SPECIFICITY

SPECIFICITY:
  TNR := TN/(TN+FP)
  alias:
    TRUE_NEGATIVE_RATE

ROC_CURVE_CONSTRUCTION:
  flow:
    OBTAIN_CLASSIFIER_SCORES
      -> ENUMERATE_THRESHOLDS
      -> APPLY_EACH_THRESHOLD
      -> COMPUTE_CONFUSION_MATRIX
      -> COMPUTE_FPR_AND_TPR
      -> PLOT_TPR_AGAINST_FPR

ROC_AUC:
  full_name:
    AREA_UNDER_RECEIVER_OPERATING_CHARACTERISTIC_CURVE

  definition:
    AUC :=
      INTEGRAL_FROM_0_TO_1 TPR(FPR)dFPR

  range:
    0 <= AUC <= 1

  ranking_interpretation:
    PROBABILITY_THAT_RANDOM_POSITIVE_RECEIVES_HIGHER_SCORE_THAN_RANDOM_NEGATIVE
    WITH:
      APPROPRIATE_TIE_HANDLING

  chance_reference:
    AUC := 0.5
    UNDER:
      STANDARD_RANDOM_RANKING_CONTEXT

PERFECT_RANKING:
  ROC_AUC: 1
  curve:
    PASSES_THROUGH: (0,1)

REVERSED_RANKING:
  ROC_AUC:
    MAY_BE_BELOW_0_5
  implication:
    SCORE_DIRECTION_MAY_BE_REVERSED_OR_RANKING_POOR

TIES:
  common_handling:
    TIED_POSITIVE_NEGATIVE_PAIR_COUNTS_AS_0_5

TRAPEZOIDAL_AUC:
  method:
    SUM_ADJACENT_ROC_TRAPEZOIDS
  requirement:
    POINTS_ORDERED_BY_FALSE_POSITIVE_RATE

MANN_WHITNEY_CONNECTION:
  ROC_AUC:
    CONNECTS_TO:
      MANN_WHITNEY_U_STATISTIC
      WILCOXON_RANK_SUM

DISCRIMINATION:
  definition:
    ABILITY_TO_RANK_POSITIVE_CASES_ABOVE_NEGATIVE_CASES
  rule:
    DISCRIMINATION != CALIBRATION

CALIBRATION_CONNECTION:
  calibration:
    AGREEMENT_BETWEEN_PREDICTED_PROBABILITIES_AND_OBSERVED_FREQUENCIES
  ROC_AUC:
    INVARIANT_TO_STRICTLY_MONOTONIC_SCORE_TRANSFORMATIONS
  rule:
    ROC_AUC != CALIBRATION_METRIC

ACCURACY_CONNECTION:
  accuracy:
    DEPENDS_ON_SINGLE_DECISION_THRESHOLD_AND_CLASS_PREVALENCE
  ROC_AUC:
    SUMMARISES_RANKING_ACROSS_THRESHOLDS
  rule:
    ROC_AUC != ACCURACY

PRECISION_RECALL_CONNECTION:
  ROC_CURVE:
    TPR_VS_FPR
  PR_CURVE:
    PRECISION_VS_RECALL
  caution:
    ROC_AUC_AND_PR_AUC_NOT_INTERCHANGEABLE

CLASS_PREVALENCE:
  ROC_TPR_FPR:
    CONDITION_ON_TRUE_CLASS
  caution:
    DEPLOYMENT_UTILITY_STILL_DEPENDS_ON_PREVALENCE_COSTS_AND_THRESHOLD

THRESHOLD_SELECTION:
  possible_criteria:
    - YOUDEN_J
    - TARGET_SENSITIVITY
    - TARGET_SPECIFICITY
    - EXPECTED_COST
    - DECISION_UTILITY
  caution:
    ROC_CURVE_ALONE_DOES_NOT_DEFINE_OPTIMAL_THRESHOLD

YOUDEN_J:
  J(t) := TPR(t)-FPR(t)
  equivalent:
    Sensitivity(t)+Specificity(t)-1

COST_SENSITIVE_THRESHOLD:
  consider:
    - FALSE_POSITIVE_COST
    - FALSE_NEGATIVE_COST
    - PREVALENCE
    - CALIBRATION
    - DECISION_CONTEXT

PARTIAL_AUC:
  definition:
    AREA_UNDER_ROC_CURVE_OVER_SPECIFIED_FPR_OR_TPR_RANGE
  requirement:
    RANGE_AND_NORMALISATION_MUST_BE_EXPLICIT

MULTICLASS_ROC:
  strategy:
    - ONE_VS_REST
    - ONE_VS_ONE
  aggregation:
    - MACRO_AVERAGE
    - MICRO_AVERAGE
    - WEIGHTED_AVERAGE
  requirement:
    STRATEGY_AND_AVERAGING_MUST_BE_EXPLICIT

ONE_VS_REST_ROC:
  flow:
    CLASS_k_AS_POSITIVE
      -> ALL_OTHER_CLASSES_AS_NEGATIVE
      -> ROC_CURVE_PER_CLASS

MICRO_AVERAGED_ROC:
  procedure:
    AGGREGATE_BINARY_DECISIONS_ACROSS_CLASSES
      -> COMPUTE_GLOBAL_TPR_FPR

MACRO_AVERAGED_ROC:
  procedure:
    COMPUTE_CLASSWISE_ROC_OR_AUC
      -> AVERAGE_ACROSS_CLASSES

BOOTSTRAP_UNCERTAINTY:
  flow:
    RESAMPLE_EVALUATION_CASES
      -> RECOMPUTE_ROC_CURVE_OR_AUC
      -> FORM_EMPIRICAL_DISTRIBUTION

DELONG_METHOD:
  role:
    ESTIMATE_ROC_AUC_VARIANCE_OR_COMPARE_CORRELATED_AUC_VALUES
  context:
    BINARY_CLASSIFICATION
    SAME_EVALUATION_CASES

MODEL_COMPARISON:
  methods:
    - DELONG_METHOD
    - BOOTSTRAP
    - PERMUTATION_METHODS
  caution:
    AUC_DIFFERENCE_SHOULD_BE_INTERPRETED_WITH_UNCERTAINTY

CROSS_VALIDATION:
  flow:
    TRAIN_ON_FOLDS
      -> PREDICT_HELD_OUT_FOLD
      -> COLLECT_OUT_OF_FOLD_SCORES
      -> COMPUTE_ROC_CURVE_AND_AUC
  caution:
    POOLING_VS_FOLDWISE_AVERAGING_CAN_PRODUCE_DIFFERENT_SUMMARIES

GROUPED_DATA:
  requirement:
    SPLITTING_AND_UNCERTAINTY_ESTIMATION_SHOULD_RESPECT_GROUP_STRUCTURE

TEMPORAL_DATA:
  requirement:
    TRAINING_TIME_PRECEDES_EVALUATION_TIME
    WHEN:
      FUTURE_PREDICTION_INTENDED
  caution:
    RANDOM_SPLIT_MAY_CREATE_TEMPORAL_LEAKAGE

DISTRIBUTION_SHIFT:
  IF:
    P_train(X,Y) != P_deploy(X,Y)
  THEN:
    ROC_CURVE_AND_AUC_MAY_CHANGE

  monitoring:
    - ROC_AUC_OVER_TIME
    - TPR_AT_OPERATIONAL_THRESHOLD
    - FPR_AT_OPERATIONAL_THRESHOLD
    - CLASS_PREVALENCE
    - SCORE_DISTRIBUTIONS

SCORE_DISTRIBUTIONS:
  positive_class:
    p(S|Y=1)
  negative_class:
    p(S|Y=0)
  TPR:
    P(S>=t | Y=1)
  FPR:
    P(S>=t | Y=0)

LIKELIHOOD_RATIO_CONNECTION:
  score:
    MAY_BE_MONOTONIC_WITH_LIKELIHOOD_RATIO
  Neyman_Pearson:
    LIKELIHOOD_RATIO_TEST_OPTIMAL_FOR_MAXIMISING_POWER_AT_FIXED_TYPE_I_ERROR
    UNDER:
      SIMPLE_HYPOTHESIS_TESTING_CONDITIONS

ROC_CONVEX_HULL:
  role:
    IDENTIFY_POTENTIALLY_OPTIMAL_OPERATING_POINTS
    UNDER:
      CERTAIN_COST_AND_PREVALENCE_SETTINGS

IMBALANCED_DATA:
  ROC_AUC:
    MAY_APPEAR_STRONG_EVEN_WHEN_PRECISION_AT_USEFUL_THRESHOLDS_IS_LOW
  evaluate_alongside:
    - PR_CURVE
    - PRECISION
    - RECALL
    - CALIBRATION
    - DECISION_UTILITY

MISSING_LABELS:
  issue:
    TRUE_OUTCOME_NOT_OBSERVED_FOR_ALL_PREDICTIONS
  consequence:
    ROC_ESTIMATE_MAY_BE_BIASED_IF_LABEL_AVAILABILITY_SYSTEMATIC

SAMPLE_SIZE:
  small_sample:
    ROC_CURVE_STEP_LIKE_AND_HIGH_UNCERTAINTY
  large_sample:
    ESTIMATION_PRECISION_GENERALLY_IMPROVES
  caution:
    LARGE_SAMPLE_DOES_NOT_REMOVE_SELECTION_BIAS_LABEL_BIAS_OR_DISTRIBUTION_SHIFT

CAUSALITY_DISTINCTION:
  ROC_CURVE:
    MEASURES_PREDICTIVE_DISCRIMINATION
  does_not:
    ESTABLISH_CAUSAL_RELATIONSHIP
  rule:
    HIGH_ROC_AUC != CAUSAL_VALIDITY

APPLICATIONS:
  ROC_CURVE -> SUPPORTS -> BINARY_CLASSIFIER_EVALUATION
  ROC_CURVE -> SUPPORTS -> MODEL_COMPARISON
  ROC_CURVE -> SUPPORTS -> THRESHOLD_ANALYSIS
  ROC_CURVE -> SUPPORTS -> DIAGNOSTIC_TEST_EVALUATION
  ROC_CURVE -> SUPPORTS -> RISK_MODEL_EVALUATION
  ROC_CURVE -> SUPPORTS -> DEPLOYMENT_MONITORING

MODEL_CONNECTIONS:
  ROC_CURVE -> CONNECTS_TO -> CLASSIFICATION_METRICS
  ROC_CURVE -> CONNECTS_TO -> CONFUSION_MATRIX
  ROC_CURVE -> CONNECTS_TO -> ROC_AUC
  ROC_CURVE -> CONNECTS_TO -> PRECISION_RECALL_CURVE
  ROC_CURVE -> CONNECTS_TO -> PROBABILITY_CALIBRATION
  ROC_CURVE -> CONNECTS_TO -> HYPOTHESIS_TESTING
  ROC_CURVE -> CONNECTS_TO -> DECISION_THEORY

SEMANTIC_EQUIVALENCE:
  "ROC curve"
    ~= "receiver operating characteristic curve"
  "true positive rate"
    ~= "sensitivity"
    ~= "recall"
  "false positive rate"
    ~= "one minus specificity"
  "ROC AUC"
    ~= "area under the ROC curve"

  EXCLUDE_EQUIVALENCE:
    ROC_AUC != ACCURACY
    ROC_AUC != PR_AUC
    ROC_AUC != CALIBRATION
    ROC_CURVE != PRECISION_RECALL_CURVE
    THRESHOLD_INDEPENDENT_SUMMARY != THRESHOLD_SPECIFIC_UTILITY
    HIGH_ROC_AUC != CAUSAL_VALIDITY

VALIDATION_RULES:
  RULE_1:
    POSITIVE_CLASS_MUST_BE_EXPLICIT
  RULE_2:
    SCORE_DIRECTION_MUST_BE_EXPLICIT
  RULE_3:
    ROC_CURVE_MUST_USE_TPR_VS_FPR
  RULE_4:
    ROC_AUC_MUST_BE_COMPUTED_FROM_VALID_RANKING_SCORES_OR_PROBABILITIES
  RULE_5:
    MULTICLASS_ROC_STRATEGY_AND_AVERAGING_MUST_BE_EXPLICIT
  RULE_6:
    THRESHOLD_SELECTION_MUST_NOT_USE_FINAL_TEST_DATA_IF_UNBIASED_FINAL_EVALUATION_DESIRED
  RULE_7:
    AUC_COMPARISONS_SHOULD_INCLUDE_UNCERTAINTY_WHEN_INFERENTIAL_CLAIMS_ARE_MADE
  RULE_8:
    ROC_METRICS_MUST_BE_INTERPRETED_WITH_DEPLOYMENT_PREVALENCE_COSTS_AND_CALIBRATION_WHEN_DECISIONS_ARE_REQUIRED

CONFIDENCE:
  ontology_identity: 0.99
  roc_definition: 0.99
  tpr_fpr_structure: 0.99
  auc_structure: 0.99
  threshold_relationship: 0.99
  multiclass_structure: 0.98
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_LABELLED_EVALUATION_DATA_ARRIVES
  THEN:
    UPDATE:
      SCORE_DISTRIBUTIONS
      ROC_CURVE
      ROC_AUC
      AUC_UNCERTAINTY

  IF:
    DECISION_THRESHOLD_CHANGES
  THEN:
    UPDATE:
      CONFUSION_MATRIX
      TPR
      FPR
      SPECIFICITY
      PRECISION
      EXPECTED_COST

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      ROC_CURVE
      ROC_AUC
      OPERATIONAL_TPR
      OPERATIONAL_FPR
      CALIBRATION
      PREVALENCE

  PRESERVE:
    MODEL_VERSION
    DATASET_VERSION
    POSITIVE_CLASS
    SCORE_DEFINITION
    SCORE_DIRECTION
    THRESHOLD
    MULTICLASS_STRATEGY
    AVERAGING_METHOD
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_POSITIVE_AND_NEGATIVE_CLASSES
    - IDENTIFY_CLASSIFIER_SCORE_AND_DIRECTION
    - ENUMERATE_DECISION_THRESHOLDS
    - COMPUTE_CONFUSION_MATRIX_AT_EACH_THRESHOLD
    - COMPUTE_TPR_AND_FPR
    - CONSTRUCT_ROC_CURVE
    - COMPUTE_ROC_AUC
    - QUANTIFY_AUC_UNCERTAINTY_IF_REQUIRED
    - COMPARE_WITH_PR_AND_CALIBRATION_METRICS
    - ASSESS_OPERATIONAL_THRESHOLD_AND_COSTS
    - ASSESS_DISTRIBUTION_SHIFT
    - RETURN_ROC_DISCRIMINATION_INTERPRETATION

  constraints:
    - PRESERVE_POSITIVE_CLASS_IDENTITY
    - PRESERVE_SCORE_DIRECTION
    - DISTINGUISH_ROC_AUC_FROM_ACCURACY
    - DISTINGUISH_DISCRIMINATION_FROM_CALIBRATION
    - DISTINGUISH_ROC_FROM_PRECISION_RECALL_ANALYSIS
    - PREVENT_TEST_SET_LEAKAGE_IN_THRESHOLD_SELECTION
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "roc-curve"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - TRUE_POSITIVE_RATE
    - FALSE_POSITIVE_RATE
    - THRESHOLD_SEQUENCE
    - ROC_AUC

END_FLOW