Cross Validation Ontology

flow "CrossValidationOntology" v4.0

# CROSS-VALIDATION ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/cross-validation-ontology.html"
  title: "Cross-Validation Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: CROSS_VALIDATION
  entity_type: MACHINE_LEARNING_EVALUATION_ONTOLOGY

DEFINE:
  CROSS_VALIDATION :=
    RESAMPLING_EVALUATION_FRAMEWORK(
      DATASET,
      TRAINING_FOLDS,
      VALIDATION_FOLDS,
      MODEL_FITTING,
      HELD_OUT_PREDICTIONS,
      PERFORMANCE_AGGREGATION
    )

PURPOSE:
  ESTIMATE:
    OUT_OF_SAMPLE_PERFORMANCE
    GENERALISATION_ERROR
    MODEL_SELECTION_PERFORMANCE
    HYPERPARAMETER_PERFORMANCE

FOUNDATION:
  DATASET: D := {z_1,...,z_n}
  FOLDS: D := F_1 UNION ... UNION F_K
  VALIDATION: V_k := F_k
  TRAINING: T_k := D MINUS F_k
  FOLD_SCORE: s_k := Metric(Model(T_k),V_k)
  CV_SCORE: CV := (1/K)*SUM[k=1..K] s_k

ENTITIES:
  CROSS_VALIDATION: RESAMPLING_EVALUATION_METHOD
  FOLD: DATA_PARTITION
  TRAINING_FOLD: MODEL_FITTING_PARTITION
  VALIDATION_FOLD: HELD_OUT_EVALUATION_PARTITION
  OUT_OF_FOLD_PREDICTION: PREDICTION_FROM_MODEL_NOT_TRAINED_ON_CASE
  K_FOLD_CV: K_PARTITION_CROSS_VALIDATION
  STRATIFIED_CV: CLASS_BALANCED_PARTITIONING_METHOD
  GROUP_CV: GROUP_PRESERVING_PARTITIONING_METHOD
  TIME_SERIES_CV: TEMPORALLY_ORDERED_VALIDATION_METHOD
  NESTED_CV: MODEL_SELECTION_AND_EVALUATION_FRAMEWORK
  LEAVE_ONE_OUT_CV: SINGLE_CASE_VALIDATION_METHOD

RELATIONSHIPS:
  DATASET -> PARTITIONED_INTO -> FOLDS
  TRAINING_FOLDS -> FIT -> MODEL
  VALIDATION_FOLD -> EVALUATES -> MODEL
  FOLD -> PRODUCES -> FOLD_SCORE
  FOLD_SCORES -> AGGREGATE_TO -> CV_SCORE
  OUT_OF_FOLD_PREDICTIONS -> SUPPORT -> GENERALISATION_ESTIMATION
  CROSS_VALIDATION -> SUPPORTS -> MODEL_SELECTION
  CROSS_VALIDATION -> SUPPORTS -> HYPERPARAMETER_SELECTION

K_FOLD_CROSS_VALIDATION:
  flow:
    PARTITION_D_INTO_K_FOLDS
      -> HOLD_OUT_ONE_FOLD
      -> TRAIN_ON_REMAINING_FOLDS
      -> PREDICT_HELD_OUT_FOLD
      -> COMPUTE_SCORE
      -> ROTATE
      -> AGGREGATE

OUT_OF_FOLD_PREDICTION:
  constraint:
    OBSERVATION_MUST_NOT_BE_USED_TO_FIT_MODEL_THAT_PREDICTS_IT

STRATIFIED_K_FOLD:
  objective:
    APPROXIMATELY_PRESERVE_CLASS_PROPORTIONS_ACROSS_FOLDS
  caution:
    DOES_NOT_FIX_GROUP_OR_TEMPORAL_LEAKAGE

REPEATED_K_FOLD:
  flow:
    REPEAT_K_FOLD_WITH_DIFFERENT_PARTITIONS
      -> COLLECT_SCORES
      -> SUMMARISE_PERFORMANCE

LEAVE_ONE_OUT_CV:
  alias: LOOCV
  K: n
  per_fold:
    ONE_OBSERVATION_VALIDATION
    n-1_OBSERVATIONS_TRAINING

HOLDOUT_VALIDATION:
  distinction:
    HOLDOUT_VALIDATION != K_FOLD_CROSS_VALIDATION

GROUP_K_FOLD:
  constraint:
    SAME_GROUP_MUST_NOT_APPEAR_IN_BOTH_TRAINING_AND_VALIDATION_WITHIN_A_FOLD
  purpose:
    PREVENT_GROUP_LEAKAGE

TIME_SERIES_CROSS_VALIDATION:
  constraint:
    TRAINING_TIME_MUST_PRECEDE_VALIDATION_TIME
  methods:
    - EXPANDING_WINDOW
    - ROLLING_WINDOW
    - BLOCKED_TIME_SPLIT
  rule:
    RANDOM_K_FOLD_GENERALLY_INAPPROPRIATE_WHEN_IT_EXPOSES_FUTURE_INFORMATION

EXPANDING_WINDOW:
  flow:
    INITIAL_TRAINING_WINDOW
      -> VALIDATE_ON_NEXT_PERIOD
      -> EXPAND_TRAINING_WINDOW
      -> REPEAT

ROLLING_WINDOW:
  flow:
    FIX_OR_LIMIT_TRAINING_WINDOW
      -> MOVE_WINDOWS_FORWARD

GAP_OR_EMBARGO:
  purpose:
    REDUCE_TEMPORAL_INFORMATION_BLEED

NESTED_CROSS_VALIDATION:
  outer_loop:
    ESTIMATE_GENERALISATION_PERFORMANCE
  inner_loop:
    SELECT_MODEL_HYPERPARAMETERS_FEATURES_AND_PIPELINE
  flow:
    OUTER_SPLIT
      -> INNER_CV_ON_OUTER_TRAIN
      -> SELECT_CONFIGURATION
      -> REFIT_ON_OUTER_TRAIN
      -> EVALUATE_ON_OUTER_TEST
      -> REPEAT
      -> AGGREGATE
  rule:
    OUTER_TEST_FOLD_MUST_NOT_INFLUENCE_INNER_MODEL_SELECTION

HYPERPARAMETER_TUNING:
  methods:
    - GRID_SEARCH
    - RANDOM_SEARCH
    - BAYESIAN_OPTIMISATION
    - SUCCESSIVE_HALVING
  flow:
    CANDIDATE_CONFIGURATION
      -> CROSS_VALIDATE
      -> SCORE
      -> SELECT
      -> REFIT

MODEL_SELECTION:
  rule:
    COMPARE_CANDIDATES_UNDER_SAME_APPROPRIATE_SPLITTING_PROTOCOL
  caution:
    FINAL_TEST_DATA_MUST_REMAIN_SEPARATE_IF_INDEPENDENT_FINAL_EVALUATION_REQUIRED

PIPELINE_CROSS_VALIDATION:
  pipeline:
    PREPROCESSING
      -> FEATURE_ENGINEERING
      -> FEATURE_SELECTION
      -> MODEL_FITTING
  rule:
    EVERY_DATA_DEPENDENT_STEP_MUST_BE_FIT_USING_TRAINING_PARTITION_ONLY

DATA_LEAKAGE:
  examples:
    - SCALING_BEFORE_SPLITTING
    - IMPUTATION_USING_FULL_DATA
    - FEATURE_SELECTION_USING_ALL_LABELS
    - TARGET_ENCODING_USING_VALIDATION_LABELS
    - DUPLICATE_ENTITIES_ACROSS_FOLDS
    - FUTURE_INFORMATION_IN_TIME_SERIES
  consequence:
    OPTIMISTIC_PERFORMANCE_ESTIMATE

SCALING:
  rule:
    FIT_SCALER_ON_TRAINING_FOLD
    APPLY_TO_VALIDATION_FOLD

IMPUTATION:
  rule:
    FIT_IMPUTER_ON_TRAINING_FOLD
    APPLY_TO_VALIDATION_FOLD

FEATURE_SELECTION:
  rule:
    SELECT_FEATURES_USING_TRAINING_FOLD_ONLY

TARGET_ENCODING:
  requirement:
    EXCLUDE_HELD_OUT_TARGET_INFORMATION

OVERSAMPLING:
  examples:
    - RANDOM_OVERSAMPLING
    - SMOTE
  rule:
    APPLY_WITHIN_TRAINING_FOLD_ONLY

UNDERSAMPLING:
  rule:
    APPLY_WITHIN_TRAINING_FOLD_ONLY

METRIC_SELECTION:
  classification:
    - ACCURACY
    - PRECISION
    - RECALL
    - F1
    - ROC_AUC
    - PR_AUC
    - LOG_LOSS
  regression:
    - MAE
    - MSE
    - RMSE
    - R_SQUARED
    - PINBALL_LOSS
    - NEGATIVE_LOG_LIKELIHOOD
  requirement:
    METRIC_MUST_MATCH_TASK_AND_DECISION_CONTEXT

LOSS_VS_EVALUATION_METRIC:
  training_loss:
    OPTIMISED_DURING_MODEL_FITTING
  evaluation_metric:
    USED_TO_ASSESS_HELD_OUT_PERFORMANCE
  rule:
    TRAINING_LOSS != EVALUATION_METRIC_BY_DEFAULT

CLASS_IMBALANCE:
  recommendation:
    CONSIDER_STRATIFICATION_CLASSWISE_AND_BALANCED_METRICS
  caution:
    STRATIFICATION_SHOULD_NOT_BREAK_GROUP_OR_TEMPORAL_CONSTRAINTS

RARE_CLASSES:
  issue:
    SOME_FOLDS_MAY_CONTAIN_TOO_FEW_POSITIVE_CASES
  consequence:
    METRICS_MAY_BE_UNDEFINED_OR_UNSTABLE

CHOOSING_K:
  common_values:
    5
    10
  rule:
    NO_SINGLE_K_IS_UNIVERSALLY_OPTIMAL

OUT_OF_FOLD_PREDICTIONS:
  property:
    EACH_OBSERVATION_RECEIVES_PREDICTION_FROM_MODEL_NOT_TRAINED_ON_THAT_OBSERVATION
  uses:
    - STACKING
    - CALIBRATION
    - ERROR_ANALYSIS
    - AGGREGATED_METRIC_COMPUTATION

STACKING_CONNECTION:
  base_models:
    GENERATE_OUT_OF_FOLD_PREDICTIONS
  meta_model:
    TRAINED_ON_OUT_OF_FOLD_BASE_MODEL_OUTPUTS
  requirement:
    META_FEATURES_MUST_AVOID_IN_SAMPLE_LEAKAGE

CALIBRATION_CONNECTION:
  flow:
    GENERATE_OUT_OF_FOLD_PROBABILITIES
      -> FIT_CALIBRATOR
      -> EVALUATE_ON_VALIDLY_HELD_OUT_DATA

BOOTSTRAP_DISTINCTION:
  CROSS_VALIDATION:
    PARTITIONS_DATA_INTO_TRAIN_AND_HELD_OUT_SUBSETS
  BOOTSTRAP:
    RESAMPLES_WITH_REPLACEMENT
  rule:
    CROSS_VALIDATION != BOOTSTRAP

MONTE_CARLO_CV:
  alias:
    REPEATED_RANDOM_SUBSAMPLING
  flow:
    RANDOM_TRAIN_VALIDATION_SPLIT
      -> FIT
      -> EVALUATE
      -> REPEAT
  distinction:
    NOT_IDENTICAL_TO_STANDARD_K_FOLD

SPATIAL_CROSS_VALIDATION:
  strategies:
    - SPATIAL_BLOCKS
    - LOCATION_GROUPS
    - DISTANCE_SEPARATION
  purpose:
    REDUCE_SPATIAL_LEAKAGE

DOMAIN_SHIFT_VALIDATION:
  objective:
    ESTIMATE_GENERALISATION_TO_NEW_DOMAIN_SITE_OR_POPULATION
  strategies:
    - LEAVE_ONE_DOMAIN_OUT
    - GROUP_BY_DOMAIN

LEAVE_ONE_GROUP_OUT:
  flow:
    HOLD_OUT_ONE_GROUP
      -> TRAIN_ON_OTHER_GROUPS
      -> EVALUATE_HELD_OUT_GROUP
      -> ROTATE

PERFORMANCE_AGGREGATION:
  options:
    - MEAN_OF_FOLD_METRICS
    - WEIGHTED_MEAN_OF_FOLD_METRICS
    - METRIC_ON_POOLED_OUT_OF_FOLD_PREDICTIONS
  caution:
    AGGREGATION_OPTIONS_MAY_NOT_BE_NUMERICALLY_EQUIVALENT
  requirement:
    AGGREGATION_METHOD_MUST_BE_DOCUMENTED

UNCERTAINTY:
  sources:
    - FINITE_SAMPLE_VARIABILITY
    - FOLD_ASSIGNMENT_VARIABILITY
    - MODEL_FITTING_RANDOMNESS
    - DATA_DEPENDENCE
  methods:
    - REPEATED_CV
    - BOOTSTRAP_OF_APPROPRIATE_UNITS
    - NESTED_CV
  caution:
    FOLD_SCORES_SHOULD_NOT_BE_TREATED_AS_SIMPLE_INDEPENDENT_REPLICATES

DISTRIBUTION_SHIFT:
  IF:
    TRAINING_DISTRIBUTION != DEPLOYMENT_DISTRIBUTION
  THEN:
    RANDOM_CV_ESTIMATE_MAY_NOT_REPRESENT_DEPLOYMENT_PERFORMANCE
  response:
    DESIGN_SPLITS_TO_MIMIC_TARGET_DEPLOYMENT_CONDITIONS

REPRODUCIBILITY:
  preserve:
    - DATASET_VERSION
    - FOLD_ASSIGNMENTS
    - SPLITTING_STRATEGY
    - RANDOM_SEED
    - PREPROCESSING_PIPELINE
    - MODEL_CONFIGURATION
    - METRIC_DEFINITION
    - SOFTWARE_ENVIRONMENT

APPLICATIONS:
  CROSS_VALIDATION -> SUPPORTS -> GENERALISATION_ESTIMATION
  CROSS_VALIDATION -> SUPPORTS -> MODEL_SELECTION
  CROSS_VALIDATION -> SUPPORTS -> HYPERPARAMETER_TUNING
  CROSS_VALIDATION -> SUPPORTS -> FEATURE_SELECTION
  CROSS_VALIDATION -> SUPPORTS -> PROBABILITY_CALIBRATION
  CROSS_VALIDATION -> SUPPORTS -> STACKING
  CROSS_VALIDATION -> SUPPORTS -> PIPELINE_VALIDATION

MODEL_CONNECTIONS:
  CROSS_VALIDATION -> CONNECTS_TO -> MACHINE_LEARNING
  CROSS_VALIDATION -> CONNECTS_TO -> STATISTICAL_LEARNING
  CROSS_VALIDATION -> CONNECTS_TO -> CLASSIFICATION_METRICS
  CROSS_VALIDATION -> CONNECTS_TO -> REGRESSION_METRICS
  CROSS_VALIDATION -> CONNECTS_TO -> PROBABILITY_CALIBRATION
  CROSS_VALIDATION -> CONNECTS_TO -> BOOTSTRAP
  CROSS_VALIDATION -> CONNECTS_TO -> MODEL_SELECTION

SEMANTIC_EQUIVALENCE:
  "cross-validation"
    ~= "repeated held-out evaluation across data partitions"

  "out-of-fold prediction"
    ~= "prediction from a model not trained on that observation"

  "leave-one-out cross-validation"
    ~= "K-fold cross-validation with K equal to sample size"

  EXCLUDE_EQUIVALENCE:
    CROSS_VALIDATION != BOOTSTRAP
    CROSS_VALIDATION != SINGLE_HOLDOUT
    VALIDATION_FOLD != TEST_SET_BY_DEFAULT
    TRAINING_SCORE != VALIDATION_SCORE
    RANDOM_K_FOLD != TIME_SERIES_CV
    GROUP_K_FOLD != STRATIFIED_K_FOLD
    INNER_CV_SCORE != OUTER_GENERALISATION_SCORE

VALIDATION_RULES:
  RULE_1:
    SPLITTING_STRATEGY_MUST_MATCH_DATA_DEPENDENCE_AND_DEPLOYMENT_CONTEXT
  RULE_2:
    VALIDATION_OBSERVATIONS_MUST_NOT_INFLUENCE_FOLD_SPECIFIC_MODEL_FITTING
  RULE_3:
    ALL_DATA_DEPENDENT_PREPROCESSING_MUST_OCCUR_WITHIN_TRAINING_FOLDS
  RULE_4:
    GROUP_TEMPORAL_OR_SPATIAL_DEPENDENCE_MUST_BE_PRESERVED_WHEN_RELEVANT
  RULE_5:
    METRIC_AND_AGGREGATION_METHOD_MUST_BE_EXPLICIT
  RULE_6:
    HYPERPARAMETER_SELECTION_MUST_BE_SEPARATED_FROM_FINAL_EVALUATION_WHEN_REQUIRED
  RULE_7:
    RANDOM_SEEDS_AND_FOLD_ASSIGNMENTS_SHOULD_BE_RECORDED
  RULE_8:
    FINAL_TEST_DATA_MUST_REMAIN_UNTOUCHED_UNTIL_FINAL_EVALUATION_WHEN_USED

CONFIDENCE:
  ontology_identity: 0.99
  k_fold_structure: 0.99
  nested_cv_structure: 0.99
  leakage_prevention_structure: 0.99
  temporal_group_structure: 0.99
  generalisation_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    DATASET_CHANGES
  THEN:
    UPDATE:
      DATASET_VERSION
      FOLD_ASSIGNMENTS
      CLASS_DISTRIBUTIONS
      GROUP_DISTRIBUTIONS
    RERUN:
      CROSS_VALIDATION

  IF:
    SPLITTING_STRATEGY_CHANGES
  THEN:
    REGENERATE:
      FOLDS
    RECOMPUTE:
      OUT_OF_FOLD_PREDICTIONS
      CV_METRICS

  IF:
    MODEL_OR_HYPERPARAMETER_SPACE_CHANGES
  THEN:
    RERUN:
      MODEL_SELECTION_PROTOCOL
    REASSESS:
      GENERALISATION_ESTIMATE

  PRESERVE:
    DATASET_VERSION
    FOLD_ASSIGNMENTS
    SPLITTING_STRATEGY
    RANDOM_SEED
    PREPROCESSING_PIPELINE
    MODEL_CONFIGURATION
    HYPERPARAMETER_SEARCH_SPACE
    METRIC_DEFINITION
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DATASET_STRUCTURE
    - IDENTIFY_DEPENDENCE_GROUP_TEMPORAL_OR_SPATIAL_CONSTRAINTS
    - SELECT_APPROPRIATE_CV_STRATEGY
    - GENERATE_FOLDS
    - FIT_DATA_DEPENDENT_PIPELINE_STEPS_WITHIN_TRAINING_FOLDS
    - GENERATE_HELD_OUT_PREDICTIONS
    - COMPUTE_FOLD_METRICS
    - AGGREGATE_PERFORMANCE
    - QUANTIFY_VARIABILITY_IF_REQUIRED
    - APPLY_NESTED_CV_FOR_MODEL_SELECTION_IF_REQUIRED
    - ASSESS_DISTRIBUTION_SHIFT_RELEVANCE
    - RETURN_GENERALISATION_INTERPRETATION

  constraints:
    - PRESERVE_TRAIN_VALIDATION_SEPARATION
    - PREVENT_DATA_LEAKAGE
    - PRESERVE_GROUP_TEMPORAL_AND_SPATIAL_BOUNDARIES
    - DISTINGUISH_INNER_FROM_OUTER_CV
    - DOCUMENT_METRIC_AGGREGATION
    - PRESERVE_FOLD_ASSIGNMENTS_AND_RANDOM_SEEDS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "cross-validation"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - TRAINING_VALIDATION_FOLDS
    - OUT_OF_FOLD_PREDICTIONS
    - GENERALISATION_ESTIMATION

END_FLOW