Statistical Validation Ontology

flow "StatisticalValidationOntology" v4.0

# STATISTICAL VALIDATION ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/scientific-statistical-methods/statistical-validation-ontology.html"
  title: "Statistical Validation Ontology"
  domain: "Probability and Statistics"
  category: "Scientific Statistical Methods"
  entity: STATISTICAL_VALIDATION
  entity_type: SCIENTIFIC_STATISTICAL_ONTOLOGY

DEFINE:
  STATISTICAL_VALIDATION :=
    FORMAL_EVIDENCE_FRAMEWORK(
      DATA,
      MODEL,
      ASSUMPTIONS,
      PERFORMANCE_METRICS,
      RESAMPLING,
      HOLDOUT_EVALUATION,
      CALIBRATION,
      ROBUSTNESS,
      GENERALISATION,
      REPRODUCIBILITY
    )

PURPOSE:
  ASSESS:
    MODEL_ADEQUACY
    ESTIMATOR_RELIABILITY
    PREDICTIVE_PERFORMANCE
    ASSUMPTION_VALIDITY
    CALIBRATION
    GENERALISABILITY
    ROBUSTNESS
    REPRODUCIBILITY

FOUNDATION:
  DATA: D
  MODEL: M(theta)
  FIT:
    theta_hat := FIT(M,D_train)
  VALIDATION:
    V := EVALUATE(MODEL,HELD_OUT_OR_RESAMPLED_DATA,METRICS)

ENTITIES:
  STATISTICAL_VALIDATION: ASSESSMENT_OF_STATISTICAL_MODEL_OR_METHOD
  TRAINING_DATA: DATA_USED_TO_ESTIMATE_MODEL
  VALIDATION_DATA: DATA_USED_TO_TUNE_OR_COMPARE_MODEL
  TEST_DATA: DATA_RESERVED_FOR_FINAL_GENERALISATION_ASSESSMENT
  PERFORMANCE_METRIC: QUANTITATIVE_EVALUATION_CRITERION
  CALIBRATION: AGREEMENT_BETWEEN_PREDICTED_AND_OBSERVED_VALUES
  DISCRIMINATION: ABILITY_TO_SEPARATE_OR_RANK_OUTCOMES
  RESAMPLING: REPEATED_DATA_PARTITION_OR_RECONSTRUCTION
  CROSS_VALIDATION: RESAMPLING_BASED_GENERALISATION_ESTIMATION
  ROBUSTNESS: STABILITY_TO_PLAUSIBLE_CHANGES
  GENERALISATION: PERFORMANCE_ON_NEW_TARGET_DATA
  EXTERNAL_VALIDATION: EVALUATION_ON_INDEPENDENT_EXTERNAL_DATA

RELATIONSHIPS:
  MODEL -> FIT_ON -> TRAINING_DATA
  MODEL -> TUNED_USING -> VALIDATION_DATA
  MODEL -> FINALLY_EVALUATED_ON -> TEST_DATA
  PREDICTIONS -> SCORED_BY -> PERFORMANCE_METRIC
  RESIDUALS -> SUPPORT -> MODEL_DIAGNOSTICS
  CROSS_VALIDATION -> ESTIMATES -> OUT_OF_SAMPLE_PERFORMANCE
  CALIBRATION -> ASSESSES -> PROBABILITY_ACCURACY
  EXTERNAL_VALIDATION -> ASSESSES -> TRANSPORTABILITY
  SENSITIVITY_ANALYSIS -> ASSESSES -> ROBUSTNESS

VALIDATION_FLOW:
  flow:
    DEFINE_TARGET_USE
      -> DEFINE_TARGET_POPULATION
      -> DEFINE_PERFORMANCE_CRITERIA
      -> SEPARATE_TRAINING_AND_EVALUATION_INFORMATION
      -> FIT_MODEL
      -> CHECK_ASSUMPTIONS
      -> GENERATE_PREDICTIONS
      -> COMPUTE_VALIDATION_METRICS
      -> ASSESS_CALIBRATION
      -> ASSESS_ROBUSTNESS
      -> ASSESS_GENERALISATION
      -> DOCUMENT_LIMITATIONS

INTERNAL_VALIDATION:
  methods:
    - CROSS_VALIDATION
    - BOOTSTRAP
    - HOLDOUT_SPLIT
    - REPEATED_SPLIT
  purpose:
    ESTIMATE_OPTIMISM_AND_EXPECTED_GENERALISATION_WITHIN_DEVELOPMENT_DOMAIN

EXTERNAL_VALIDATION:
  dimensions:
    - TEMPORAL
    - GEOGRAPHIC
    - INSTITUTIONAL
    - POPULATION
    - DEVICE_OR_SYSTEM
  purpose:
    ASSESS_TRANSPORTABILITY_AND_GENERALISATION

TRAIN_VALIDATION_TEST_SPLIT:
  training:
    FIT_PARAMETERS
  validation:
    SELECT_MODEL
    TUNE_HYPERPARAMETERS
    SELECT_THRESHOLD
  test:
    FINAL_GENERALISATION_ASSESSMENT
  rule:
    TEST_DATA_MUST_NOT_INFLUENCE_MODEL_DEVELOPMENT

DATA_LEAKAGE:
  examples:
    - GLOBAL_PREPROCESSING_BEFORE_SPLIT
    - FEATURE_SELECTION_USING_TEST_DATA
    - TARGET_LEAKAGE
    - DUPLICATE_RECORDS_ACROSS_SPLITS
    - FUTURE_INFORMATION_IN_TIME_SERIES
  consequence:
    OPTIMISTIC_VALIDATION_PERFORMANCE
  rule:
    PREPROCESSING_MUST_BE_FIT_INSIDE_TRAINING_PARTITION

HOLDOUT_VALIDATION:
  requirement:
    D_train INTERSECTION D_test = EMPTY
    AT:
      INDEPENDENT_UNIT_LEVEL
  caution:
    RANDOM_ROW_SPLIT_MAY_BE_INVALID_FOR_CLUSTERED_OR_REPEATED_DATA

K_FOLD_CROSS_VALIDATION:
  folds:
    K
  flow:
    PARTITION_DATA_INTO_K_FOLDS
      -> HOLD_OUT_ONE_FOLD
      -> FIT_ON_REMAINING_FOLDS
      -> SCORE_HELD_OUT_FOLD
      -> REPEAT
      -> AGGREGATE_SCORES

STRATIFIED_CROSS_VALIDATION:
  purpose:
    PRESERVE_APPROXIMATE_CLASS_PROPORTIONS_ACROSS_FOLDS

GROUPED_CROSS_VALIDATION:
  grouping:
    SUBJECT
    SITE
    CLUSTER
    ENTITY
  requirement:
    ALL_OBSERVATIONS_FROM_GROUP_REMAIN_IN_SAME_FOLD

TIME_SERIES_CROSS_VALIDATION:
  requirement:
    TRAINING_TIME_PRECEDES_VALIDATION_TIME
  methods:
    - ROLLING_ORIGIN
    - EXPANDING_WINDOW
    - SLIDING_WINDOW
  rule:
    FUTURE_DATA_MUST_NOT_PREDICT_PAST_DATA

NESTED_CROSS_VALIDATION:
  outer_loop:
    ESTIMATE_GENERALISATION
  inner_loop:
    HYPERPARAMETER_AND_MODEL_SELECTION
  purpose:
    AVOID_OPTIMISTIC_BIAS_FROM_TUNING_ON_EVALUATION_FOLDS

BOOTSTRAP_VALIDATION:
  flow:
    SAMPLE_WITH_REPLACEMENT
      -> FIT_MODEL
      -> EVALUATE
      -> REPEAT
  uses:
    - OPTIMISM_CORRECTION
    - PERFORMANCE_UNCERTAINTY
    - ESTIMATOR_STABILITY
  caution:
    BOOTSTRAP_SCHEME_MUST_RESPECT_DEPENDENCE_STRUCTURE

MODEL_ASSUMPTION_VALIDATION:
  inspect:
    - LINEARITY
    - DISTRIBUTIONAL_FORM
    - HOMOSCEDASTICITY
    - INDEPENDENCE
    - LINK_FUNCTION
    - STATIONARITY
    - PROPORTIONAL_HAZARDS
    - RANDOM_EFFECT_STRUCTURE

RESIDUAL_VALIDATION:
  residual:
    e_i := y_i-y_hat_i
  inspect:
    - RESIDUAL_VS_FITTED
    - Q_Q_PLOT
    - SCALE_LOCATION
    - AUTOCORRELATION
    - INFLUENCE
    - OUTLIERS

GOODNESS_OF_FIT:
  methods:
    - DEVIANCE
    - CHI_SQUARE
    - RESIDUAL_SUMMARY
    - POSTERIOR_PREDICTIVE_CHECK
  caution:
    GOOD_IN_SAMPLE_FIT != GOOD_OUT_OF_SAMPLE_PERFORMANCE

PREDICTIVE_VALIDATION:
  requirement:
    EVALUATE_ON_DATA_NOT_USED_TO_FIT_OR_TUNE_MODEL
  dimensions:
    - ACCURACY
    - CALIBRATION
    - DISCRIMINATION
    - SHARPNESS
    - ROBUSTNESS

REGRESSION_METRICS:
  metrics:
    - MAE
    - MSE
    - RMSE
    - R_SQUARED
    - LOG_SCORE
    - PREDICTION_INTERVAL_COVERAGE

CLASSIFICATION_METRICS:
  metrics:
    - ACCURACY
    - SENSITIVITY
    - SPECIFICITY
    - PRECISION
    - RECALL
    - F1_SCORE
    - ROC_AUC
    - PR_AUC
    - LOG_LOSS
    - BRIER_SCORE
  caution:
    ACCURACY_CAN_BE_MISLEADING_WITH_CLASS_IMBALANCE

CALIBRATION:
  ideal:
    P(Y=1|p_hat=p) approximately p
  methods:
    - CALIBRATION_PLOT
    - CALIBRATION_INTERCEPT
    - CALIBRATION_SLOPE
    - BRIER_SCORE
  rule:
    CALIBRATION != DISCRIMINATION

DISCRIMINATION:
  metrics:
    - ROC_AUC
    - CONCORDANCE_INDEX
  rule:
    HIGH_DISCRIMINATION != GOOD_CALIBRATION

THRESHOLD_VALIDATION:
  threshold:
    tau
  evaluate:
    - SENSITIVITY
    - SPECIFICITY
    - PRECISION
    - FALSE_POSITIVE_RATE
    - FALSE_NEGATIVE_RATE
    - EXPECTED_COST
  rule:
    THRESHOLD_SHOULD_BE_SELECTED_FROM_DECISION_OBJECTIVE

PREDICTION_INTERVAL_VALIDATION:
  nominal_coverage:
    1-alpha
  empirical_coverage:
    observed_fraction(Y_i IN PI_i)
  also_assess:
    INTERVAL_WIDTH

PARAMETER_VALIDATION:
  methods:
    - STANDARD_ERROR_CHECK
    - BOOTSTRAP_STABILITY
    - PROFILE_LIKELIHOOD
    - POSTERIOR_DIAGNOSTICS
    - SIMULATION_RECOVERY

SIMULATION_RECOVERY:
  flow:
    CHOOSE_KNOWN_PARAMETERS
      -> SIMULATE_DATA
      -> FIT_MODEL
      -> COMPARE_ESTIMATES_TO_TRUE_PARAMETERS
      -> REPEAT

BAYESIAN_VALIDATION:
  methods:
    - PRIOR_PREDICTIVE_CHECK
    - POSTERIOR_PREDICTIVE_CHECK
    - SIMULATION_BASED_CALIBRATION
    - MCMC_DIAGNOSTICS
    - SENSITIVITY_TO_PRIOR

POSTERIOR_PREDICTIVE_CHECK:
  flow:
    SAMPLE_theta_FROM_POSTERIOR
      -> SIMULATE_REPLICATED_DATA
      -> COMPARE_WITH_OBSERVED_DATA

MCMC_VALIDATION:
  diagnostics:
    - R_HAT
    - EFFECTIVE_SAMPLE_SIZE
    - TRACE_PLOTS
    - AUTOCORRELATION
    - DIVERGENCES_WHEN_RELEVANT
  caution:
    CONVERGED_SAMPLER_DOES_NOT_PROVE_MODEL_IS_CORRECT


MODEL_COMPARISON:
  methods:
    - CROSS_VALIDATION
    - INFORMATION_CRITERIA
    - LIKELIHOOD_RATIO
    - BAYES_FACTOR
    - PREDICTIVE_SCORE
  requirement:
    COMPARISON_METHOD_MUST_MATCH_INFERENTIAL_GOAL

VALIDATION_AFTER_MODEL_SELECTION:
  issue:
    SELECTING_MODEL_AND_REPORTING_SAME_DATA_PERFORMANCE
      -> OPTIMISM
  response:
    - NESTED_CROSS_VALIDATION
    - INDEPENDENT_TEST_SET
    - EXTERNAL_VALIDATION

OVERFITTING:
  definition:
    MODEL_CAPTURES_SAMPLE_SPECIFIC_NOISE_AND_PERFORMS_POORLY_ON_NEW_DATA
  controls:
    - REGULARISATION
    - CROSS_VALIDATION
    - SIMPLER_MODEL
    - MORE_DATA
    - EARLY_STOPPING

UNDERFITTING:
  definition:
    MODEL_TOO_SIMPLE_OR_INADEQUATE_TO_CAPTURE_RELEVANT_STRUCTURE

GENERALISATION:
  target:
    PERFORMANCE_ON_NEW_DATA_FROM_TARGET_DISTRIBUTION
  depends_on:
    - REPRESENTATIVE_DEVELOPMENT_DATA
    - STABLE_DATA_GENERATING_PROCESS
    - CORRECT_VALIDATION_DESIGN
    - ADEQUATE_SAMPLE_SIZE

TRANSPORTABILITY:
  target:
    PERFORMANCE_IN_DIFFERENT_BUT_RELATED_POPULATION_OR_SETTING
  assess:
    - POPULATION_SHIFT
    - PREVALENCE_SHIFT
    - MEASUREMENT_SHIFT
    - PRACTICE_OR_PROCESS_SHIFT

DISTRIBUTION_SHIFT:
  forms:
    - COVARIATE_SHIFT
    - LABEL_SHIFT
    - CONCEPT_DRIFT
  consequence:
    VALIDATION_PERFORMANCE_MAY_NOT_TRANSFER_TO_NEW_DOMAIN

ROBUSTNESS_VALIDATION:
  vary:
    - MODEL_ASSUMPTIONS
    - HYPERPARAMETERS
    - DATA_EXCLUSIONS
    - MISSING_DATA_METHOD
    - OUTLIER_HANDLING
    - ERROR_DISTRIBUTION
  observe:
    - ESTIMATES
    - UNCERTAINTY
    - PERFORMANCE
    - DECISIONS

SENSITIVITY_ANALYSIS_CONNECTION:
  purpose:
    TEST_STABILITY_OF_VALIDATION_CONCLUSIONS_TO_PLAUSIBLE_ANALYTIC_CHOICES

DATA_QUALITY_CONNECTION:
  requirement:
    VALIDATION_DATA_MUST_BE_ASSESSED_FOR:
      COMPLETENESS
      ACCURACY
      REPRESENTATIVENESS
      DUPLICATION
      LEAKAGE
      MEASUREMENT_QUALITY

OUTLIER_DETECTION_CONNECTION:
  caution:
    OUTLIER_REMOVAL_RULES_MUST_BE_DEFINED_WITHIN_VALIDATION_PIPELINE
    AND:
      MUST_NOT_USE_TEST_OUTCOMES_TO_OPTIMISE_CLEANING

MEASUREMENT_UNCERTAINTY_CONNECTION:
  principle:
    VALIDATION_SHOULD_ACCOUNT_FOR_MEASUREMENT_UNCERTAINTY_WHEN_MATERIAL_TO_TARGET_METRIC

REPRODUCIBILITY:
  preserve:
    - DATA_VERSION
    - SPLIT_IDENTIFIERS
    - RANDOM_SEED
    - PREPROCESSING_PIPELINE
    - MODEL_VERSION
    - HYPERPARAMETERS
    - METRIC_DEFINITIONS
    - THRESHOLDS
    - SOFTWARE_VERSION

APPLICATIONS:
  STATISTICAL_VALIDATION -> SUPPORTS -> SCIENTIFIC_STATISTICS
  STATISTICAL_VALIDATION -> SUPPORTS -> MACHINE_LEARNING
  STATISTICAL_VALIDATION -> SUPPORTS -> REGRESSION
  STATISTICAL_VALIDATION -> SUPPORTS -> CLASSIFICATION
  STATISTICAL_VALIDATION -> SUPPORTS -> BAYESIAN_STATISTICS
  STATISTICAL_VALIDATION -> SUPPORTS -> EXPERIMENTAL_RESEARCH
  STATISTICAL_VALIDATION -> SUPPORTS -> DECISION_ANALYSIS

MODEL_CONNECTIONS:
  STATISTICAL_VALIDATION -> CONNECTS_TO -> DATA_QUALITY
  STATISTICAL_VALIDATION -> CONNECTS_TO -> OUTLIER_DETECTION
  STATISTICAL_VALIDATION -> CONNECTS_TO -> ERROR_DISTRIBUTION
  STATISTICAL_VALIDATION -> CONNECTS_TO -> MEASUREMENT_UNCERTAINTY
  STATISTICAL_VALIDATION -> CONNECTS_TO -> CROSS_VALIDATION
  STATISTICAL_VALIDATION -> CONNECTS_TO -> SENSITIVITY_ANALYSIS
  STATISTICAL_VALIDATION -> CONNECTS_TO -> MACHINE_LEARNING_STATISTICS

SEMANTIC_EQUIVALENCE:
  "statistical validation"
    ~= "empirical assessment of statistical model or method performance"

  "internal validation"
    ~= "validation using development data through resampling or partitioning"

  "external validation"
    ~= "validation on independently collected external data"

  "calibration"
    ~= "agreement between predicted and observed outcome frequencies or values"

  EXCLUDE_EQUIVALENCE:
    VALIDATION != TRAINING
    VALIDATION_DATA != TEST_DATA_BY_DEFAULT
    GOODNESS_OF_FIT != GENERALISATION
    CALIBRATION != DISCRIMINATION
    INTERNAL_VALIDATION != EXTERNAL_VALIDATION
    CROSS_VALIDATION != EXTERNAL_VALIDATION
    CONVERGENCE != MODEL_VALIDITY
    HIGH_ACCURACY != VALID_MODEL_IN_ALL_CONTEXTS
    REPRODUCIBILITY != EXTERNAL_VALIDITY

VALIDATION_RULES:
  RULE_1:
    TARGET_USE_POPULATION_AND_PERFORMANCE_METRICS_MUST_BE_DEFINED
  RULE_2:
    TRAINING_VALIDATION_AND_TEST_INFORMATION_MUST_BE_SEPARATED_APPROPRIATELY
  RULE_3:
    PREPROCESSING_FEATURE_SELECTION_AND_TUNING_MUST_OCCUR_WITHIN_TRAINING_PROCESS
  RULE_4:
    DEPENDENCE_GROUPING_AND_TIME_ORDER_MUST_BE_PRESERVED_IN_DATA_SPLITS
  RULE_5:
    CALIBRATION_AND_DISCRIMINATION_SHOULD_BE_ASSESSED_SEPARATELY_WHEN_RELEVANT
  RULE_6:
    PERFORMANCE_UNCERTAINTY_SHOULD_BE_REPORTED
  RULE_7:
    MODEL_SELECTION_MUST_BE_ACCOUNTED_FOR_IN_FINAL_VALIDATION
  RULE_8:
    EXTERNAL_GENERALISATION_CLAIMS_REQUIRE_EVIDENCE_FROM_RELEVANT_TARGET_POPULATION_OR_SETTING

CONFIDENCE:
  ontology_identity: 0.99
  internal_validation_structure: 0.99
  external_validation_structure: 0.99
  cross_validation_structure: 0.99
  calibration_structure: 0.99
  robustness_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_VALIDATION_DATA_ARRIVES
  THEN:
    UPDATE:
      PERFORMANCE_METRICS
      CALIBRATION
      DISCRIMINATION
      UNCERTAINTY_INTERVALS
    REASSESS:
      GENERALISATION

  IF:
    MODEL_OR_HYPERPARAMETERS_CHANGE
  THEN:
    INVALIDATE:
      PREVIOUS_FINAL_VALIDATION_IF_TEST_DATA_INFLUENCED
    RERUN:
      VALIDATION_PIPELINE

  IF:
    TARGET_POPULATION_CHANGES
  THEN:
    REASSESS:
      REPRESENTATIVENESS
      TRANSPORTABILITY
      EXTERNAL_VALIDITY

  PRESERVE:
    TARGET_USE
    TARGET_POPULATION
    DATA_SPLITS
    MODEL_VERSION
    PREPROCESSING
    HYPERPARAMETERS
    METRIC_DEFINITIONS
    THRESHOLDS
    RANDOM_SEEDS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_TARGET_USE_AND_POPULATION
    - IDENTIFY_MODEL_AND_VALIDATION_OBJECTIVE
    - DEFINE_PERFORMANCE_METRICS
    - IDENTIFY_DATA_DEPENDENCE_AND_TIME_STRUCTURE
    - CONSTRUCT_LEAKAGE_SAFE_VALIDATION_DESIGN
    - FIT_PREPROCESSING_AND_MODEL_ON_TRAINING_DATA
    - TUNE_ONLY_WITHIN_VALIDATION_PROCESS
    - EVALUATE_HELD_OUT_PERFORMANCE
    - ASSESS_CALIBRATION_AND_DISCRIMINATION
    - QUANTIFY_PERFORMANCE_UNCERTAINTY
    - ASSESS_ROBUSTNESS_AND_DISTRIBUTION_SHIFT
    - ASSESS_EXTERNAL_GENERALISATION_WHEN_DATA_AVAILABLE
    - RETURN_STATISTICAL_VALIDATION_INTERPRETATION

  constraints:
    - PRESERVE_TRAIN_VALIDATION_TEST_SEPARATION
    - PREVENT_DATA_LEAKAGE
    - PRESERVE_GROUP_AND_TIME_DEPENDENCE
    - DISTINGUISH_CALIBRATION_FROM_DISCRIMINATION
    - DISTINGUISH_GOODNESS_OF_FIT_FROM_GENERALISATION
    - QUANTIFY_VALIDATION_UNCERTAINTY
    - DISTINGUISH_INTERNAL_FROM_EXTERNAL_VALIDATION
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "statistical-validation"
  namespace: "probability-statistics/scientific-statistical-methods"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - INTERNAL_VALIDATION
    - EXTERNAL_VALIDATION
    - CALIBRATION_AND_DISCRIMINATION
    - ROBUSTNESS_AND_GENERALISATION

END_FLOW