Machine Learning Statistics Ontology

flow "MachineLearningStatisticsOntology" v4.0

# MACHINE LEARNING STATISTICS ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/machine-learning-statistics-ontology.html"
  title: "Machine Learning Statistics Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: MACHINE_LEARNING_STATISTICS
  entity_type: STATISTICAL_MACHINE_LEARNING_ONTOLOGY

DEFINE:
  MACHINE_LEARNING_STATISTICS :=
    FORMAL_ONTOLOGY(
      DATA,
      PROBABILITY,
      STATISTICAL_INFERENCE,
      LEARNING_ALGORITHMS,
      LOSS_FUNCTIONS,
      GENERALISATION,
      MODEL_EVALUATION,
      UNCERTAINTY
    )

FOUNDATION:
  DATASET: D := {(x_i,y_i)}_[i=1..n]
  DATA_GENERATING_DISTRIBUTION: P(X,Y)
  MODEL: f_theta
  PARAMETERS: theta
  EMPIRICAL_RISK:
    R_hat(theta) := (1/n)*SUM[i=1..n] ell(f_theta(x_i),y_i)
  POPULATION_RISK:
    R(theta) := E_P[ell(f_theta(X),Y)]

ENTITIES:
  DATASET: OBSERVATION_COLLECTION
  FEATURE: INPUT_VARIABLE
  TARGET: OUTPUT_VARIABLE
  MODEL: PREDICTIVE_OR_PROBABILISTIC_MAPPING
  PARAMETER: LEARNED_MODEL_QUANTITY
  HYPERPARAMETER: CONFIGURATION_QUANTITY
  ESTIMATOR: PARAMETER_ESTIMATION_RULE
  LOSS_FUNCTION: MODEL_ERROR_MEASURE
  EMPIRICAL_RISK: SAMPLE_AVERAGE_LOSS
  POPULATION_RISK: EXPECTED_TARGET_LOSS
  GENERALISATION_ERROR: OUT_OF_SAMPLE_ERROR
  CALIBRATION: PROBABILITY_RELIABILITY_PROPERTY

RELATIONSHIPS:
  DATASET -> CONTAINS -> OBSERVATIONS
  MODEL -> PARAMETERISED_BY -> PARAMETERS
  MODEL -> CONFIGURED_BY -> HYPERPARAMETERS
  TRAINING_DATA -> FITS -> MODEL
  LOSS_FUNCTION -> EVALUATES -> MODEL_OUTPUT
  EMPIRICAL_RISK -> APPROXIMATES -> POPULATION_RISK
  VALIDATION_DATA -> SUPPORTS -> MODEL_SELECTION
  TEST_DATA -> SUPPORTS -> GENERALISATION_EVALUATION

STATISTICAL_FOUNDATIONS:
  SAMPLE_MEAN:
    x_bar := (1/n)*SUM[i=1..n] x_i

  SAMPLE_VARIANCE:
    s_squared := 1/(n-1)*SUM[i=1..n](x_i-x_bar)^2

  CORRELATION:
    rho_XY := Cov(X,Y)/(sigma_X*sigma_Y)

  caution:
    CORRELATION != CAUSATION
    ZERO_CORRELATION != INDEPENDENCE_BY_DEFAULT

SAMPLING_DISTRIBUTION:
  role:
    SUPPORT_STANDARD_ERRORS_CONFIDENCE_INTERVALS_AND_TESTS

LAW_OF_LARGE_NUMBERS:
  role:
    SAMPLE_AVERAGES_CONVERGE_TO_EXPECTATIONS
  under:
    APPROPRIATE_CONDITIONS

CENTRAL_LIMIT_THEOREM:
  role:
    STANDARDISED_SUMS_OR_MEANS_APPROACH_NORMALITY
  under:
    APPROPRIATE_CONDITIONS

ESTIMATION:
  estimator:
    theta_hat := T(D)

  bias:
    Bias(theta_hat) := E[theta_hat]-theta

  variance:
    Var(theta_hat) := E[(theta_hat-E[theta_hat])^2]

  mse:
    MSE(theta_hat) := Var(theta_hat)+Bias(theta_hat)^2

LIKELIHOOD:
  L(theta;D) := p(D|theta)

MAXIMUM_LIKELIHOOD:
  theta_hat_MLE := argmax_theta L(theta;D)
  equivalent:
    MINIMISE_NEGATIVE_LOG_LIKELIHOOD

BAYESIAN_STRUCTURE:
  prior: p(theta)
  likelihood: p(D|theta)
  posterior:
    p(theta|D) proportional_to p(D|theta)*p(theta)

  predictive:
    p(y_star|x_star,D) :=
      INTEGRAL p(y_star|x_star,theta)p(theta|D)dtheta

REGULARISATION:
  objective:
    DATA_LOSS + lambda*Omega(theta)

  purpose:
    CONTROL_EFFECTIVE_MODEL_COMPLEXITY_AND_OVERFITTING

BIAS_VARIANCE_TRADEOFF:
  components:
    - BIAS_SQUARED
    - VARIANCE
    - IRREDUCIBLE_NOISE

OVERFITTING:
  condition:
    LOW_TRAINING_ERROR_AND_HIGHER_OUT_OF_SAMPLE_ERROR

UNDERFITTING:
  condition:
    MODEL_TOO_RESTRICTED_TO_CAPTURE_RELEVANT_STRUCTURE

GENERALISATION:
  empirical_risk: R_hat(theta)
  population_risk: R(theta)
  gap: R(theta)-R_hat(theta)

TRAIN_VALIDATION_TEST:
  TRAIN: FIT_PARAMETERS
  VALIDATION: SELECT_HYPERPARAMETERS_AND_MODELS
  TEST: FINAL_OUT_OF_SAMPLE_EVALUATION

  constraint:
    TEST_SET_SHOULD_NOT_GUIDE_ITERATIVE_MODEL_SELECTION

CROSS_VALIDATION:
  flow:
    SPLIT_INTO_k_FOLDS
      -> TRAIN_ON_k_MINUS_1
      -> VALIDATE_ON_REMAINING
      -> ROTATE
      -> AGGREGATE

  caution:
    PREPROCESSING_AND_FEATURE_SELECTION_MUST_OCCUR_WITHIN_TRAINING_FOLD

BOOTSTRAP:
  flow:
    RESAMPLE_WITH_REPLACEMENT
      -> FIT_OR_RECOMPUTE
      -> APPROXIMATE_SAMPLING_DISTRIBUTION

CONFIDENCE_INTERVAL:
  interpretation:
    FREQUENTIST_PROCEDURE_WITH_LONG_RUN_COVERAGE

  distinction:
    CONFIDENCE_INTERVAL != CREDIBLE_INTERVAL

HYPOTHESIS_TESTING:
  null: H_0
  alternative: H_1
  statistic: T(D)

P_VALUE:
  definition:
    NULL_MODEL_PROBABILITY_OF_RESULT_AT_LEAST_AS_EXTREME_AS_OBSERVED

  caution:
    P_VALUE != PROBABILITY_H0_IS_TRUE
    P_VALUE != EFFECT_SIZE

CROSS_ENTROPY:
  classification_loss:
    L := -SUM_k y_k log q_k

  connects_to:
    - MAXIMUM_LIKELIHOOD
    - INFORMATION_THEORY
    - MACHINE_LEARNING

KL_DIVERGENCE:
  relation:
    H(P,Q) := H(P)+D_KL(P||Q)

  implication:
    MINIMISING_EXPECTED_CROSS_ENTROPY_OVER_Q
      -> MINIMISING_FORWARD_KL
    WHEN:
      P_FIXED

MUTUAL_INFORMATION:
  role:
    MEASURE_STATISTICAL_DEPENDENCE

  uses:
    - FEATURE_SELECTION
    - REPRESENTATION_ANALYSIS
    - DEPENDENCE_DISCOVERY

  caution:
    MUTUAL_INFORMATION != CAUSATION

INFORMATION_GAIN:
  role:
    REDUCTION_IN_TARGET_ENTROPY

  uses:
    - DECISION_TREE_SPLITTING
    - FEATURE_SELECTION
    - ACTIVE_LEARNING

LINEAR_REGRESSION:
  model:
    y := X beta + epsilon

  OLS:
    beta_hat := argmin_beta ||y-X beta||^2

LOGISTIC_REGRESSION:
  model:
    logit(P(Y=1|X)) := beta_0 + beta^T X

  fitting:
    MAXIMUM_LIKELIHOOD

DECISION_TREE:
  classification_criteria:
    - INFORMATION_GAIN
    - GINI_IMPURITY

  regression_criteria:
    - VARIANCE_REDUCTION
    - SQUARED_ERROR_REDUCTION

ENSEMBLE_METHODS:
  BAGGING:
    REDUCES_VARIANCE_THROUGH_MODEL_AVERAGING

  RANDOM_FOREST:
    COMBINES_BOOTSTRAP_SAMPLING_AND_RANDOM_FEATURE_SELECTION

  BOOSTING:
    SEQUENTIALLY_IMPROVES_PREDICTIVE_OBJECTIVE

CLASSIFICATION_METRICS:
  - ACCURACY
  - PRECISION
  - RECALL
  - F1
  - ROC_AUC
  - PR_AUC
  - LOG_LOSS

REGRESSION_METRICS:
  - MSE
  - RMSE
  - MAE
  - R_SQUARED
  - PREDICTIVE_LOG_LIKELIHOOD

CALIBRATION:
  definition:
    PREDICTED_PROBABILITIES_CORRESPOND_TO_OBSERVED_FREQUENCIES

  distinction:
    ACCURACY != CALIBRATION

UNCERTAINTY:
  ALEATORIC:
    DATA_OR_OBSERVATION_VARIABILITY

  EPISTEMIC:
    MODEL_OR_PARAMETER_UNCERTAINTY

  methods:
    - BAYESIAN_INFERENCE
    - BOOTSTRAP
    - ENSEMBLES
    - CONFORMAL_PREDICTION

CONFORMAL_PREDICTION:
  purpose:
    PREDICTION_SETS_OR_INTERVALS

  guarantee:
    FINITE_SAMPLE_MARGINAL_COVERAGE
    UNDER:
      EXCHANGEABILITY

  caution:
    MARGINAL_COVERAGE != CONDITIONAL_COVERAGE_BY_DEFAULT

DATA_LEAKAGE:
  examples:
    - TEST_DATA_USED_IN_TRAINING
    - GLOBAL_PREPROCESSING_BEFORE_SPLIT
    - FUTURE_INFORMATION_IN_TIME_SERIES
    - TARGET_LEAKAGE

  consequence:
    BIASED_GENERALISATION_ESTIMATE

DISTRIBUTION_SHIFT:
  training: P_train(X,Y)
  deployment: P_deploy(X,Y)

  forms:
    - COVARIATE_SHIFT
    - LABEL_SHIFT
    - CONCEPT_SHIFT

  implication:
    GENERALISATION_PERFORMANCE_MAY_CHANGE

MISSING_DATA:
  mechanisms:
    - MCAR
    - MAR
    - MNAR

  methods:
    - COMPLETE_CASE
    - IMPUTATION
    - MULTIPLE_IMPUTATION
    - MODEL_BASED_METHODS

CLASS_IMBALANCE:
  responses:
    - CLASS_WEIGHTING
    - RESAMPLING
    - THRESHOLD_SELECTION
    - PR_BASED_METRICS

STATISTICAL_SIGNIFICANCE_VS_PREDICTION:
  significant_feature:
    NOT_NECESSARILY_STRONG_PREDICTOR

  predictive_feature:
    NOT_NECESSARILY_CAUSAL_VARIABLE

  rule:
    INFERENCE_GOAL != PREDICTION_GOAL

CAUSALITY_DISTINCTION:
  statistical_machine_learning:
    OFTEN_ESTIMATES_ASSOCIATION_AND_PREDICTION

  causal_inference:
    REQUIRES:
      CAUSAL_ASSUMPTIONS
      IDENTIFICATION
      EXPERIMENTAL_OR_QUASI_EXPERIMENTAL_DESIGN

  rule:
    PREDICTIVE_ASSOCIATION != CAUSAL_EFFECT

MODEL_SELECTION:
  criteria:
    - VALIDATION_LOSS
    - CROSS_VALIDATION
    - AIC
    - BIC
    - COMPUTATIONAL_COST
    - ROBUSTNESS

AIC:
  AIC := 2k - 2*log(L_hat)

BIC:
  BIC := k*log(n) - 2*log(L_hat)

HYPERPARAMETER_OPTIMISATION:
  methods:
    - GRID_SEARCH
    - RANDOM_SEARCH
    - BAYESIAN_OPTIMISATION

  requirement:
    USE_VALIDATION_PROTOCOL

REPRODUCIBILITY:
  preserve:
    - DATASET_VERSION
    - SPLIT_DEFINITION
    - RANDOM_SEED
    - FEATURE_PIPELINE
    - MODEL_VERSION
    - HYPERPARAMETERS
    - SOFTWARE_ENVIRONMENT
    - METRIC_DEFINITIONS

APPLICATIONS:
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> MODEL_TRAINING
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> MODEL_EVALUATION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> FEATURE_SELECTION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> HYPERPARAMETER_SELECTION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> UNCERTAINTY_QUANTIFICATION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> CLASSIFICATION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> REGRESSION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> FORECASTING

MODEL_CONNECTIONS:
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> STATISTICS
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> MACHINE_LEARNING
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> STATISTICAL_LEARNING
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> PROBABILITY_THEORY
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> BAYESIAN_STATISTICS
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> INFORMATION_THEORY

SEMANTIC_EQUIVALENCE:
  "machine learning statistics"
    ~= "statistical methods for machine learning"

  "generalisation error"
    ~= "out-of-sample predictive error"

  "empirical risk"
    ~= "average observed training loss"

  EXCLUDE_EQUIVALENCE:
    EMPIRICAL_RISK != POPULATION_RISK
    PARAMETER != HYPERPARAMETER
    ESTIMATOR != ESTIMATE
    TRAINING_ERROR != GENERALISATION_ERROR
    CONFIDENCE_INTERVAL != CREDIBLE_INTERVAL
    P_VALUE != PROBABILITY_NULL_IS_TRUE
    ACCURACY != CALIBRATION
    PREDICTION != CAUSATION

VALIDATION_RULES:
  RULE_1:
    DATA_GENERATING_OR_TARGET_DISTRIBUTION_MUST_BE_DEFINED_OR_STATED_AS_UNKNOWN

  RULE_2:
    TRAIN_VALIDATION_TEST_ROLES_MUST_BE_PRESERVED

  RULE_3:
    PREPROCESSING_AND_FEATURE_SELECTION_MUST_NOT_LEAK_EVALUATION_INFORMATION

  RULE_4:
    LOSS_AND_METRICS_MUST_MATCH_TASK_AND_DECISION_CONTEXT

  RULE_5:
    STATISTICAL_INFERENCE_MUST_STATE_MODEL_AND_SAMPLING_ASSUMPTIONS

  RULE_6:
    GENERALISATION_CLAIMS_MUST_CONSIDER_DISTRIBUTION_SHIFT

  RULE_7:
    UNCERTAINTY_MEASURE_MUST_MATCH_INFERENTIAL_CONTEXT

  RULE_8:
    STATISTICAL_ASSOCIATION_MUST_NOT_BE_INTERPRETED_AS_CAUSAL_EFFECT_WITHOUT_CAUSAL_ASSUMPTIONS

CONFIDENCE:
  ontology_identity: 0.99
  statistical_foundation: 0.99
  machine_learning_relationship: 0.99
  generalisation_structure: 0.99
  model_evaluation_structure: 0.99
  uncertainty_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_TRAINING_DATA_ARRIVES
  THEN:
    UPDATE:
      DATASET_VERSION
      DESCRIPTIVE_STATISTICS
      PARAMETER_ESTIMATES
      EMPIRICAL_RISK
    RETRAIN_OR_UPDATE:
      MODEL
    REEVALUATE:
      VALIDATION_PERFORMANCE
      CALIBRATION
      UNCERTAINTY

  IF:
    HYPERPARAMETERS_CHANGE
  THEN:
    RETRAIN:
      MODEL
    REVALIDATE:
      PERFORMANCE

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    DETECT:
      DISTRIBUTION_SHIFT
    REASSESS:
      GENERALISATION
      CALIBRATION
      ROBUSTNESS

  PRESERVE:
    DATASET_VERSION
    FEATURE_DEFINITIONS
    TARGET_DEFINITION
    MODEL_VERSION
    TRAIN_VALIDATION_TEST_SPLITS
    HYPERPARAMETERS
    RANDOM_SEED_WHEN_RECORDED
    METRIC_DEFINITIONS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DATASET_AND_TARGET
    - IDENTIFY_DATA_GENERATING_OR_TARGET_DISTRIBUTION
    - COMPUTE_DESCRIPTIVE_STATISTICS
    - IDENTIFY_MODEL_PARAMETERS_AND_HYPERPARAMETERS
    - IDENTIFY_LOSS_FUNCTION
    - ESTIMATE_PARAMETERS
    - QUANTIFY_UNCERTAINTY
    - VERIFY_TRAIN_VALIDATION_TEST_SEPARATION
    - ESTIMATE_GENERALISATION_PERFORMANCE
    - ASSESS_CALIBRATION_AND_DISTRIBUTION_SHIFT
    - DISTINGUISH_PREDICTIVE_FROM_CAUSAL_INTERPRETATION
    - RETURN_STATISTICAL_MACHINE_LEARNING_INTERPRETATION

  constraints:
    - PRESERVE_POPULATION_SAMPLE_DISTINCTION
    - PRESERVE_TRAIN_VALIDATION_TEST_SEPARATION
    - PREVENT_DATA_LEAKAGE
    - DISTINGUISH_PARAMETER_FROM_HYPERPARAMETER
    - DISTINGUISH_ESTIMATOR_FROM_ESTIMATE
    - DISTINGUISH_PREDICTION_FROM_CAUSATION
    - EXPOSE_STATISTICAL_AND_MODEL_ASSUMPTIONS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "machine-learning-statistics-ontology"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - EMPIRICAL_POPULATION_RISK
    - STATISTICAL_INFERENCE
    - GENERALISATION_EVALUATION

END_FLOW