Machine Learning Statistics

flow "MachineLearningStatistics" v4.0

# MACHINE LEARNING STATISTICS
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/machine-learning-statistics.html"
  title: "Machine Learning Statistics"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: MACHINE_LEARNING_STATISTICS
  entity_type: STATISTICAL_MACHINE_LEARNING_DOMAIN

DEFINE:
  MACHINE_LEARNING_STATISTICS :=
    INTEGRATED_FRAMEWORK(
      PROBABILITY,
      STATISTICAL_INFERENCE,
      MACHINE_LEARNING,
      ESTIMATION,
      GENERALISATION,
      MODEL_EVALUATION,
      UNCERTAINTY
    )

PURPOSE:
  CONNECT:
    STATISTICAL_REASONING
    WITH:
      MACHINE_LEARNING
  TO:
    ESTIMATE_MODELS
    QUANTIFY_UNCERTAINTY
    EVALUATE_PREDICTIONS
    CONTROL_OVERFITTING
    MEASURE_GENERALISATION

FOUNDATION:
  DATASET: D := {(x_i,y_i)}_[i=1..n]
  DATA_GENERATING_DISTRIBUTION: P(X,Y)
  MODEL: f_theta
  EMPIRICAL_RISK:
    R_hat(theta) := (1/n) * SUM[i=1..n] ell(f_theta(x_i),y_i)
  POPULATION_RISK:
    R(theta) := E_P[ell(f_theta(X),Y)]

ENTITIES:
  DATASET: OBSERVATION_COLLECTION
  FEATURE: INPUT_VARIABLE
  TARGET: OUTPUT_VARIABLE
  MODEL: PREDICTIVE_OR_PROBABILISTIC_MAPPING
  PARAMETER: LEARNED_MODEL_QUANTITY
  HYPERPARAMETER: MODEL_CONFIGURATION_QUANTITY
  ESTIMATOR: PARAMETER_ESTIMATION_RULE
  LOSS_FUNCTION: ERROR_OR_OBJECTIVE_MEASURE
  EMPIRICAL_RISK: SAMPLE_LOSS
  POPULATION_RISK: EXPECTED_TARGET_LOSS
  GENERALISATION_ERROR: OUT_OF_SAMPLE_ERROR
  STANDARD_ERROR: ESTIMATOR_VARIABILITY_MEASURE
  PREDICTIVE_DISTRIBUTION: DISTRIBUTION_OVER_FUTURE_OUTPUTS

RELATIONSHIPS:
  DATASET -> CONTAINS -> OBSERVATIONS
  MODEL -> PARAMETERISED_BY -> PARAMETERS
  TRAINING_DATA -> ESTIMATES -> PARAMETERS
  LOSS_FUNCTION -> EVALUATES -> PREDICTIONS
  EMPIRICAL_RISK -> ESTIMATES -> POPULATION_RISK
  VALIDATION_DATA -> SUPPORTS -> MODEL_SELECTION
  TEST_DATA -> SUPPORTS -> GENERALISATION_EVALUATION


DESCRIPTIVE_STATISTICS:
  location: [MEAN, MEDIAN, QUANTILES]
  dispersion: [VARIANCE, STANDARD_DEVIATION, INTERQUARTILE_RANGE]
  dependence: [COVARIANCE, CORRELATION]

CORRELATION:
  rho_XY := Cov(X,Y)/(sigma_X*sigma_Y)
  caution:
    CORRELATION != CAUSATION
    ZERO_CORRELATION != INDEPENDENCE_BY_DEFAULT

SAMPLING_DISTRIBUTION:
  definition: DISTRIBUTION_OF_STATISTIC_OVER_REPEATED_SAMPLES
  supports: [STANDARD_ERROR, CONFIDENCE_INTERVAL, HYPOTHESIS_TESTING]

LAW_OF_LARGE_NUMBERS:
  role: SAMPLE_AVERAGES_CONVERGE_TO_EXPECTATIONS
  under: APPROPRIATE_CONDITIONS

CENTRAL_LIMIT_THEOREM:
  role: STANDARDISED_SUMS_OR_MEANS_APPROACH_NORMALITY
  under: APPROPRIATE_CONDITIONS

PARAMETER_ESTIMATION:
  estimator: theta_hat := T(D)

BIAS:
  Bias(theta_hat) := E[theta_hat]-theta

VARIANCE:
  Var(theta_hat) := E[(theta_hat-E[theta_hat])^2]

MEAN_SQUARED_ERROR:
  MSE(theta_hat) := Var(theta_hat) + Bias(theta_hat)^2

CONSISTENCY:
  theta_hat_n -> theta
  mode: IN_PROBABILITY
  as: n -> infinity

LIKELIHOOD:
  L(theta;D) := p(D|theta)

MAXIMUM_LIKELIHOOD:
  theta_hat_MLE := argmax_theta L(theta;D)
  equivalent: MINIMISE_NEGATIVE_LOG_LIKELIHOOD

CROSS_ENTROPY_CONNECTION:
  classification_loss: -SUM_k y_k log q_k
  connects_to: [MAXIMUM_LIKELIHOOD, KL_DIVERGENCE]

KL_DIVERGENCE_CONNECTION:
  H(P,Q) := H(P)+D_KL(P||Q)
  implication:
    MINIMISING_EXPECTED_CROSS_ENTROPY_OVER_Q
      -> MINIMISING_FORWARD_KL
  when: P_FIXED

BAYESIAN_ESTIMATION:
  prior: p(theta)
  likelihood: p(D|theta)
  posterior: p(theta|D) proportional_to p(D|theta)*p(theta)
  predictive:
    p(y_star|x_star,D) :=
      INTEGRAL p(y_star|x_star,theta)p(theta|D)dtheta

MAP_ESTIMATION:
  theta_hat_MAP := argmax_theta p(theta|D)

REGULARISATION:
  objective: DATA_LOSS + lambda*PENALTY
  L2: CONNECTS_TO_GAUSSIAN_PRIOR
  L1: CONNECTS_TO_LAPLACE_PRIOR

BIAS_VARIANCE_TRADEOFF:
  error_components: [BIAS_SQUARED, VARIANCE, IRREDUCIBLE_NOISE]

OVERFITTING:
  condition: LOW_TRAINING_ERROR_BUT_HIGH_OUT_OF_SAMPLE_ERROR

UNDERFITTING:
  condition: MODEL_TOO_RESTRICTED_TO_CAPTURE_RELEVANT_STRUCTURE

GENERALISATION:
  training_risk: R_hat(theta)
  target_risk: R(theta)
  generalisation_gap: R(theta)-R_hat(theta)


TRAIN_VALIDATION_TEST:
  TRAIN: FIT_PARAMETERS
  VALIDATION: SELECT_HYPERPARAMETERS_AND_MODELS
  TEST: FINAL_OUT_OF_SAMPLE_EVALUATION
  rule: TEST_SET_SHOULD_NOT_GUIDE_ITERATIVE_MODEL_SELECTION

CROSS_VALIDATION:
  flow:
    SPLIT_DATA
      -> TRAIN_ON_k_MINUS_1_FOLDS
      -> VALIDATE_ON_REMAINING_FOLD
      -> ROTATE
      -> AGGREGATE
  caution: PREPROCESSING_MUST_BE_FIT_WITHIN_TRAINING_FOLDS

BOOTSTRAP:
  flow:
    RESAMPLE_WITH_REPLACEMENT
      -> RECOMPUTE_STATISTIC_OR_MODEL
      -> APPROXIMATE_SAMPLING_DISTRIBUTION

CONFIDENCE_INTERVAL:
  frequentist: PROCEDURE_WITH_SPECIFIED_LONG_RUN_COVERAGE
  caution: CONFIDENCE_INTERVAL != CREDIBLE_INTERVAL

HYPOTHESIS_TESTING:
  null: H_0
  alternative: H_1
  test_statistic: T(D)

P_VALUE:
  definition: NULL_MODEL_PROBABILITY_OF_RESULT_AT_LEAST_AS_EXTREME_AS_OBSERVED
  caution:
    P_VALUE != PROBABILITY_H0_IS_TRUE
    P_VALUE != EFFECT_SIZE

MULTIPLE_TESTING:
  controls: [BONFERRONI, HOLM, FALSE_DISCOVERY_RATE]

FEATURE_SELECTION:
  methods: [HYPOTHESIS_TESTS, MUTUAL_INFORMATION, REGULARISED_MODELS, MODEL_COMPARISON]
  caution: FEATURE_SELECTION_MUST_OCCUR_WITHIN_TRAINING_OR_CROSS_VALIDATION

LINEAR_REGRESSION:
  model: y := X beta + epsilon
  OLS: beta_hat := argmin_beta ||y-X beta||^2

LOGISTIC_REGRESSION:
  model: logit(P(Y=1|X)) := beta_0 + beta^T X
  fitting: MAXIMUM_LIKELIHOOD

TREE_STATISTICS:
  classification: [INFORMATION_GAIN, GINI_IMPURITY]
  regression: [VARIANCE_REDUCTION, SQUARED_ERROR_REDUCTION]

ENSEMBLE_STATISTICS:
  BAGGING: REDUCES_VARIANCE_THROUGH_MODEL_AVERAGING
  RANDOM_FOREST: COMBINES_BOOTSTRAP_SAMPLES_AND_RANDOM_FEATURE_SUBSETS
  BOOSTING: SEQUENTIALLY_REDUCES_PREDICTIVE_LOSS

CLASSIFICATION_METRICS:
  - ACCURACY
  - PRECISION
  - RECALL
  - F1
  - ROC_AUC
  - PR_AUC
  - LOG_LOSS

REGRESSION_METRICS:
  - MSE
  - RMSE
  - MAE
  - R_SQUARED
  - PREDICTIVE_LOG_LIKELIHOOD

CALIBRATION:
  goal: PREDICTED_PROBABILITIES_CORRESPOND_TO_OBSERVED_FREQUENCIES
  caution: ACCURACY != CALIBRATION

UNCERTAINTY:
  ALEATORIC: DATA_VARIABILITY
  EPISTEMIC: MODEL_OR_PARAMETER_UNCERTAINTY
  methods: [BAYESIAN_INFERENCE, BOOTSTRAP, ENSEMBLES, CONFORMAL_PREDICTION]

CONFORMAL_PREDICTION:
  purpose: PREDICTION_SETS_OR_INTERVALS
  guarantee: FINITE_SAMPLE_MARGINAL_COVERAGE
  under: EXCHANGEABILITY
  caution: MARGINAL_COVERAGE != CONDITIONAL_COVERAGE_BY_DEFAULT


DISTRIBUTION_SHIFT:
  training: P_train(X,Y)
  deployment: P_deploy(X,Y)
  forms: [COVARIATE_SHIFT, LABEL_SHIFT, CONCEPT_SHIFT]
  implication: MODEL_PERFORMANCE_MAY_CHANGE

DATA_LEAKAGE:
  examples:
    - TEST_DATA_USED_IN_TRAINING
    - GLOBAL_PREPROCESSING_BEFORE_SPLIT
    - FUTURE_INFORMATION_IN_TIME_SERIES
    - TARGET_LEAKAGE
  consequence: BIASED_GENERALISATION_ESTIMATE

MISSING_DATA:
  mechanisms: [MCAR, MAR, MNAR]
  methods: [COMPLETE_CASE, IMPUTATION, MULTIPLE_IMPUTATION, MODEL_BASED_METHODS]

CLASS_IMBALANCE:
  responses: [CLASS_WEIGHTS, RESAMPLING, THRESHOLD_SELECTION, APPROPRIATE_METRICS]

STATISTICAL_SIGNIFICANCE_VS_PREDICTION:
  statistically_significant_feature: NOT_NECESSARILY_STRONG_PREDICTOR
  strong_predictor: NOT_NECESSARILY_CAUSAL_VARIABLE
  rule: INFERENCE_GOAL != PREDICTION_GOAL

CAUSALITY_DISTINCTION:
  machine_learning_statistics: OFTEN_MODELS_ASSOCIATION_AND_PREDICTION
  causal_inference:
    REQUIRES:
      CAUSAL_ASSUMPTIONS
      IDENTIFICATION
      EXPERIMENTAL_OR_QUASI_EXPERIMENTAL_DESIGN
  rule: PREDICTIVE_ASSOCIATION != CAUSAL_EFFECT

MODEL_SELECTION:
  criteria:
    - VALIDATION_LOSS
    - CROSS_VALIDATION
    - AIC
    - BIC
    - COMPUTATIONAL_COST
    - ROBUSTNESS

AIC:
  AIC := 2k - 2*log(L_hat)

BIC:
  BIC := k*log(n) - 2*log(L_hat)

HYPERPARAMETER_OPTIMISATION:
  methods: [GRID_SEARCH, RANDOM_SEARCH, BAYESIAN_OPTIMISATION]
  requirement: USE_VALIDATION_PROTOCOL

STATISTICAL_POWER:
  power: 1-beta
  depends_on: [SAMPLE_SIZE, EFFECT_SIZE, VARIANCE, SIGNIFICANCE_THRESHOLD, TEST_DESIGN]

SAMPLE_SIZE:
  larger_sample: GENERALLY_REDUCES_ESTIMATION_VARIABILITY
  caution:
    LARGE_SAMPLE_DOES_NOT_FIX:
      BIAS
      CONFOUNDING
      DATA_LEAKAGE
      DISTRIBUTION_SHIFT

REPRODUCIBILITY:
  preserve:
    - DATASET_VERSION
    - SPLIT_DEFINITION
    - RANDOM_SEED
    - FEATURE_PIPELINE
    - MODEL_VERSION
    - HYPERPARAMETERS
    - SOFTWARE_ENVIRONMENT
    - METRIC_DEFINITIONS

APPLICATIONS:
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> MODEL_TRAINING
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> MODEL_EVALUATION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> FEATURE_SELECTION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> HYPERPARAMETER_SELECTION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> UNCERTAINTY_QUANTIFICATION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> CLASSIFICATION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> REGRESSION
  MACHINE_LEARNING_STATISTICS -> SUPPORTS -> FORECASTING

MODEL_CONNECTIONS:
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> STATISTICS
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> MACHINE_LEARNING
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> STATISTICAL_LEARNING
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> PROBABILITY_THEORY
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> BAYESIAN_STATISTICS
  MACHINE_LEARNING_STATISTICS -> CONNECTS_TO -> INFORMATION_THEORY


SEMANTIC_EQUIVALENCE:
  "machine learning statistics"
    ~= "statistical methods for machine learning"

  "generalisation error"
    ~= "out-of-sample predictive error"

  "empirical risk"
    ~= "average observed training loss"

  EXCLUDE_EQUIVALENCE:
    EMPIRICAL_RISK != POPULATION_RISK
    PARAMETER != HYPERPARAMETER
    TRAINING_ERROR != GENERALISATION_ERROR
    CONFIDENCE_INTERVAL != CREDIBLE_INTERVAL
    P_VALUE != PROBABILITY_NULL_IS_TRUE
    ACCURACY != CALIBRATION
    PREDICTION != CAUSATION

VALIDATION_RULES:
  RULE_1:
    DATA_GENERATING_OR_TARGET_DISTRIBUTION_MUST_BE_DEFINED_OR_STATED_AS_UNKNOWN

  RULE_2:
    TRAIN_VALIDATION_TEST_ROLES_MUST_BE_PRESERVED

  RULE_3:
    PREPROCESSING_AND_FEATURE_SELECTION_MUST_NOT_LEAK_EVALUATION_INFORMATION

  RULE_4:
    LOSS_AND_METRICS_MUST_MATCH_TASK_AND_DECISION_CONTEXT

  RULE_5:
    STATISTICAL_INFERENCE_MUST_STATE_MODEL_AND_SAMPLING_ASSUMPTIONS

  RULE_6:
    GENERALISATION_CLAIMS_MUST_CONSIDER_DISTRIBUTION_SHIFT

  RULE_7:
    UNCERTAINTY_MEASURE_MUST_MATCH_INFERENTIAL_CONTEXT

  RULE_8:
    STATISTICAL_ASSOCIATION_MUST_NOT_BE_INTERPRETED_AS_CAUSAL_EFFECT_WITHOUT_CAUSAL_ASSUMPTIONS

CONFIDENCE:
  ontology_identity: 0.99
  statistical_foundation: 0.99
  machine_learning_relationship: 0.99
  generalisation_structure: 0.99
  model_evaluation_structure: 0.99
  uncertainty_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_TRAINING_DATA_ARRIVES
  THEN:
    UPDATE:
      DATASET_VERSION
      DESCRIPTIVE_STATISTICS
      PARAMETER_ESTIMATES
      EMPIRICAL_RISK
    RETRAIN_OR_UPDATE:
      MODEL
    REEVALUATE:
      VALIDATION_PERFORMANCE
      CALIBRATION
      UNCERTAINTY

  IF:
    HYPERPARAMETERS_CHANGE
  THEN:
    RETRAIN:
      MODEL
    REVALIDATE:
      PERFORMANCE

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    DETECT:
      DISTRIBUTION_SHIFT
    REASSESS:
      GENERALISATION
      CALIBRATION
      ROBUSTNESS

  PRESERVE:
    DATASET_VERSION
    FEATURE_DEFINITIONS
    TARGET_DEFINITION
    MODEL_VERSION
    TRAIN_VALIDATION_TEST_SPLITS
    HYPERPARAMETERS
    RANDOM_SEED_WHEN_RECORDED
    METRIC_DEFINITIONS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DATASET_AND_TARGET
    - IDENTIFY_DATA_GENERATING_OR_TARGET_DISTRIBUTION
    - COMPUTE_DESCRIPTIVE_STATISTICS
    - IDENTIFY_MODEL_AND_PARAMETERS
    - IDENTIFY_LOSS_FUNCTION
    - ESTIMATE_PARAMETERS
    - QUANTIFY_UNCERTAINTY
    - VERIFY_TRAIN_VALIDATION_TEST_SEPARATION
    - ESTIMATE_GENERALISATION_PERFORMANCE
    - ASSESS_CALIBRATION_AND_DISTRIBUTION_SHIFT
    - DISTINGUISH_PREDICTIVE_FROM_CAUSAL_INTERPRETATION
    - RETURN_STATISTICAL_MACHINE_LEARNING_INTERPRETATION

  constraints:
    - PRESERVE_POPULATION_SAMPLE_DISTINCTION
    - PRESERVE_TRAIN_VALIDATION_TEST_SEPARATION
    - PREVENT_DATA_LEAKAGE
    - DISTINGUISH_PARAMETER_FROM_HYPERPARAMETER
    - DISTINGUISH_PREDICTION_FROM_CAUSATION
    - EXPOSE_STATISTICAL_AND_MODEL_ASSUMPTIONS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "machine-learning-statistics"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - EMPIRICAL_POPULATION_RISK
    - STATISTICAL_INFERENCE
    - GENERALISATION_EVALUATION

END_FLOW