Statistical Learning Ontology

flow "StatisticalLearningOntology" v4.0

# STATISTICAL LEARNING ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/statistical-learning-ontology.html"
  title: "Statistical Learning Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: STATISTICAL_LEARNING
  entity_type: STATISTICAL_MACHINE_LEARNING_FRAMEWORK

DEFINE:
  STATISTICAL_LEARNING :=
    LEARNING_FRAMEWORK(
      DATA,
      PROBABILITY,
      STATISTICAL_MODELS,
      LOSS_FUNCTIONS,
      ESTIMATION,
      GENERALISATION,
      PREDICTION
    )

PURPOSE:
  LEARN:
    PREDICTIVE_STRUCTURE
    DISTRIBUTIONAL_STRUCTURE
    FUNCTIONAL_RELATIONSHIPS
  FROM:
    OBSERVED_DATA
  WHILE:
    QUANTIFYING_UNCERTAINTY
    CONTROLLING_GENERALISATION_ERROR

FOUNDATION:
  OBSERVATION:
    Z := (X,Y)

  DATASET:
    D_n := {Z_1,...,Z_n}

  DATA_GENERATING_DISTRIBUTION:
    P

  HYPOTHESIS_CLASS:
    F

  PREDICTOR:
    f IN F

  LOSS:
    ell(f(X),Y)

  POPULATION_RISK:
    R(f) :=
      E_P[ell(f(X),Y)]

  EMPIRICAL_RISK:
    R_hat_n(f) :=
      (1/n)
      * SUM[i=1..n]
        ell(f(X_i),Y_i)

ENTITIES:
  TRAINING_SAMPLE: OBSERVED_DATASET
  HYPOTHESIS_CLASS: CANDIDATE_MODEL_SET
  PREDICTOR: PREDICTIVE_FUNCTION
  LOSS_FUNCTION: PREDICTION_ERROR_MEASURE
  POPULATION_RISK: EXPECTED_LOSS
  EMPIRICAL_RISK: SAMPLE_AVERAGE_LOSS
  GENERALISATION_ERROR: OUT_OF_SAMPLE_ERROR
  REGULARISER: COMPLEXITY_CONTROL
  MODEL_COMPLEXITY: HYPOTHESIS_CLASS_CAPACITY

RELATIONSHIPS:
  DATA_GENERATING_DISTRIBUTION -> GENERATES -> TRAINING_SAMPLE
  TRAINING_SAMPLE -> FITS -> PREDICTOR
  PREDICTOR -> BELONGS_TO -> HYPOTHESIS_CLASS
  LOSS_FUNCTION -> EVALUATES -> PREDICTION
  EMPIRICAL_RISK -> APPROXIMATES -> POPULATION_RISK
  MODEL_COMPLEXITY -> INFLUENCES -> GENERALISATION
  REGULARISATION -> CONTROLS -> EFFECTIVE_MODEL_COMPLEXITY

SUPERVISED_LEARNING:
  tasks:
    - REGRESSION
    - CLASSIFICATION
    - RANKING
    - STRUCTURED_PREDICTION

REGRESSION:
  squared_loss:
    ell(y_hat,y) :=
      (y-y_hat)^2

  population_optimum:
    f_star(x) :=
      E[Y|X=x]
    UNDER:
      SQUARED_ERROR_LOSS

CLASSIFICATION:
  zero_one_loss:
    ell(y_hat,y) :=
      1[y_hat != y]

  Bayes_classifier:
    f_star(x) :=
      argmax_y P(Y=y|X=x)

EMPIRICAL_RISK_MINIMISATION:
  alias:
    ERM

  estimator:
    f_hat :=
      argmin_{f IN F}
        R_hat_n(f)

  caution:
    LOW_EMPIRICAL_RISK
      DOES_NOT_GUARANTEE
    LOW_POPULATION_RISK

REGULARISED_EMPIRICAL_RISK:
  objective:
    f_hat :=
      argmin_f
        [
          R_hat_n(f)
          + lambda*Omega(f)
        ]

GENERALISATION_GAP:
  gap(f) :=
    R(f)-R_hat_n(f)

  depends_on:
    - SAMPLE_SIZE
    - MODEL_COMPLEXITY
    - LOSS_PROPERTIES
    - DATA_DEPENDENCE
    - REGULARISATION

IID_ASSUMPTION:
  standard:
    Z_1,...,Z_n IID_FROM P

  caution:
    TIME_SERIES_SPATIAL_AND_CLUSTERED_DATA_MAY_VIOLATE_IID

BIAS_VARIANCE:
  squared_error_context:
    ERROR_MAY_DECOMPOSE_INTO:
      BIAS_SQUARED
      VARIANCE
      IRREDUCIBLE_NOISE

MODEL_COMPLEXITY:
  measures:
    - PARAMETER_COUNT
    - TREE_DEPTH
    - VC_DIMENSION
    - RADEMACHER_COMPLEXITY
    - PARAMETER_NORM

VC_DIMENSION:
  role:
    CAPACITY_MEASURE_FOR_BINARY_HYPOTHESIS_CLASS

  definition:
    LARGEST_SET_SIZE_THAT_CAN_BE_SHATTERED

PAC_LEARNING:
  full_name:
    PROBABLY_APPROXIMATELY_CORRECT_LEARNING

  goal:
    WITH_PROBABILITY_AT_LEAST:
      1-delta
    RETURN:
      HYPOTHESIS_WITH_ERROR_AT_MOST:
        epsilon

UNIFORM_CONVERGENCE:
  concept:
    sup_{f IN F}
      |R(f)-R_hat_n(f)|
      -> 0

  role:
    SUPPORT_GENERALISATION_AND_ERM_CONSISTENCY

CONSISTENCY:
  property:
    R(f_hat_n)
      -> R_star
    UNDER:
      SPECIFIED_CONDITIONS

LOSS_FUNCTIONS:
  SQUARED_ERROR: REGRESSION
  ABSOLUTE_ERROR: REGRESSION
  ZERO_ONE_LOSS: CLASSIFICATION
  LOG_LOSS: PROBABILISTIC_PREDICTION
  HINGE_LOSS: MARGIN_CLASSIFICATION

SURROGATE_LOSS:
  purpose:
    REPLACE_DIFFICULT_TASK_LOSS_WITH_OPTIMISABLE_APPROXIMATION

  examples:
    - LOGISTIC_LOSS
    - HINGE_LOSS

MAXIMUM_LIKELIHOOD_CONNECTION:
  model:
    p_theta(y|x)

  negative_log_likelihood:
    R_hat(theta) :=
      -(1/n)
      * SUM_i log p_theta(y_i|x_i)

  relation:
    MINIMISING_NEGATIVE_LOG_LIKELIHOOD
      EQUIVALENT_TO
    MAXIMISING_LIKELIHOOD

CROSS_ENTROPY_CONNECTION:
  population_log_loss:
    E_P[-log q(Y|X)]

  decomposes_into:
    CONDITIONAL_ENTROPY
    +
    EXPECTED_KL_DIVERGENCE

BAYESIAN_LEARNING:
  prior:
    p(theta)

  likelihood:
    p(D|theta)

  posterior:
    p(theta|D)

  predictive:
    p(y_star|x_star,D)
      =
    INTEGRAL
      p(y_star|x_star,theta)
      p(theta|D)
      dtheta

PARAMETRIC_LEARNING:
  model:
    f_theta

  parameter_dimension:
    FINITE

NONPARAMETRIC_LEARNING:
  model_complexity:
    MAY_GROW_WITH_DATA

  caution:
    NONPARAMETRIC != PARAMETER_FREE

LINEAR_REGRESSION:
  model:
    Y := beta_0 + beta^T X + epsilon

  least_squares:
    beta_hat :=
      argmin_beta
        SUM_i (y_i-beta_0-beta^T x_i)^2

RIDGE_REGRESSION:
  objective:
    SUM_i (y_i-x_i^T beta)^2
    + lambda||beta||_2^2

LASSO:
  objective:
    SUM_i (y_i-x_i^T beta)^2
    + lambda||beta||_1

  effect:
    MAY_PRODUCE_SPARSE_PARAMETERS

LOGISTIC_REGRESSION:
  model:
    P(Y=1|X=x) :=
      sigmoid(beta_0+beta^T x)

  learning:
    MAXIMUM_LIKELIHOOD
    OR:
      CROSS_ENTROPY_MINIMISATION

K_NEAREST_NEIGHBOURS:
  prediction:
    BASED_ON_LOCAL_NEIGHBOURHOOD

  hyperparameter:
    k

KERNEL_METHODS:
  similarity:
    k(x,x_prime)

  examples:
    - KERNEL_REGRESSION
    - SUPPORT_VECTOR_MACHINE
    - GAUSSIAN_PROCESS_CONNECTION

SUPPORT_VECTOR_MACHINE:
  objective:
    MAXIMISE_MARGIN
    WHILE:
      PENALISING_VIOLATIONS

  loss:
    HINGE_LOSS

DECISION_TREE:
  learning:
    RECURSIVE_PARTITIONING

  criteria:
    - INFORMATION_GAIN
    - GINI_IMPURITY
    - VARIANCE_REDUCTION

ENSEMBLE_METHODS:
  BAGGING:
    REDUCE_VARIANCE_BY_AVERAGING

  BOOSTING:
    SEQUENTIALLY_REDUCE_PREDICTION_ERROR

  RANDOM_FOREST:
    BAGGED_RANDOMISED_DECISION_TREES

NEURAL_NETWORKS:
  predictor:
    COMPOSITION_OF_PARAMETERISED_TRANSFORMATIONS

  training:
    GRADIENT_BASED_EMPIRICAL_RISK_MINIMISATION
    OR_RELATED_OBJECTIVE

OPTIMISATION:
  gradient_descent:
    theta_(t+1) :=
      theta_t
      - eta*gradient_theta R_hat(theta_t)

  distinction:
    OPTIMISATION_ERROR != GENERALISATION_ERROR

ERROR_COMPONENTS:
  may_include:
    - APPROXIMATION_ERROR
    - ESTIMATION_ERROR
    - OPTIMISATION_ERROR

CROSS_VALIDATION:
  role:
    ESTIMATE_GENERALISATION
    SELECT_HYPERPARAMETERS

  requirement:
    PREPROCESSING_MUST_OCCUR_WITHIN_TRAINING_FOLD

TRAIN_VALIDATION_TEST:
  TRAIN:
    FIT_PARAMETERS

  VALIDATION:
    SELECT_HYPERPARAMETERS_AND_MODELS

  TEST:
    FINAL_GENERALISATION_ESTIMATE

DATA_LEAKAGE:
  examples:
    - FITTING_SCALER_ON_FULL_DATA
    - FEATURE_SELECTION_USING_TEST_LABELS
    - FUTURE_INFORMATION_IN_TIME_SERIES
    - DUPLICATES_ACROSS_SPLITS

  consequence:
    OPTIMISTIC_GENERALISATION_ESTIMATE

DISTRIBUTION_SHIFT:
  training:
    P_train

  deployment:
    P_deploy

  IF:
    P_train != P_deploy

  THEN:
    STANDARD_IID_GENERALISATION_MAY_FAIL

  forms:
    - COVARIATE_SHIFT
    - LABEL_SHIFT
    - CONCEPT_SHIFT

DOMAIN_ADAPTATION:
  objective:
    TRANSFER_FROM_SOURCE_DOMAIN_TO_TARGET_DOMAIN

  methods:
    - REWEIGHTING
    - REPRESENTATION_ALIGNMENT
    - FINE_TUNING

ACTIVE_LEARNING:
  objective:
    SELECT_INFORMATIVE_UNLABELLED_EXAMPLES_FOR_LABELLING

  criteria:
    - UNCERTAINTY_SAMPLING
    - EXPECTED_INFORMATION_GAIN
    - EXPECTED_MODEL_CHANGE

CALIBRATION:
  goal:
    PREDICTED_PROBABILITIES_MATCH_EMPIRICAL_FREQUENCIES
    UNDER:
      SPECIFIED_CALIBRATION_DEFINITION

UNCERTAINTY_QUANTIFICATION:
  sources:
    - ALEATORIC_UNCERTAINTY
    - EPISTEMIC_UNCERTAINTY

  methods:
    - BAYESIAN_POSTERIOR_PREDICTIVE
    - ENSEMBLES
    - BOOTSTRAP
    - CONFORMAL_PREDICTION

CONFORMAL_PREDICTION:
  purpose:
    CONSTRUCT_PREDICTION_SETS_OR_INTERVALS

  guarantee:
    FINITE_SAMPLE_MARGINAL_COVERAGE
    UNDER:
      EXCHANGEABILITY

  caution:
    MARGINAL_COVERAGE != CONDITIONAL_COVERAGE_BY_DEFAULT

CAUSALITY_DISTINCTION:
  statistical_learning:
    PRIMARILY_PREDICTIVE_OR_ASSOCIATIONAL

  causal_learning:
    REQUIRES_CAUSAL_ASSUMPTIONS_OR_IDENTIFICATION

  rule:
    PREDICTIVE_ACCURACY != CAUSAL_VALIDITY

APPLICATIONS:
  STATISTICAL_LEARNING -> SUPPORTS -> CLASSIFICATION
  STATISTICAL_LEARNING -> SUPPORTS -> REGRESSION
  STATISTICAL_LEARNING -> SUPPORTS -> FORECASTING
  STATISTICAL_LEARNING -> SUPPORTS -> FEATURE_SELECTION
  STATISTICAL_LEARNING -> SUPPORTS -> MODEL_SELECTION
  STATISTICAL_LEARNING -> SUPPORTS -> UNCERTAINTY_QUANTIFICATION
  STATISTICAL_LEARNING -> SUPPORTS -> REPRESENTATION_LEARNING

MODEL_CONNECTIONS:
  STATISTICAL_LEARNING -> CONNECTS_TO -> STATISTICS
  STATISTICAL_LEARNING -> CONNECTS_TO -> MACHINE_LEARNING
  STATISTICAL_LEARNING -> CONNECTS_TO -> PROBABILITY_THEORY
  STATISTICAL_LEARNING -> CONNECTS_TO -> INFORMATION_THEORY
  STATISTICAL_LEARNING -> CONNECTS_TO -> OPTIMISATION
  STATISTICAL_LEARNING -> CONNECTS_TO -> BAYESIAN_STATISTICS

SEMANTIC_EQUIVALENCE:
  "statistical learning"
    ~= "learning predictive or distributional structure from data using statistical principles"

  "empirical risk minimisation"
    ~= "minimisation of average training loss"

  "population risk"
    ~= "expected prediction loss under target distribution"

  "generalisation gap"
    ~= "difference between population and empirical risk"

  EXCLUDE_EQUIVALENCE:
    EMPIRICAL_RISK != POPULATION_RISK
    TRAINING_ERROR != GENERALISATION_ERROR
    OPTIMISATION_ERROR != ESTIMATION_ERROR
    PARAMETRIC != LINEAR
    NONPARAMETRIC != PARAMETER_FREE
    PREDICTION != CAUSATION
    VALIDATION_SET != TEST_SET

VALIDATION_RULES:
  RULE_1:
    DATA_GENERATING_OR_TARGET_DISTRIBUTION_MUST_BE_DEFINED_OR_STATED_AS_UNKNOWN

  RULE_2:
    LOSS_FUNCTION_MUST_MATCH_TASK_AND_TARGET

  RULE_3:
    TRAIN_VALIDATION_TEST_ROLES_MUST_BE_PRESERVED

  RULE_4:
    DATA_DEPENDENT_PREPROCESSING_MUST_NOT_LEAK_EVALUATION_INFORMATION

  RULE_5:
    GENERALISATION_CLAIMS_MUST_STATE_RELEVANT_SAMPLING_OR_DEPENDENCE_ASSUMPTIONS

  RULE_6:
    MODEL_COMPLEXITY_AND_REGULARISATION_MUST_BE_TRACKED_WHEN_RELEVANT

  RULE_7:
    DISTRIBUTION_SHIFT_MUST_BE_CONSIDERED_FOR_DEPLOYMENT

  RULE_8:
    PREDICTIVE_RELATIONSHIPS_MUST_NOT_BE_INTERPRETED_AS_CAUSAL_WITHOUT_CAUSAL_ASSUMPTIONS

CONFIDENCE:
  ontology_identity: 0.99
  risk_minimisation_structure: 0.99
  generalisation_structure: 0.99
  complexity_structure: 0.99
  statistical_model_relationship: 0.99
  machine_learning_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_TRAINING_DATA_ARRIVES
  THEN:
    UPDATE:
      DATASET_VERSION
      EMPIRICAL_RISK
      PARAMETER_ESTIMATES
    RETRAIN_OR_INCREMENTALLY_UPDATE:
      PREDICTOR
    REASSESS:
      GENERALISATION
      CALIBRATION
      UNCERTAINTY

  IF:
    HYPOTHESIS_CLASS_OR_REGULARISATION_CHANGES
  THEN:
    RETRAIN:
      MODEL
    REEVALUATE:
      EMPIRICAL_RISK
      VALIDATION_RISK
      COMPLEXITY

  IF:
    DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    DETECT:
      DISTRIBUTION_SHIFT
    REASSESS:
      GENERALISATION
      CALIBRATION
      ROBUSTNESS

  PRESERVE:
    DATASET_VERSION
    TARGET_DEFINITION
    FEATURE_DEFINITIONS
    HYPOTHESIS_CLASS
    LOSS_FUNCTION
    REGULARISATION
    TRAIN_VALIDATION_TEST_SPLITS
    MODEL_VERSION
    RANDOM_SEED_WHEN_RECORDED
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DATA_GENERATING_OR_TARGET_DISTRIBUTION
    - IDENTIFY_TRAINING_SAMPLE
    - IDENTIFY_HYPOTHESIS_CLASS
    - IDENTIFY_LOSS_FUNCTION
    - COMPUTE_EMPIRICAL_RISK
    - APPLY_REGULARISATION_IF_REQUIRED
    - FIT_PREDICTOR
    - ESTIMATE_GENERALISATION_PERFORMANCE
    - ASSESS_MODEL_COMPLEXITY
    - CHECK_DATA_LEAKAGE_AND_DISTRIBUTION_SHIFT
    - QUANTIFY_UNCERTAINTY_IF_REQUIRED
    - RETURN_STATISTICAL_LEARNING_INTERPRETATION

  constraints:
    - PRESERVE_EMPIRICAL_AND_POPULATION_RISK_DISTINCTION
    - PRESERVE_TRAIN_VALIDATION_TEST_SEPARATION
    - PREVENT_DATA_LEAKAGE
    - DISTINGUISH_OPTIMISATION_FROM_GENERALISATION
    - DISTINGUISH_PREDICTION_FROM_CAUSATION
    - EXPOSE_MODEL_COMPLEXITY_AND_REGULARISATION
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "statistical-learning"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - EMPIRICAL_RISK
    - POPULATION_RISK
    - GENERALISATION_STRUCTURE

END_FLOW