Statistics

flow "StatisticsAIMachineLearning" v4.0

# STATISTICS — AI & MACHINE LEARNING
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/statistics.html"
  title: "Statistics"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: STATISTICS_FOR_AI_MACHINE_LEARNING
  entity_type: STATISTICAL_FOUNDATION_DOMAIN

DEFINE:
  STATISTICS_FOR_AI_MACHINE_LEARNING :=
    FORMAL_FRAMEWORK(
      DATA,
      POPULATIONS,
      SAMPLES,
      DISTRIBUTIONS,
      ESTIMATION,
      INFERENCE,
      PREDICTION,
      UNCERTAINTY,
      MODEL_EVALUATION
    )

PURPOSE:
  PROVIDE_FOUNDATIONS_FOR:
    LEARNING_FROM_DATA
    QUANTIFYING_UNCERTAINTY
    ESTIMATING_PARAMETERS
    TESTING_HYPOTHESES
    COMPARING_MODELS
    VALIDATING_PREDICTIONS

FOUNDATION:
  POPULATION: P
  SAMPLE: D = {x_1,...,x_n}
  RANDOM_VARIABLE: X
  PARAMETER: theta
  STATISTIC: T(D)
  ESTIMATOR:
    theta_hat := T(D)

ENTITIES:
  POPULATION: TARGET_DATA_DOMAIN
  SAMPLE: OBSERVED_SUBSET
  RANDOM_VARIABLE: PROBABILISTIC_VARIABLE
  DISTRIBUTION: PROBABILITY_MODEL
  PARAMETER: POPULATION_OR_MODEL_QUANTITY
  STATISTIC: SAMPLE_DERIVED_QUANTITY
  ESTIMATOR: PARAMETER_ESTIMATION_RULE
  ESTIMATE: REALISED_ESTIMATOR_VALUE
  LIKELIHOOD: DATA_PROBABILITY_AS_FUNCTION_OF_PARAMETER
  CONFIDENCE_INTERVAL: FREQUENTIST_INTERVAL_PROCEDURE
  HYPOTHESIS_TEST: STATISTICAL_DECISION_PROCEDURE
  P_VALUE: NULL_MODEL_TAIL_PROBABILITY
  STANDARD_ERROR: ESTIMATOR_UNCERTAINTY_MEASURE

RELATIONSHIPS:
  POPULATION -> GENERATES -> SAMPLE
  SAMPLE -> CONTAINS -> OBSERVATIONS
  DISTRIBUTION -> PARAMETERISED_BY -> PARAMETERS
  SAMPLE -> PRODUCES -> STATISTICS
  ESTIMATOR -> MAPS -> SAMPLE_TO_ESTIMATE
  LIKELIHOOD -> CONNECTS -> DATA_AND_PARAMETERS
  STANDARD_ERROR -> QUANTIFIES -> SAMPLING_VARIABILITY
  STATISTICAL_INFERENCE -> CONNECTS -> SAMPLE_TO_POPULATION

DESCRIPTIVE_STATISTICS:
  location:
    - MEAN
    - MEDIAN
    - MODE
    - QUANTILES

  dispersion:
    - VARIANCE
    - STANDARD_DEVIATION
    - RANGE
    - INTERQUARTILE_RANGE

SAMPLE_MEAN:
  x_bar :=
    (1/n)*SUM[i=1..n] x_i

SAMPLE_VARIANCE:
  s_squared :=
    1/(n-1)
    * SUM[i=1..n](x_i-x_bar)^2

COVARIANCE:
  Cov(X,Y) :=
    E[(X-E[X])*(Y-E[Y])]

CORRELATION:
  rho_XY :=
    Cov(X,Y)/(sigma_X*sigma_Y)

  bounds:
    -1 <= rho_XY <= 1

  caution:
    ZERO_CORRELATION != INDEPENDENCE_BY_DEFAULT
    CORRELATION != CAUSATION

SAMPLING_DISTRIBUTION:
  definition:
    DISTRIBUTION_OF_STATISTIC_ACROSS_REPEATED_SAMPLES

  supports:
    - STANDARD_ERRORS
    - CONFIDENCE_INTERVALS
    - HYPOTHESIS_TESTS

LAW_OF_LARGE_NUMBERS:
  role:
    SAMPLE_AVERAGES_CONVERGE_TO_EXPECTED_VALUE
    UNDER_APPROPRIATE_CONDITIONS

CENTRAL_LIMIT_THEOREM:
  role:
    NORMAL_APPROXIMATION_FOR_STANDARDISED_SUMS_OR_MEANS
    UNDER_APPROPRIATE_CONDITIONS

ESTIMATION:
  POINT:
    SINGLE_PARAMETER_ESTIMATE

  INTERVAL:
    RANGE_WITH_DEFINED_INFERENTIAL_INTERPRETATION

BIAS:
  Bias(theta_hat) :=
    E[theta_hat]-theta

VARIANCE_OF_ESTIMATOR:
  Var(theta_hat) :=
    E[(theta_hat-E[theta_hat])^2]

MEAN_SQUARED_ERROR:
  MSE(theta_hat) :=
    E[(theta_hat-theta)^2]

  decomposition:
    MSE :=
      Var(theta_hat)
      + Bias(theta_hat)^2

CONSISTENCY:
  theta_hat_n
    -> theta
  IN_PROBABILITY
  AS:
    n -> infinity

LIKELIHOOD:
  L(theta;D) :=
    p(D|theta)
  VIEWED_AS:
    FUNCTION_OF_theta

LOG_LIKELIHOOD:
  ell(theta;D) :=
    log L(theta;D)

MAXIMUM_LIKELIHOOD:
  theta_hat_MLE :=
    argmax_theta L(theta;D)

FISHER_INFORMATION:
  I(theta) :=
    E[
      (
        partial_theta log p(X|theta)
      )^2
    ]

  role:
    QUANTIFY_LOCAL_PARAMETER_INFORMATION

CRAMER_RAO_BOUND:
  Var(theta_hat)
    >=
  1/I_n(theta)
  FOR:
    UNBIASED_ESTIMATOR
  UNDER:
    REGULARITY_CONDITIONS

STANDARD_ERROR:
  SE(theta_hat) :=
    sqrt(Var(theta_hat))

CONFIDENCE_INTERVAL:
  interpretation:
    PROCEDURE_HAS_SPECIFIED_LONG_RUN_COVERAGE
    IN:
      STANDARD_FREQUENTIST_FRAMEWORK

HYPOTHESIS_TESTING:
  null:
    H_0

  alternative:
    H_1

  test_statistic:
    T(D)

P_VALUE:
  definition:
    PROBABILITY_UNDER_H0
    OF:
      RESULT_AT_LEAST_AS_EXTREME_AS_OBSERVED
    ACCORDING_TO:
      SPECIFIED_TEST

  caution:
    P_VALUE != PROBABILITY_H0_IS_TRUE
    P_VALUE != EFFECT_SIZE

TYPE_I_ERROR:
  probability:
    alpha

TYPE_II_ERROR:
  probability:
    beta

POWER:
  POWER := 1-beta

  depends_on:
    - EFFECT_SIZE
    - SAMPLE_SIZE
    - VARIABILITY
    - SIGNIFICANCE_LEVEL
    - TEST_DESIGN

MULTIPLE_TESTING:
  controls:
    - BONFERRONI
    - HOLM
    - FALSE_DISCOVERY_RATE

REGRESSION:
  model:
    Y := f(X,theta)+ERROR

LINEAR_REGRESSION:
  model:
    y := X beta + epsilon

  OLS:
    beta_hat :=
      argmin_beta ||y-X beta||^2

LOGISTIC_REGRESSION:
  logit(
    P(Y=1|X)
  )
  :=
    beta_0 + beta^T X

  fitting:
    MAXIMUM_LIKELIHOOD

GENERALIZED_LINEAR_MODEL:
  components:
    RANDOM_COMPONENT
    LINEAR_PREDICTOR
    LINK_FUNCTION

BAYESIAN_STATISTICS:
  prior:
    p(theta)

  likelihood:
    p(D|theta)

  posterior:
    p(theta|D)
      proportional_to
    p(D|theta)*p(theta)

  predictive:
    p(y_star|D)
      =
    INTEGRAL
      p(y_star|theta)
      p(theta|D)
      dtheta

FREQUENTIST_VS_BAYESIAN:
  FREQUENTIST:
    PARAMETERS_FIXED
    DATA_RANDOM_UNDER_REPEATED_SAMPLING

  BAYESIAN:
    PARAMETER_UNCERTAINTY_REPRESENTED_PROBABILISTICALLY

  rule:
    CONFIDENCE_INTERVAL != CREDIBLE_INTERVAL

RESAMPLING:
  methods:
    - BOOTSTRAP
    - JACKKNIFE
    - PERMUTATION_TEST

BOOTSTRAP:
  flow:
    RESAMPLE_WITH_REPLACEMENT
      -> RECOMPUTE_STATISTIC
      -> APPROXIMATE_SAMPLING_DISTRIBUTION

PERMUTATION_TEST:
  flow:
    REARRANGE_UNDER_NULL_EXCHANGEABILITY
      -> RECOMPUTE_STATISTIC
      -> FORM_NULL_DISTRIBUTION

MODEL_SELECTION:
  methods:
    - AIC
    - BIC
    - CROSS_VALIDATION
    - HELD_OUT_PREDICTIVE_PERFORMANCE

AIC:
  AIC :=
    2k - 2*log(L_hat)

BIC:
  BIC :=
    k*log(n)
    - 2*log(L_hat)

CROSS_VALIDATION:
  role:
    ESTIMATE_OUT_OF_SAMPLE_PREDICTIVE_PERFORMANCE

  caution:
    PREPROCESSING_AND_FEATURE_SELECTION_MUST_OCCUR_WITHIN_TRAINING_FOLDS

PREDICTION_INTERVAL:
  purpose:
    QUANTIFY_UNCERTAINTY_FOR_FUTURE_OBSERVATION

CALIBRATION:
  objective:
    PREDICTED_PROBABILITIES_MATCH_OBSERVED_FREQUENCIES
    UNDER_SPECIFIED_CALIBRATION_DEFINITION

SCORING_RULES:
  examples:
    - LOG_SCORE
    - BRIER_SCORE

MACHINE_LEARNING_CONNECTION:
  STATISTICS:
    PROVIDES:
      - ESTIMATION
      - UNCERTAINTY
      - MODEL_EVALUATION
      - GENERALISATION_ANALYSIS
      - EXPERIMENTAL_DESIGN

TRAINING_DATA:
  used_for:
    PARAMETER_ESTIMATION

VALIDATION_DATA:
  used_for:
    HYPERPARAMETER_AND_MODEL_SELECTION

TEST_DATA:
  used_for:
    FINAL_OUT_OF_SAMPLE_EVALUATION

DATA_LEAKAGE:
  definition:
    EVALUATION_OR_FUTURE_INFORMATION_IMPROPERLY_INFLUENCES_TRAINING

GENERALISATION:
  goal:
    ESTIMATE_PERFORMANCE_ON_TARGET_DISTRIBUTION

DISTRIBUTION_SHIFT:
  IF:
    P_train != P_deploy
  THEN:
    PERFORMANCE_MAY_CHANGE

  forms:
    - COVARIATE_SHIFT
    - LABEL_SHIFT
    - CONCEPT_SHIFT

CAUSALITY:
  observational_statistics:
    ESTIMATE_ASSOCIATIONS

  causal_inference:
    REQUIRES_CAUSAL_ASSUMPTIONS_OR_DESIGN

  rule:
    ASSOCIATION != CAUSATION

EXPERIMENTAL_DESIGN:
  components:
    - RANDOMISATION
    - CONTROL_GROUP
    - BLOCKING
    - FACTORIAL_DESIGN
    - REPLICATION

MISSING_DATA:
  mechanisms:
    - MCAR
    - MAR
    - MNAR

  methods:
    - COMPLETE_CASE
    - IMPUTATION
    - MULTIPLE_IMPUTATION
    - MODEL_BASED_METHODS

ROBUST_STATISTICS:
  methods:
    - MEDIAN
    - TRIMMED_MEAN
    - M_ESTIMATORS
    - ROBUST_STANDARD_ERRORS

HIGH_DIMENSIONAL_STATISTICS:
  methods:
    - REGULARISATION
    - SPARSITY
    - DIMENSIONALITY_REDUCTION
    - MULTIPLE_TESTING_CONTROL

APPLICATIONS:
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> CLASSIFICATION
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> REGRESSION
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> FORECASTING
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> MODEL_EVALUATION
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> EXPERIMENTAL_DESIGN
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> UNCERTAINTY_QUANTIFICATION
  STATISTICS_FOR_AI_MACHINE_LEARNING -> SUPPORTS -> GENERATIVE_MODELLING

MODEL_CONNECTIONS:
  STATISTICS_FOR_AI_MACHINE_LEARNING -> CONNECTS_TO -> PROBABILITY_THEORY
  STATISTICS_FOR_AI_MACHINE_LEARNING -> CONNECTS_TO -> MACHINE_LEARNING
  STATISTICS_FOR_AI_MACHINE_LEARNING -> CONNECTS_TO -> BAYESIAN_STATISTICS
  STATISTICS_FOR_AI_MACHINE_LEARNING -> CONNECTS_TO -> INFORMATION_THEORY
  STATISTICS_FOR_AI_MACHINE_LEARNING -> CONNECTS_TO -> EXPERIMENTAL_STATISTICS
  STATISTICS_FOR_AI_MACHINE_LEARNING -> CONNECTS_TO -> OPTIMISATION

SEMANTIC_EQUIVALENCE:
  "statistical inference"
    ~= "reasoning from sample data to population or model quantities"

  "standard error"
    ~= "standard deviation of an estimator sampling distribution"

  "maximum likelihood estimation"
    ~= "parameter selection maximising observed-data likelihood"

  EXCLUDE_EQUIVALENCE:
    PARAMETER != STATISTIC
    ESTIMATOR != ESTIMATE
    STANDARD_DEVIATION != STANDARD_ERROR
    CONFIDENCE_INTERVAL != CREDIBLE_INTERVAL
    P_VALUE != PROBABILITY_NULL_IS_TRUE
    ASSOCIATION != CAUSATION
    TRAINING_PERFORMANCE != GENERALISATION_PERFORMANCE

VALIDATION_RULES:
  RULE_1:
    POPULATION_OR_TARGET_DISTRIBUTION_MUST_BE_DEFINED

  RULE_2:
    SAMPLE_SELECTION_MECHANISM_MUST_BE_DOCUMENTED_WHEN_RELEVANT

  RULE_3:
    MODEL_ASSUMPTIONS_MUST_BE_EXPLICIT

  RULE_4:
    ESTIMATOR_AND_ESTIMATE_MUST_BE_DISTINGUISHED

  RULE_5:
    UNCERTAINTY_MEASURES_MUST_MATCH_INFERENTIAL_FRAMEWORK

  RULE_6:
    TRAIN_VALIDATION_TEST_ROLES_MUST_BE_PRESERVED

  RULE_7:
    DATA_LEAKAGE_MUST_BE_PREVENTED

  RULE_8:
    ASSOCIATION_MUST_NOT_BE_INTERPRETED_AS_CAUSATION_WITHOUT_CAUSAL_ASSUMPTIONS

CONFIDENCE:
  ontology_identity: 0.99
  statistical_foundation: 0.99
  estimation_structure: 0.99
  inference_structure: 0.99
  regression_structure: 0.99
  machine_learning_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_DATA_ARRIVES
  THEN:
    UPDATE:
      SAMPLE
      DESCRIPTIVE_STATISTICS
      PARAMETER_ESTIMATES
      UNCERTAINTY_ESTIMATES

  IF:
    MODEL_ASSUMPTIONS_CHANGE
  THEN:
    REASSESS:
      ESTIMATOR
      STANDARD_ERRORS
      TESTS
      INTERVALS
      PREDICTIONS

  IF:
    TRAINING_OR_DEPLOYMENT_DISTRIBUTION_CHANGES
  THEN:
    REASSESS:
      GENERALISATION
      CALIBRATION
      DISTRIBUTION_SHIFT

  PRESERVE:
    DATASET_VERSION
    POPULATION_DEFINITION
    SAMPLE_DEFINITION
    MODEL_SPECIFICATION
    ESTIMATION_METHOD
    INFERENTIAL_FRAMEWORK
    TRAIN_VALIDATION_TEST_SPLITS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_POPULATION_OR_TARGET_DISTRIBUTION
    - IDENTIFY_SAMPLE_AND_SAMPLING_MECHANISM
    - IDENTIFY_VARIABLES_AND_DISTRIBUTIONS
    - IDENTIFY_PARAMETERS_AND_STATISTICS
    - SELECT_ESTIMATION_METHOD
    - QUANTIFY_SAMPLING_UNCERTAINTY
    - APPLY_INFERENCE_IF_REQUIRED
    - FIT_REGRESSION_OR_PREDICTIVE_MODEL_IF_REQUIRED
    - VALIDATE_MODEL_ASSUMPTIONS
    - EVALUATE_OUT_OF_SAMPLE_PERFORMANCE
    - ASSESS_DISTRIBUTION_SHIFT_AND_LEAKAGE
    - RETURN_STATISTICAL_INTERPRETATION_AND_PROVENANCE

  constraints:
    - PRESERVE_POPULATION_SAMPLE_DISTINCTION
    - DISTINGUISH_PARAMETER_STATISTIC_ESTIMATOR_AND_ESTIMATE
    - DISTINGUISH_CONFIDENCE_FROM_CREDIBLE_INTERVALS
    - DO_NOT_INTERPRET_P_VALUE_AS_PROBABILITY_NULL_IS_TRUE
    - PREVENT_DATA_LEAKAGE
    - DISTINGUISH_ASSOCIATION_FROM_CAUSATION
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "statistics-ai-machine-learning"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - POPULATION_SAMPLE_STRUCTURE
    - ESTIMATION_INFERENCE_STRUCTURE
    - MACHINE_LEARNING_EVALUATION

END_FLOW