Feature Distribution Ontology

flow "FeatureDistributionOntology" v4.0

# FEATURE DISTRIBUTION ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/ai-machine-learning/feature-distribution-ontology.html"
  title: "Feature Distribution Ontology"
  domain: "Probability and Statistics"
  category: "AI & Machine Learning"
  entity: FEATURE_DISTRIBUTION
  entity_type: STATISTICAL_MACHINE_LEARNING_ONTOLOGY

DEFINE:
  FEATURE_DISTRIBUTION :=
    PROBABILITY_STRUCTURE(
      FEATURE_VARIABLES,
      MARGINAL_DISTRIBUTIONS,
      JOINT_DISTRIBUTION,
      CONDITIONAL_DISTRIBUTIONS,
      DEPENDENCE,
      DISTRIBUTION_SHIFT
    )

PURPOSE:
  REPRESENT:
    FEATURE_VALUE_DISTRIBUTIONS
    FEATURE_VARIABILITY
    FEATURE_DEPENDENCE
    TRAINING_DEPLOYMENT_DIFFERENCES
    DISTRIBUTION_SHIFT

FOUNDATION:
  FEATURE_VECTOR:
    X := (X_1,...,X_d)

  FEATURE_SPACE:
    X_space

  JOINT_DISTRIBUTION:
    P_X

  MARGINAL_DISTRIBUTION:
    P_(X_j)

  CONDITIONAL_DISTRIBUTION:
    P(X_j | X_-j)

ENTITIES:
  FEATURE: MODEL_INPUT_VARIABLE
  FEATURE_VECTOR: MULTIVARIATE_INPUT
  FEATURE_SPACE: DOMAIN_OF_INPUTS
  FEATURE_DISTRIBUTION: PROBABILITY_DISTRIBUTION_OVER_FEATURES
  MARGINAL_DISTRIBUTION: SINGLE_FEATURE_DISTRIBUTION
  JOINT_DISTRIBUTION: MULTIVARIATE_FEATURE_DISTRIBUTION
  CONDITIONAL_DISTRIBUTION: CONDITIONED_FEATURE_DISTRIBUTION
  TRAINING_DISTRIBUTION: FEATURE_DISTRIBUTION_IN_TRAINING_DATA
  DEPLOYMENT_DISTRIBUTION: FEATURE_DISTRIBUTION_IN_OPERATION
  DISTRIBUTION_SHIFT: CHANGE_BETWEEN_DISTRIBUTIONS
  OUTLIER: EXTREME_OR_LOW_DENSITY_OBSERVATION

RELATIONSHIPS:
  DATASET -> CONTAINS -> FEATURE_VALUES
  FEATURE_VECTOR -> CONTAINS -> FEATURES
  FEATURE -> HAS -> MARGINAL_DISTRIBUTION
  FEATURE_VECTOR -> HAS -> JOINT_DISTRIBUTION
  FEATURE_DEPENDENCE -> SHAPES -> JOINT_DISTRIBUTION
  TRAINING_DATA -> ESTIMATES -> TRAINING_DISTRIBUTION
  DEPLOYMENT_DATA -> ESTIMATES -> DEPLOYMENT_DISTRIBUTION
  DISTRIBUTION_SHIFT -> COMPARES -> REFERENCE_AND_CURRENT_DISTRIBUTIONS

FEATURE_TYPES:
  NUMERICAL:
    - CONTINUOUS
    - DISCRETE
  CATEGORICAL:
    - NOMINAL
    - ORDINAL
    - BINARY
  STRUCTURED:
    - TEMPORAL
    - SPATIAL
    - EMBEDDING
    - TEXT_DERIVED
    - IMAGE_DERIVED

MARGINAL_DISTRIBUTION:
  feature: X_j
  distribution: p(x_j)

JOINT_DISTRIBUTION:
  distribution:
    p(x_1,...,x_d)

  purpose:
    REPRESENT_FEATURE_VALUES_DEPENDENCE_AND_INTERACTIONS

CONDITIONAL_DISTRIBUTION:
  form:
    p(x_j | x_-j)

INDEPENDENCE:
  IF:
    FEATURES_MUTUALLY_INDEPENDENT
  THEN:
    p(x_1,...,x_d)
      =
    PRODUCT[j=1..d] p(x_j)

  caution:
    ZERO_CORRELATION != INDEPENDENCE_BY_DEFAULT

EMPIRICAL_DISTRIBUTION:
  sample:
    x_1,...,x_n

  empirical_measure:
    P_hat_n :=
      (1/n)
      * SUM[i=1..n] delta_(x_i)

SUMMARY_STATISTICS:
  numerical:
    - MEAN
    - MEDIAN
    - VARIANCE
    - STANDARD_DEVIATION
    - QUANTILES
    - SKEWNESS
    - KURTOSIS

  categorical:
    - FREQUENCY
    - PROPORTION
    - MODE
    - ENTROPY

COVARIANCE_MATRIX:
  Sigma :=
    E[
      (X-E[X])
      (X-E[X])^T
    ]

  role:
    REPRESENT_LINEAR_VARIABILITY_AND_DEPENDENCE

CORRELATION_MATRIX:
  rho_ij :=
    Cov(X_i,X_j)/(sigma_i*sigma_j)

  caution:
    DOES_NOT_CAPTURE_ALL_NONLINEAR_DEPENDENCE

MUTUAL_INFORMATION:
  I(X_i;X_j):
    MEASURES_GENERAL_STATISTICAL_DEPENDENCE

  independence:
    I(X_i;X_j)=0
    IFF:
      X_i_INDEPENDENT_X_j

FEATURE_ENTROPY:
  H(X_j) :=
    -SUM_x p(x) log p(x)

  role:
    QUANTIFY_FEATURE_UNCERTAINTY

NORMAL_DISTRIBUTION:
  X ~ N(mu,sigma_squared)

  caution:
    NORMALITY_MUST_NOT_BE_ASSUMED_WITHOUT_JUSTIFICATION

MULTIVARIATE_NORMAL:
  X ~ N_d(mu,Sigma)

  parameters:
    mu: MEAN_VECTOR
    Sigma: COVARIANCE_MATRIX

HEAVY_TAILED_FEATURE:
  property:
    EXTREME_VALUES_MORE_PROBABLE_THAN_LIGHT_TAILED_REFERENCE

MULTIMODAL_FEATURE:
  property:
    MULTIPLE_MODES

  possible_causes:
    - LATENT_SUBPOPULATIONS
    - MIXTURE_PROCESSES
    - SEASONALITY
    - DATA_COLLECTION_DIFFERENCES

ZERO_INFLATED_FEATURE:
  property:
    EXCESS_MASS_AT_ZERO

MISSING_FEATURE_VALUES:
  mechanisms:
    - MCAR
    - MAR
    - MNAR

  impact:
    OBSERVED_DISTRIBUTION_MAY_DIFFER_FROM_COMPLETE_DATA_DISTRIBUTION

OUTLIERS:
  definition:
    EXTREME_OR_LOW_DENSITY_OBSERVATIONS_RELATIVE_TO_REFERENCE

  caution:
    OUTLIER_MAY_BE:
      ERROR
      RARE_VALID_CASE
      DISTRIBUTION_SHIFT
      NOVEL_SUBPOPULATION

FEATURE_SCALING:
  STANDARDISATION:
    z := (x-mu)/sigma

  MIN_MAX:
    x_scaled := (x-min)/(max-min)

  ROBUST_SCALING:
    MAY_USE_MEDIAN_AND_INTERQUARTILE_RANGE

  requirement:
    SCALING_PARAMETERS_FIT_ON_TRAINING_DATA_ONLY

FEATURE_TRANSFORMATION:
  examples:
    - LOG_TRANSFORM
    - POWER_TRANSFORM
    - QUANTILE_TRANSFORM
    - NORMALISATION

  effect:
    CHANGES_FEATURE_DISTRIBUTION_AND_MODEL_INPUT_GEOMETRY

TRAINING_DISTRIBUTION:
  P_train(X)

VALIDATION_DISTRIBUTION:
  P_validation(X)

TEST_DISTRIBUTION:
  P_test(X)

DEPLOYMENT_DISTRIBUTION:
  P_deploy(X)

COVARIATE_SHIFT:
  condition:
    P_train(X) != P_deploy(X)

  standard_assumption:
    P_train(Y|X)
      approximately_equals
    P_deploy(Y|X)

LABEL_SHIFT_CONNECTION:
  condition:
    P_train(Y) != P_deploy(Y)

  standard_assumption:
    P_train(X|Y)
      approximately_equals
    P_deploy(X|Y)

CONCEPT_SHIFT:
  condition:
    P_train(Y|X) != P_deploy(Y|X)

  caution:
    FEATURE_DISTRIBUTION_MONITORING_ALONE_MAY_NOT_DETECT_TARGET_RELATIONSHIP_CHANGE

DISTRIBUTION_SHIFT:
  general:
    P_reference != P_current

  forms:
    - COVARIATE_SHIFT
    - LABEL_SHIFT
    - CONCEPT_SHIFT
    - SUPPORT_SHIFT
    - SUBPOPULATION_SHIFT

SUPPORT_SHIFT:
  condition:
    CURRENT_DATA_OCCUPIES_FEATURE_REGIONS_ABSENT_OR_RARE_IN_TRAINING

  risk:
    MODEL_EXTRAPOLATION

OUT_OF_DISTRIBUTION:
  condition:
    LOW_SUPPORT_OR_INCONSISTENT_WITH_TRAINING_DISTRIBUTION

  caution:
    OOD_DEFINITION_IS_MODEL_AND_CONTEXT_DEPENDENT

DISTRIBUTION_COMPARISON:
  univariate:
    - HISTOGRAM_COMPARISON
    - EMPIRICAL_CDF
    - KOLMOGOROV_SMIRNOV
    - WASSERSTEIN_DISTANCE
    - POPULATION_STABILITY_INDEX

  multivariate:
    - ENERGY_DISTANCE
    - MAXIMUM_MEAN_DISCREPANCY
    - CLASSIFIER_TWO_SAMPLE_TEST
    - MULTIVARIATE_DIVERGENCE_ESTIMATION

KL_DIVERGENCE:
  role:
    DISTRIBUTIONAL_DIVERGENCE

  caution:
    ASYMMETRIC
    MAY_BE_INFINITE
    NOT_A_METRIC

JENSEN_SHANNON_DIVERGENCE:
  role:
    SYMMETRIC_DIVERGENCE_BUILT_FROM_KL

WASSERSTEIN_DISTANCE:
  role:
    COMPARE_DISTRIBUTIONS_BY_TRANSPORT_COST

KOLMOGOROV_SMIRNOV:
  statistic:
    D :=
      sup_x |F_reference(x)-F_current(x)|

MAXIMUM_MEAN_DISCREPANCY:
  alias:
    MMD

  role:
    COMPARE_DISTRIBUTIONS_IN_KERNEL_FEATURE_SPACE

DRIFT_MONITORING:
  flow:
    COLLECT_REFERENCE_FEATURES
      -> ESTIMATE_REFERENCE_DISTRIBUTION
      -> COLLECT_CURRENT_FEATURES
      -> ESTIMATE_CURRENT_DISTRIBUTION
      -> COMPARE
      -> FLAG_MATERIAL_CHANGE
      -> INVESTIGATE_MODEL_IMPACT

  caution:
    STATISTICAL_SHIFT_DOES_NOT_ALWAYS_IMPLY_MATERIAL_MODEL_DEGRADATION

FEATURE_DEPENDENCE_SHIFT:
  issue:
    MARGINALS_MAY_APPEAR_STABLE_WHILE_DEPENDENCE_STRUCTURE_CHANGES

  methods:
    - CORRELATION_MATRIX_COMPARISON
    - MUTUAL_INFORMATION_COMPARISON
    - COPULA_ANALYSIS
    - MULTIVARIATE_TWO_SAMPLE_TESTS

CLASS_CONDITIONAL_FEATURE_DISTRIBUTION:
  form:
    P(X|Y)

  uses:
    - CLASSIFICATION_ANALYSIS
    - LABEL_SHIFT_ANALYSIS
    - FAIRNESS_DIAGNOSTICS

FEATURE_TARGET_RELATION:
  conditional:
    P(Y|X)

  caution:
    P(X)_ALONE_DOES_NOT_DEFINE_P(Y|X)

FEATURE_IMPORTANCE_CONNECTION:
  feature_distribution:
    INFLUENCES_OBSERVED_FEATURE_IMPORTANCE

  caution:
    IMPORTANCE_MAY_CHANGE_UNDER_DISTRIBUTION_SHIFT_OR_CORRELATED_FEATURES

FEATURE_ENGINEERING:
  transformation:
    Z := g(X)

  implication:
    FEATURE_ENGINEERING_INDUCES_NEW_FEATURE_DISTRIBUTION

EMBEDDING_DISTRIBUTION:
  embedding:
    Z IN R^d

  properties:
    HIGH_DIMENSIONAL
    CONTINUOUS
    DEPENDENCE_RICH

DIMENSIONALITY_REDUCTION:
  transformation:
    Z := g(X)

  methods:
    - PCA
    - AUTOENCODER
    - MANIFOLD_METHOD

PCA_CONNECTION:
  covariance_matrix:
    Sigma

  role:
    REPRESENT_MAJOR_LINEAR_VARIATION

GENERATIVE_MODELLING_CONNECTION:
  objective:
    LEARN_P_data(X)

  evaluation:
    COMPARE_P_model_WITH_P_data

ANOMALY_DETECTION:
  objective:
    IDENTIFY_LOW_PROBABILITY_LOW_DENSITY_OR_STRUCTURALLY_UNUSUAL_FEATURE_VECTORS

  methods:
    - DENSITY_MODELS
    - ISOLATION_FOREST
    - ONE_CLASS_METHODS
    - DISTANCE_BASED_METHODS

  caution:
    ANOMALY_SCORE != ERROR_OR_FRAUD_BY_DEFAULT

DATA_QUALITY:
  checks:
    - VALID_RANGE
    - VALID_CATEGORY
    - MISSINGNESS_RATE
    - OUTLIER_RATE
    - DUPLICATE_RATE
    - DISTRIBUTION_SHIFT

MODEL_CONNECTIONS:
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> MACHINE_LEARNING
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> STATISTICS
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> PROBABILITY_THEORY
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> COVARIANCE_MATRIX
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> MUTUAL_INFORMATION
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> DISTRIBUTION_SHIFT
  FEATURE_DISTRIBUTION -> CONNECTS_TO -> ANOMALY_DETECTION

APPLICATIONS:
  FEATURE_DISTRIBUTION -> SUPPORTS -> DATA_EXPLORATION
  FEATURE_DISTRIBUTION -> SUPPORTS -> FEATURE_ENGINEERING
  FEATURE_DISTRIBUTION -> SUPPORTS -> MODEL_VALIDATION
  FEATURE_DISTRIBUTION -> SUPPORTS -> DRIFT_MONITORING
  FEATURE_DISTRIBUTION -> SUPPORTS -> OUTLIER_DETECTION
  FEATURE_DISTRIBUTION -> SUPPORTS -> DOMAIN_ADAPTATION
  FEATURE_DISTRIBUTION -> SUPPORTS -> GENERATIVE_MODEL_EVALUATION

SEMANTIC_EQUIVALENCE:
  "feature distribution"
    ~= "probability distribution of model input variables"

  "marginal feature distribution"
    ~= "distribution of an individual feature"

  "joint feature distribution"
    ~= "multivariate distribution over feature vector"

  EXCLUDE_EQUIVALENCE:
    FEATURE_DISTRIBUTION != TARGET_DISTRIBUTION
    MARGINAL_DISTRIBUTION != JOINT_DISTRIBUTION
    CORRELATION != DEPENDENCE_BY_DEFAULT
    DISTRIBUTION_SHIFT != CONCEPT_SHIFT_BY_DEFAULT
    OUTLIER != ERROR
    STATISTICAL_DRIFT != MODEL_FAILURE

VALIDATION_RULES:
  RULE_1:
    FEATURE_VARIABLES_AND_DOMAINS_MUST_BE_DEFINED

  RULE_2:
    PROBABILITY_DISTRIBUTIONS_MUST_BE_NORMALISED

  RULE_3:
    TRAINING_AND_DEPLOYMENT_DISTRIBUTIONS_MUST_NOT_BE_CONFLATED

  RULE_4:
    FEATURE_TRANSFORMATIONS_MUST_BE_RECORDED_AND_REPRODUCIBLE

  RULE_5:
    SCALING_AND_PREPROCESSING_PARAMETERS_MUST_BE_FIT_ON_TRAINING_DATA

  RULE_6:
    DISTRIBUTION_SHIFT_METRIC_MUST_MATCH_FEATURE_TYPE_AND_DIMENSION

  RULE_7:
    MARGINAL_STABILITY_MUST_NOT_BE_ASSUMED_TO_IMPLY_JOINT_STABILITY

  RULE_8:
    FEATURE_DISTRIBUTION_CHANGE_MUST_NOT_BE_INTERPRETED_AS_CAUSAL_CHANGE_BY_DEFAULT

CONFIDENCE:
  ontology_identity: 0.99
  marginal_joint_structure: 0.99
  dependence_structure: 0.99
  distribution_shift_structure: 0.99
  drift_monitoring_structure: 0.99
  machine_learning_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_TRAINING_DATA_ARRIVES
  THEN:
    UPDATE:
      EMPIRICAL_FEATURE_DISTRIBUTIONS
      SUMMARY_STATISTICS
      COVARIANCE_STRUCTURE
      MISSINGNESS_RATES

  IF:
    NEW_DEPLOYMENT_DATA_ARRIVES
  THEN:
    UPDATE:
      CURRENT_FEATURE_DISTRIBUTIONS
    COMPARE_WITH:
      TRAINING_DISTRIBUTIONS
    ASSESS:
      DISTRIBUTION_SHIFT
      SUPPORT_SHIFT
      DEPENDENCE_SHIFT

  IF:
    FEATURE_ENGINEERING_CHANGES
  THEN:
    UPDATE:
      FEATURE_DEFINITIONS
      TRANSFORMATIONS
      DISTRIBUTIONS

  PRESERVE:
    FEATURE_IDENTITIES
    FEATURE_TYPES
    FEATURE_DOMAINS
    TRAINING_REFERENCE_DISTRIBUTION
    TRANSFORMATION_PARAMETERS
    MODEL_VERSION
    DATASET_VERSION
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_FEATURES_AND_FEATURE_TYPES
    - IDENTIFY_FEATURE_DOMAINS
    - ESTIMATE_MARGINAL_DISTRIBUTIONS
    - ESTIMATE_JOINT_OR_DEPENDENCE_STRUCTURE
    - COMPUTE_SUMMARY_STATISTICS
    - IDENTIFY_REFERENCE_DISTRIBUTION
    - IDENTIFY_CURRENT_OR_DEPLOYMENT_DISTRIBUTION
    - COMPARE_DISTRIBUTIONS
    - DETECT_SUPPORT_AND_DEPENDENCE_SHIFT
    - ASSESS_MODEL_RELEVANCE_OF_SHIFT
    - RETURN_FEATURE_DISTRIBUTION_INTERPRETATION

  constraints:
    - PRESERVE_FEATURE_IDENTITIES_AND_TYPES
    - PRESERVE_REFERENCE_CURRENT_DISTINCTION
    - DISTINGUISH_MARGINAL_FROM_JOINT_DISTRIBUTION
    - DISTINGUISH_CORRELATION_FROM_GENERAL_DEPENDENCE
    - DO_NOT_EQUATE_DRIFT_WITH_MODEL_FAILURE
    - DO_NOT_EQUATE_OUTLIER_WITH_ERROR
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "feature-distribution"
  namespace: "probability-statistics/ai-machine-learning"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - FEATURE_VECTOR
    - MARGINAL_JOINT_DISTRIBUTIONS
    - DISTRIBUTION_SHIFT

END_FLOW