Mutual Information Ontology

flow "MutualInformationOntology" v4.0

# MUTUAL INFORMATION ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory/mutual-information-ontology.html"
  title: "Mutual Information Ontology"
  domain: "Probability and Statistics"
  category: "Information Theory"
  entity: MUTUAL_INFORMATION
  entity_type: INFORMATION_THEORETIC_DEPENDENCE_MEASURE

DEFINE:
  MUTUAL_INFORMATION :=
    INFORMATION_MEASURE(
      JOINT_DISTRIBUTION,
      MARGINAL_DISTRIBUTIONS,
      DEPENDENCE,
      UNCERTAINTY_REDUCTION
    )

PURPOSE:
  QUANTIFY:
    INFORMATION_SHARED_BETWEEN_VARIABLES
    STATISTICAL_DEPENDENCE
    REDUCTION_IN_UNCERTAINTY
  WITHOUT:
    REQUIRING_LINEAR_RELATIONSHIP

FOUNDATION:
  RANDOM_VARIABLES:
    X
    Y

  JOINT_DISTRIBUTION:
    p(x,y)

  MARGINALS:
    p(x)
    p(y)

  MUTUAL_INFORMATION:
    I(X;Y) :=
      SUM_x SUM_y
        p(x,y)
        log_b[
          p(x,y)/(p(x)p(y))
        ]

ENTITIES:
  MUTUAL_INFORMATION: DEPENDENCE_INFORMATION
  RANDOM_VARIABLE: PROBABILISTIC_VARIABLE
  JOINT_DISTRIBUTION: MULTIVARIATE_PROBABILITY_DISTRIBUTION
  MARGINAL_DISTRIBUTION: SINGLE_VARIABLE_DISTRIBUTION
  ENTROPY: UNCERTAINTY_MEASURE
  CONDITIONAL_ENTROPY: RESIDUAL_UNCERTAINTY
  KL_DIVERGENCE: DISTRIBUTIONAL_DIVERGENCE
  CONDITIONAL_MUTUAL_INFORMATION: CONDITIONAL_DEPENDENCE_MEASURE
  INFORMATION_GAIN: UNCERTAINTY_REDUCTION
  CHANNEL: STOCHASTIC_MAPPING

RELATIONSHIPS:
  VARIABLE_PAIR -> HAS -> JOINT_DISTRIBUTION
  JOINT_DISTRIBUTION -> DETERMINES -> MARGINAL_DISTRIBUTIONS
  JOINT_AND_MARGINAL_DISTRIBUTIONS -> DETERMINE -> MUTUAL_INFORMATION
  MUTUAL_INFORMATION -> MEASURES -> STATISTICAL_DEPENDENCE
  MUTUAL_INFORMATION -> EQUALS -> ENTROPY_REDUCTION
  MUTUAL_INFORMATION -> EQUALS -> KL_DIVERGENCE_FROM_INDEPENDENCE_MODEL

DISCRETE_FORM:
  I(X;Y) :=
    SUM_x SUM_y
      p(x,y)
      log_b[
        p(x,y)/(p(x)p(y))
      ]

ENTROPY_IDENTITIES:
  I(X;Y) :=
    H(X)+H(Y)-H(X,Y)

  I(X;Y) :=
    H(X)-H(X|Y)

  I(X;Y) :=
    H(Y)-H(Y|X)

KL_DIVERGENCE_FORM:
  I(X;Y) :=
    D_KL(
      p(x,y)
      ||
      p(x)p(y)
    )

  interpretation:
    DIVERGENCE_FROM_INDEPENDENCE_MODEL

NONNEGATIVITY:
  I(X;Y) >= 0

  equality:
    I(X;Y)=0
    IFF:
      X_AND_Y_INDEPENDENT

SYMMETRY:
  I(X;Y) :=
    I(Y;X)

  implication:
    MUTUAL_INFORMATION_HAS_NO_INTRINSIC_DIRECTION

BOUNDS:
  I(X;Y) <= H(X)
  I(X;Y) <= H(Y)

  therefore:
    I(X;Y) <= min(H(X),H(Y))

DETERMINISTIC_RELATION:
  IF:
    Y := f(X)

  THEN:
    H(Y|X) := 0
    I(X;Y) := H(Y)

  IF:
    f_IS_BIJECTIVE_ON_SUPPORT

  THEN:
    I(X;Y) := H(X) := H(Y)
    FOR:
      DISCRETE_VARIABLES

INDEPENDENCE:
  IF:
    p(x,y) := p(x)p(y)

  THEN:
    I(X;Y) := 0

  converse:
    I(X;Y)=0
      -> INDEPENDENCE

CONDITIONAL_MUTUAL_INFORMATION:
  I(X;Y|Z) :=
    H(X|Z)-H(X|Y,Z)

  equivalent:
    I(X;Y|Z) :=
      H(X,Z)
      + H(Y,Z)
      - H(Z)
      - H(X,Y,Z)

  property:
    I(X;Y|Z) >= 0

CHAIN_RULE:
  I(X;Y,Z) :=
    I(X;Y)
    + I(X;Z|Y)

  general:
    I(X;Y_1,...,Y_n)
      :=
    SUM[i=1..n]
      I(
        X;
        Y_i
        |
        Y_1,...,Y_(i-1)
      )

DATA_PROCESSING_INEQUALITY:
  IF:
    X -> Y -> Z
    IS_MARKOV_CHAIN

  THEN:
    I(X;Z) <= I(X;Y)

  interpretation:
    POST_PROCESSING_CANNOT_INCREASE_INFORMATION_ABOUT_ORIGINAL_VARIABLE

CONTINUOUS_FORM:
  I(X;Y) :=
    INTEGRAL_INTEGRAL
      f(x,y)
      log_b[
        f(x,y)/(f_X(x)f_Y(y))
      ]
      dx dy

  property:
    I(X;Y) >= 0

GAUSSIAN_MUTUAL_INFORMATION:
  jointly_gaussian_scalar:
    correlation:
      rho

  mutual_information:
    I(X;Y) :=
      -0.5*log_b(1-rho^2)

  require:
    abs(rho) < 1

  caution:
    OUTSIDE_GAUSSIAN_CONTEXT:
      CORRELATION_DOES_NOT_DETERMINE_MUTUAL_INFORMATION

MULTIVARIATE_GAUSSIAN:
  mutual_information:
    I(X;Y) :=
      0.5
      * log_b[
          det(Sigma_X)
          * det(Sigma_Y)
          /
          det(Sigma_joint)
        ]

  require:
    RELEVANT_COVARIANCE_MATRICES_POSITIVE_DEFINITE

MUTUAL_INFORMATION_VS_CORRELATION:
  CORRELATION:
    PRIMARILY_LINEAR_ASSOCIATION

  MUTUAL_INFORMATION:
    GENERAL_STATISTICAL_DEPENDENCE

  rule:
    MUTUAL_INFORMATION != CORRELATION

NORMALISED_MUTUAL_INFORMATION:
  possible_forms:
    - I(X;Y)/sqrt(H(X)H(Y))
    - I(X;Y)/max(H(X),H(Y))
    - 2I(X;Y)/(H(X)+H(Y))

  caution:
    NO_SINGLE_UNIVERSAL_NORMALISATION

POINTWISE_MUTUAL_INFORMATION:
  alias:
    PMI

  PMI(x,y) :=
    log_b[
      p(x,y)/(p(x)p(y))
    ]

  relation:
    I(X;Y) :=
      E[PMI(X,Y)]

  caution:
    PMI_MAY_BE_NEGATIVE
    WHILE:
      MUTUAL_INFORMATION_IS_NONNEGATIVE

CHANNEL_INTERPRETATION:
  input:
    X

  output:
    Y

  channel:
    p(y|x)

  I(X;Y):
    INFORMATION_TRANSMITTED_FROM_INPUT_TO_OUTPUT
    FOR:
      SPECIFIED_INPUT_DISTRIBUTION

CHANNEL_CAPACITY_CONNECTION:
  C :=
    max_over_p(x)
      I(X;Y)

  interpretation:
    MAXIMUM_MUTUAL_INFORMATION_OVER_ALLOWED_INPUT_DISTRIBUTIONS

INFORMATION_GAIN:
  IG(TARGET;FEATURE) :=
    H(TARGET)
    - H(TARGET|FEATURE)

  equivalent:
    I(TARGET;FEATURE)

FEATURE_SELECTION:
  criterion:
    SELECT_FEATURES_WITH_HIGH_MUTUAL_INFORMATION_WITH_TARGET

  caution:
    FEATURE_REDUNDANCY_SHOULD_ALSO_BE_CONSIDERED

INFORMATION_BOTTLENECK:
  variables:
    X
    Y
    T

  objective:
    COMPRESS_X_INTO_T
    WHILE:
      PRESERVING_INFORMATION_ABOUT_Y

  tradeoff:
    MINIMISE:
      I(X;T)
    WHILE:
      MAXIMISING:
        I(T;Y)

BAYESIAN_EXPERIMENT_DESIGN:
  parameter:
    theta

  observation:
    Y

  expected_information_gain:
    I(theta;Y)

  equivalent:
    EXPECTED_KL_DIVERGENCE(
      POSTERIOR
      ||
      PRIOR
    )

TOTAL_CORRELATION_CONNECTION:
  TC(X_1,...,X_n) :=
    SUM_i H(X_i)
    - H(X_1,...,X_n)

  two_variable_case:
    TC(X,Y) := I(X;Y)

ESTIMATION_DISCRETE:
  empirical_joint:
    p_hat(x,y)

  plug_in:
    I_hat :=
      SUM_x SUM_y
        p_hat(x,y)
        log_b[
          p_hat(x,y)/(p_hat(x)p_hat(y))
        ]

  issues:
    FINITE_SAMPLE_BIAS
    SPARSE_CONTINGENCY_TABLES

ESTIMATION_CONTINUOUS:
  methods:
    - BINNING
    - KERNEL_DENSITY_ESTIMATION
    - K_NEAREST_NEIGHBOUR
    - PARAMETRIC_GAUSSIAN
    - NEURAL_ESTIMATION

  caution:
    ESTIMATE_DEPENDS_ON:
      SAMPLE_SIZE
      DIMENSION
      ESTIMATOR
      HYPERPARAMETERS

HYPOTHESIS_TESTING_CONNECTION:
  null:
    X_INDEPENDENT_Y

  discrete_relation:
    G_TEST_STATISTIC
      =
    2*n*I_e(X;Y)
    UNDER:
      EMPIRICAL_DISTRIBUTION_FORMULATION

  significance_methods:
    - ASYMPTOTIC_CHI_SQUARE
    - PERMUTATION_TEST

PERMUTATION_TEST:
  procedure:
    PERMUTE_ONE_VARIABLE
      -> DESTROY_DEPENDENCE
      -> PRESERVE_MARGINAL_VALUES
      -> RECOMPUTE_MI

  compare:
    OBSERVED_MI
    WITH:
      NULL_MI_DISTRIBUTION

CAUSALITY_DISTINCTION:
  MUTUAL_INFORMATION:
    MEASURES_STATISTICAL_DEPENDENCE

  DOES_NOT_BY_ITSELF:
    IDENTIFY_CAUSAL_DIRECTION
    IDENTIFY_CAUSAL_EFFECT

APPLICATIONS:
  MUTUAL_INFORMATION -> SUPPORTS -> FEATURE_SELECTION
  MUTUAL_INFORMATION -> SUPPORTS -> CHANNEL_ANALYSIS
  MUTUAL_INFORMATION -> SUPPORTS -> IMAGE_REGISTRATION
  MUTUAL_INFORMATION -> SUPPORTS -> CLUSTERING_EVALUATION
  MUTUAL_INFORMATION -> SUPPORTS -> REPRESENTATION_LEARNING
  MUTUAL_INFORMATION -> SUPPORTS -> EXPERIMENT_DESIGN
  MUTUAL_INFORMATION -> SUPPORTS -> DEPENDENCE_ANALYSIS

MODEL_CONNECTIONS:
  MUTUAL_INFORMATION -> CONNECTS_TO -> INFORMATION_THEORY
  MUTUAL_INFORMATION -> CONNECTS_TO -> SHANNON_ENTROPY
  MUTUAL_INFORMATION -> CONNECTS_TO -> CONDITIONAL_ENTROPY
  MUTUAL_INFORMATION -> CONNECTS_TO -> KL_DIVERGENCE
  MUTUAL_INFORMATION -> CONNECTS_TO -> CHANNEL_CAPACITY
  MUTUAL_INFORMATION -> CONNECTS_TO -> DATA_PROCESSING_INEQUALITY
  MUTUAL_INFORMATION -> CONNECTS_TO -> BAYESIAN_INFORMATION_GAIN

SEMANTIC_EQUIVALENCE:
  "mutual information"
    ~= "shared information between random variables"

  "mutual information"
    ~= "KL divergence between joint distribution and product of marginals"

  "information gain"
    ~= "mutual information"
    IN:
      STANDARD_FEATURE_TARGET_CONTEXT

  EXCLUDE_EQUIVALENCE:
    MUTUAL_INFORMATION != CORRELATION
    MUTUAL_INFORMATION != COVARIANCE
    MUTUAL_INFORMATION != CAUSAL_EFFECT
    MUTUAL_INFORMATION != POINTWISE_MUTUAL_INFORMATION
    CONDITIONAL_MUTUAL_INFORMATION != MUTUAL_INFORMATION_BY_DEFAULT
    NORMALISED_MUTUAL_INFORMATION != UNIQUE_STANDARD_QUANTITY

VALIDATION_RULES:
  RULE_1:
    JOINT_DISTRIBUTION_MUST_BE_NORMALISED

  RULE_2:
    MARGINALS_MUST_BE_CONSISTENT_WITH_JOINT_DISTRIBUTION

  RULE_3:
    LOG_BASE_MUST_BE_EXPLICIT_OR_INFERABLE_FROM_UNIT

  RULE_4:
    MUTUAL_INFORMATION_MUST_SATISFY:
      I(X;Y) >= 0

  RULE_5:
    ZERO_MUTUAL_INFORMATION_IMPLIES_INDEPENDENCE
    UNDER:
      VALID_PROBABILITY_MODEL

  RULE_6:
    MUTUAL_INFORMATION_MUST_NOT_BE_INTERPRETED_AS_CAUSATION

  RULE_7:
    NORMALISED_MUTUAL_INFORMATION_MUST_DOCUMENT_NORMALISATION_DEFINITION

  RULE_8:
    SAMPLE_ESTIMATES_MUST_DISTINGUISH_ESTIMATED_MI_FROM_POPULATION_MI

CONFIDENCE:
  ontology_identity: 0.99
  mutual_information_definition: 0.99
  entropy_relationship: 0.99
  kl_divergence_relationship: 0.99
  data_processing_structure: 0.99
  channel_capacity_relationship: 0.99
  estimation_structure: 0.98

UPDATE_MODEL:
  IF:
    JOINT_DISTRIBUTION_CHANGES

  THEN:
    UPDATE:
      MARGINAL_DISTRIBUTIONS

    RECOMPUTE:
      ENTROPIES
      CONDITIONAL_ENTROPIES
      MUTUAL_INFORMATION

  IF:
    NEW_SAMPLE_DATA_ARRIVES

  THEN:
    UPDATE:
      JOINT_COUNTS_OR_DENSITY_ESTIMATE
      MARGINAL_ESTIMATES

    RECOMPUTE:
      MUTUAL_INFORMATION_ESTIMATE

    REASSESS:
      ESTIMATOR_BIAS
      UNCERTAINTY

  IF:
    CONDITIONING_VARIABLE_ADDED

  THEN:
    COMPUTE:
      CONDITIONAL_MUTUAL_INFORMATION

  PRESERVE:
    VARIABLE_IDENTITIES
    JOINT_DISTRIBUTION
    MARGINAL_DISTRIBUTIONS
    LOG_BASE
    ESTIMATION_METHOD
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_RANDOM_VARIABLES
    - IDENTIFY_JOINT_DISTRIBUTION
    - DERIVE_MARGINAL_DISTRIBUTIONS
    - VERIFY_NORMALISATION
    - IDENTIFY_LOG_BASE
    - COMPUTE_ENTROPIES
    - COMPUTE_MUTUAL_INFORMATION
    - TEST_INDEPENDENCE_IF_REQUIRED
    - COMPUTE_CONDITIONAL_MUTUAL_INFORMATION_IF_REQUIRED
    - APPLY_DATA_PROCESSING_INEQUALITY_IF_RELEVANT
    - ESTIMATE_FROM_DATA_IF_REQUIRED
    - RETURN_DEPENDENCE_AND_INFORMATION_INTERPRETATION

  constraints:
    - PRESERVE_JOINT_AND_MARGINAL_CONSISTENCY
    - EXPOSE_LOG_BASE_AND_INFORMATION_UNIT
    - DO_NOT_EQUATE_MUTUAL_INFORMATION_WITH_CORRELATION
    - DO_NOT_INTERPRET_MUTUAL_INFORMATION_AS_CAUSATION
    - DOCUMENT_NORMALISATION_IF_USED
    - DISTINGUISH_POPULATION_FROM_ESTIMATED_MUTUAL_INFORMATION
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "mutual-information"
  namespace: "probability-statistics/information-theory"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - JOINT_DISTRIBUTION
    - MARGINAL_DISTRIBUTIONS
    - DEPENDENCE_INFORMATION

END_FLOW