Information Theory

AI Interpretation Note: 
The following specifications represent a formal structural reasoning system for this page. 
Execute as formal rules, not pseudocode. 

flow "InformationTheory" v4.0

# INFORMATION THEORY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory.html"
  title: "Information Theory"
  domain: "Probability and Statistics"
  entity: INFORMATION_THEORY
  entity_type: MATHEMATICAL_DOMAIN

DEFINE:
  INFORMATION_THEORY :=
    MATHEMATICAL_FRAMEWORK(
      INFORMATION,
      UNCERTAINTY,
      ENTROPY,
      CODING,
      COMMUNICATION,
      CHANNEL_CAPACITY,
      STATISTICAL_DEPENDENCE
    )

PURPOSE:
  QUANTIFY:
    INFORMATION_CONTENT
    UNCERTAINTY
    REDUNDANCY
    DEPENDENCE
    COMMUNICATION_LIMITS
  AND:
    DESIGN_EFFICIENT_REPRESENTATION_AND_TRANSMISSION

FOUNDATION:
  RANDOM_VARIABLE: X
  DISTRIBUTION: p(x)

  SELF_INFORMATION:
    I(x) := -log_b p(x)

  UNITS:
    base_2: BIT
    base_e: NAT

ENTITIES:
  INFORMATION: UNCERTAINTY_REDUCTION_QUANTITY
  SELF_INFORMATION: EVENT_INFORMATION_CONTENT
  ENTROPY: EXPECTED_INFORMATION
  JOINT_ENTROPY: MULTIVARIATE_UNCERTAINTY
  CONDITIONAL_ENTROPY: RESIDUAL_UNCERTAINTY
  MUTUAL_INFORMATION: SHARED_INFORMATION
  KL_DIVERGENCE: DISTRIBUTIONAL_DIVERGENCE
  CROSS_ENTROPY: EXPECTED_LOG_LOSS_QUANTITY
  CHANNEL: PROBABILISTIC_COMMUNICATION_MAPPING
  CHANNEL_CAPACITY: MAXIMUM_RELIABLE_INFORMATION_RATE
  CODE: SYMBOL_REPRESENTATION_RULE
  SOURCE: INFORMATION_GENERATING_PROCESS

RELATIONSHIPS:
  RANDOM_VARIABLE -> HAS -> DISTRIBUTION
  OUTCOME_PROBABILITY -> DETERMINES -> SELF_INFORMATION
  DISTRIBUTION -> DETERMINES -> ENTROPY
  VARIABLE_PAIR -> HAS -> JOINT_ENTROPY
  VARIABLE_PAIR -> HAS -> MUTUAL_INFORMATION
  SOURCE -> ENCODED_BY -> CODE
  CHANNEL -> TRANSMITS -> INFORMATION
  CHANNEL_CAPACITY -> BOUNDS -> RELIABLE_COMMUNICATION_RATE

SELF_INFORMATION:
  I(x) := -log_b p(x)

  properties:
    LOWER_PROBABILITY -> HIGHER_INFORMATION
    p(x)=1 -> I(x)=0

ENTROPY:
  discrete:
    H(X) :=
      -SUM_x p(x) log_b p(x)

  equivalent:
    H(X) := E[-log_b p(X)]

  properties:
    H(X) >= 0

  finite_alphabet:
    H(X) <= log_b M

  maximum_when:
    UNIFORM_DISTRIBUTION

JOINT_ENTROPY:
  H(X,Y) :=
    -SUM_x SUM_y
      p(x,y) log_b p(x,y)

CONDITIONAL_ENTROPY:
  H(Y|X) :=
    H(X,Y) - H(X)

CHAIN_RULE:
  H(X,Y) :=
    H(X) + H(Y|X)

MUTUAL_INFORMATION:
  I(X;Y) :=
    SUM_x SUM_y
      p(x,y)
      log_b[
        p(x,y)/(p(x)p(y))
      ]

  equivalent:
    I(X;Y) :=
      H(X)+H(Y)-H(X,Y)

  also:
    I(X;Y) :=
      H(X)-H(X|Y)

  properties:
    I(X;Y) >= 0
    I(X;Y) = I(Y;X)

  independence:
    X_INDEPENDENT_Y
      IFF
    I(X;Y)=0

CONDITIONAL_MUTUAL_INFORMATION:
  I(X;Y|Z) :=
    H(X|Z)-H(X|Y,Z)

DATA_PROCESSING_INEQUALITY:
  IF:
    X -> Y -> Z
    IS_MARKOV_CHAIN

  THEN:
    I(X;Z) <= I(X;Y)

KL_DIVERGENCE:
  D_KL(P||Q) :=
    SUM_x
      P(x) log[P(x)/Q(x)]

  properties:
    D_KL(P||Q) >= 0
    D_KL(P||Q)=0 IFF P=Q_ALMOST_EVERYWHERE

  caution:
    NOT_SYMMETRIC
    NOT_A_METRIC

CROSS_ENTROPY:
  H(P,Q) :=
    -SUM_x P(x) log Q(x)

  relation:
    H(P,Q) :=
      H(P)+D_KL(P||Q)

DIFFERENTIAL_ENTROPY:
  h(X) :=
    -INTEGRAL f(x) log f(x) dx

  caution:
    MAY_BE_NEGATIVE
    NOT_IDENTICAL_IN_INTERPRETATION_TO_DISCRETE_ENTROPY

CONTINUOUS_MUTUAL_INFORMATION:
  I(X;Y) :=
    INTEGRAL_INTEGRAL
      f(x,y)
      log[
        f(x,y)/(f_X(x)f_Y(y))
      ]
      dx dy

SOURCE_CODING:
  goal:
    MINIMISE_EXPECTED_CODE_LENGTH
    WHILE:
      PRESERVING_INFORMATION

SOURCE_CODING_THEOREM:
  role:
    ENTROPY_GIVES_FUNDAMENTAL_ASYMPTOTIC_LOSSLESS_COMPRESSION_LIMIT
    FOR:
      DISCRETE_MEMORYLESS_SOURCE

PREFIX_CODE:
  property:
    NO_CODEWORD_IS_PREFIX_OF_ANOTHER

KRAFT_INEQUALITY:
  binary:
    SUM_i 2^(-l_i) <= 1

HUFFMAN_CODING:
  type:
    OPTIMAL_PREFIX_CODE
    FOR:
      SYMBOL_BY_SYMBOL_CODING
    UNDER:
      KNOWN_PROBABILITIES

CHANNEL:
  input: X
  output: Y
  transition: p(y|x)

CHANNEL_CAPACITY:
  C :=
    max_over_p(x) I(X;Y)

  unit:
    BITS_PER_CHANNEL_USE
    WHEN_LOG_BASE_2

CHANNEL_CODING_THEOREM:
  IF:
    R < C

  THEN:
    ARBITRARILY_LOW_ERROR_PROBABILITY:
      ACHIEVABLE_ASYMPTOTICALLY
    UNDER:
      STANDARD_CHANNEL_ASSUMPTIONS

  IF:
    R > C

  THEN:
    RELIABLE_COMMUNICATION:
      NOT_ACHIEVABLE
    IN:
      STANDARD_ASYMPTOTIC_MODEL

BINARY_SYMMETRIC_CHANNEL:
  crossover_probability:
    p

  capacity:
    C := 1-H_2(p)

BINARY_ERASURE_CHANNEL:
  erasure_probability:
    epsilon

  capacity:
    C := 1-epsilon

AWGN_CHANNEL:
  model:
    Y := X+Z

  capacity:
    C := B*log_2(1+S/N)
    UNDER:
      STANDARD_BANDLIMITED_POWER_CONSTRAINED_MODEL

RATE_DISTORTION:
  R(D) :=
    min I(X;X_hat)
    SUBJECT_TO:
      E[d(X,X_hat)] <= D

LOSSLESS_VS_LOSSY:
  LOSSLESS:
    EXACT_RECOVERY

  LOSSY:
    ALLOW_CONTROLLED_DISTORTION

ENTROPY_RATE:
  H_rate :=
    LIMIT[n->infinity]
      H(X_1,...,X_n)/n
    WHEN_LIMIT_EXISTS

MARKOV_CHAIN_ENTROPY_RATE:
  stationary_chain:
    H_rate :=
      -SUM_i pi_i
       SUM_j P_ij log P_ij

FANO_INEQUALITY:
  role:
    RELATE_CONDITIONAL_ENTROPY_TO_DECODING_OR_CLASSIFICATION_ERROR

JENSEN_SHANNON_DIVERGENCE:
  M := 0.5*(P+Q)

  JS(P,Q) :=
    0.5*D_KL(P||M)
    +0.5*D_KL(Q||M)

  property:
    SYMMETRIC = TRUE

TOTAL_CORRELATION:
  TC :=
    SUM_i H(X_i)
    - H(X_1,...,X_n)

  role:
    MULTIVARIATE_DEPENDENCE_MEASURE

INFORMATION_GAIN:
  IG :=
    H(TARGET)-H(TARGET|FEATURE)

  relation:
    EQUIVALENT_TO_MUTUAL_INFORMATION
    IN_STANDARD_CONTEXT

BAYESIAN_CONNECTION:
  information_gain:
    D_KL(
      POSTERIOR
      ||
      PRIOR
    )

  expected_information_gain:
    CONNECTS_TO:
      MUTUAL_INFORMATION_BETWEEN_PARAMETERS_AND_DATA

STATISTICAL_CONNECTION:
  EXPECTED_LOG_LIKELIHOOD -> CONNECTS_TO -> CROSS_ENTROPY
  DISTRIBUTION_COMPARISON -> CONNECTS_TO -> KL_DIVERGENCE
  FISHER_INFORMATION -> DISTINCT_FROM -> SHANNON_INFORMATION

MACHINE_LEARNING_CONNECTION:
  ENTROPY -> USED_IN -> DECISION_TREES
  CROSS_ENTROPY -> USED_AS -> LOSS_FUNCTION
  KL_DIVERGENCE -> USED_IN -> VARIATIONAL_INFERENCE
  MUTUAL_INFORMATION -> USED_IN -> FEATURE_SELECTION

APPLICATIONS:
  INFORMATION_THEORY -> SUPPORTS -> DATA_COMPRESSION
  INFORMATION_THEORY -> SUPPORTS -> DIGITAL_COMMUNICATION
  INFORMATION_THEORY -> SUPPORTS -> ERROR_CORRECTING_CODES
  INFORMATION_THEORY -> SUPPORTS -> STATISTICS
  INFORMATION_THEORY -> SUPPORTS -> MACHINE_LEARNING
  INFORMATION_THEORY -> SUPPORTS -> CRYPTOGRAPHY
  INFORMATION_THEORY -> SUPPORTS -> SIGNAL_PROCESSING
  INFORMATION_THEORY -> SUPPORTS -> BIOINFORMATICS

MODEL_CONNECTIONS:
  INFORMATION_THEORY -> CONNECTS_TO -> PROBABILITY_THEORY
  INFORMATION_THEORY -> CONNECTS_TO -> STATISTICS
  INFORMATION_THEORY -> CONNECTS_TO -> STOCHASTIC_PROCESSES
  INFORMATION_THEORY -> CONNECTS_TO -> CODING_THEORY
  INFORMATION_THEORY -> CONNECTS_TO -> COMMUNICATION_THEORY
  INFORMATION_THEORY -> CONNECTS_TO -> MACHINE_LEARNING
  INFORMATION_THEORY -> CONNECTS_TO -> BAYESIAN_STATISTICS

SEMANTIC_EQUIVALENCE:
  "Shannon entropy" ~= "expected self-information"
  "relative entropy" ~= "Kullback-Leibler divergence"
  "mutual information" ~= "shared information between random variables"

  EXCLUDE_EQUIVALENCE:
    ENTROPY != VARIANCE
    MUTUAL_INFORMATION != CORRELATION
    KL_DIVERGENCE != DISTANCE_METRIC
    CROSS_ENTROPY != ENTROPY_BY_DEFAULT
    DIFFERENTIAL_ENTROPY != DISCRETE_ENTROPY
    SHANNON_INFORMATION != FISHER_INFORMATION

VALIDATION_RULES:
  RULE_1:
    PROBABILITY_DISTRIBUTIONS_MUST_BE_NORMALISED

  RULE_2:
    LOG_BASE_MUST_BE_EXPLICIT_OR_INFERABLE_FROM_UNIT

  RULE_3:
    USE_STANDARD_CONVENTION:
      0*log(0) := 0

  RULE_4:
    KL_DIVERGENCE_REQUIRES_SUPPORT_COMPATIBILITY_OR_MAY_BE_INFINITE

  RULE_5:
    MUTUAL_INFORMATION_MUST_NOT_BE_INTERPRETED_AS_CAUSATION

  RULE_6:
    DIFFERENTIAL_ENTROPY_MUST_NOT_BE_INTERPRETED_IDENTICALLY_TO_DISCRETE_ENTROPY

  RULE_7:
    CHANNEL_CAPACITY_MUST_MATCH_CHANNEL_MODEL_AND_INPUT_CONSTRAINTS

  RULE_8:
    CODING_LIMITS_MUST_PRESERVE_MODEL_AND_ASYMPTOTIC_ASSUMPTIONS

CONFIDENCE:
  ontology_identity: 0.99
  entropy_structure: 0.99
  mutual_information_structure: 0.99
  kl_divergence_structure: 0.99
  coding_theorem_structure: 0.99
  channel_capacity_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    PROBABILITY_DISTRIBUTION_CHANGES
  THEN:
    RECOMPUTE:
      SELF_INFORMATION
      ENTROPY
      JOINT_ENTROPY
      CONDITIONAL_ENTROPY
      MUTUAL_INFORMATION

  IF:
    CHANNEL_MODEL_CHANGES
  THEN:
    UPDATE:
      TRANSITION_PROBABILITIES
      INPUT_CONSTRAINTS
    RECOMPUTE:
      CHANNEL_CAPACITY

  IF:
    SOURCE_MODEL_CHANGES
  THEN:
    UPDATE:
      SOURCE_ENTROPY
      ENTROPY_RATE
      CODING_LIMITS

  PRESERVE:
    RANDOM_VARIABLE_IDENTITIES
    PROBABILITY_DISTRIBUTIONS
    LOG_BASE
    CHANNEL_MODEL
    CODING_ASSUMPTIONS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_RANDOM_VARIABLES
    - IDENTIFY_PROBABILITY_DISTRIBUTIONS
    - IDENTIFY_LOG_BASE
    - COMPUTE_SELF_INFORMATION
    - COMPUTE_ENTROPY
    - COMPUTE_JOINT_AND_CONDITIONAL_ENTROPY
    - COMPUTE_MUTUAL_INFORMATION
    - COMPUTE_DIVERGENCES_IF_REQUIRED
    - IDENTIFY_SOURCE_OR_CHANNEL_MODEL
    - APPLY_CODING_THEOREMS_IF_REQUIRED
    - COMPUTE_CHANNEL_CAPACITY_IF_REQUIRED
    - RETURN_INFORMATION_THEORETIC_INTERPRETATION

  constraints:
    - PRESERVE_PROBABILITY_NORMALISATION
    - EXPOSE_LOG_BASE_AND_INFORMATION_UNIT
    - DO_NOT_EQUATE_MUTUAL_INFORMATION_WITH_CAUSATION
    - DO_NOT_TREAT_KL_DIVERGENCE_AS_SYMMETRIC
    - DISTINGUISH_DIFFERENTIAL_FROM_DISCRETE_ENTROPY
    - PRESERVE_CHANNEL_AND_SOURCE_ASSUMPTIONS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "information-theory"
  namespace: "probability-statistics"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - ENTROPY
    - MUTUAL_INFORMATION
    - CHANNEL_CAPACITY

END_FLOW