Ontology

flow "InformationTheoryOntology3" v4.0

# INFORMATION THEORY ONTOLOGY 3
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory/ontology-3.html"
  title: "Information Theory Ontology 3"
  domain: "Probability and Statistics"
  category: "Information Theory"
  entity: INFORMATION_THEORY_ONTOLOGY_3
  entity_type: SEMANTIC_INFORMATION_FRAMEWORK

DEFINE:
  INFORMATION_THEORY_ONTOLOGY_3 :=
    FORMAL_SEMANTIC_FRAMEWORK(
      INFORMATION,
      ENTROPY,
      MUTUAL_INFORMATION,
      INFORMATION_GAIN,
      CROSS_ENTROPY,
      KL_DIVERGENCE,
      CODING,
      CHANNELS,
      INFORMATION_FLOW
    )

PURPOSE:
  REPRESENT:
    INFORMATION_QUANTITIES
    PROBABILISTIC_DEPENDENCE
    DISTRIBUTIONAL_DIVERGENCE
    SOURCE_ENCODING
    CHANNEL_TRANSMISSION
    UNCERTAINTY_REDUCTION
  FOR:
    HUMAN_REASONING
    MACHINE_REASONING
    RECURSIVE_AI_CONSUMPTION

FOUNDATION:
  PROBABILITY_SPACE:
    (Omega,F,P)

  RANDOM_VARIABLE:
    X:
      Omega -> X_space

  DISTRIBUTION:
    p(x)

  SELF_INFORMATION:
    I(x) :=
      -log_b p(x)

  SHANNON_ENTROPY:
    H(X) :=
      E[I(X)]

ENTITIES:
  INFORMATION: QUANTIFIED_UNCERTAINTY_REDUCTION
  SELF_INFORMATION: OUTCOME_SURPRISAL
  SHANNON_ENTROPY: EXPECTED_INFORMATION
  JOINT_ENTROPY: JOINT_UNCERTAINTY
  CONDITIONAL_ENTROPY: RESIDUAL_UNCERTAINTY
  MUTUAL_INFORMATION: SHARED_INFORMATION
  INFORMATION_GAIN: UNCERTAINTY_REDUCTION
  CROSS_ENTROPY: EXPECTED_MODEL_LOG_LOSS
  KL_DIVERGENCE: DISTRIBUTIONAL_DIVERGENCE
  SOURCE: INFORMATION_GENERATOR
  CHANNEL: STOCHASTIC_MAPPING
  CODE: SYMBOL_REPRESENTATION
  CHANNEL_CAPACITY: RELIABLE_INFORMATION_LIMIT
  ENTROPY_RATE: LONG_RUN_INFORMATION_RATE

RELATIONSHIPS:
  OUTCOME_PROBABILITY -> DETERMINES -> SELF_INFORMATION
  DISTRIBUTION -> DETERMINES -> ENTROPY
  JOINT_DISTRIBUTION -> DETERMINES -> MUTUAL_INFORMATION
  CONDITIONING -> REDUCES_OR_PRESERVES -> DISCRETE_ENTROPY
  ENTROPY_REDUCTION -> DEFINES -> INFORMATION_GAIN
  TRUE_AND_MODEL_DISTRIBUTIONS -> DEFINE -> CROSS_ENTROPY
  DISTRIBUTION_PAIR -> DEFINES -> KL_DIVERGENCE
  SOURCE -> PRODUCES -> SYMBOL_SEQUENCE
  CODE -> REPRESENTS -> SOURCE_SYMBOLS
  CHANNEL -> MAPS -> INPUT_TO_OUTPUT
  CHANNEL_CAPACITY -> BOUNDS -> RELIABLE_TRANSMISSION

SELF_INFORMATION:
  I(x) :=
    -log_b p(x)

  properties:
    p(x)=1:
      I(x)=0

    LOWER_PROBABILITY:
      HIGHER_INFORMATION

SHANNON_ENTROPY:
  H(X) :=
    -SUM_x p(x) log_b p(x)

  properties:
    H(X) >= 0
    FOR:
      DISCRETE_X

  finite_alphabet:
    H(X) <= log_b |X_space|

  maximum:
    UNIFORM_DISTRIBUTION

JOINT_ENTROPY:
  H(X,Y) :=
    -SUM_x SUM_y
      p(x,y) log_b p(x,y)

CONDITIONAL_ENTROPY:
  H(Y|X) :=
    H(X,Y)-H(X)

  discrete_property:
    H(Y|X) <= H(Y)

CHAIN_RULE_ENTROPY:
  H(X,Y) :=
    H(X)+H(Y|X)

  general:
    H(X_1,...,X_n)
      =
    SUM[i=1..n]
      H(X_i | X_1,...,X_(i-1))

MUTUAL_INFORMATION:
  I(X;Y) :=
    H(X)+H(Y)-H(X,Y)

  equivalent:
    I(X;Y) :=
      H(X)-H(X|Y)

  divergence_form:
    I(X;Y) :=
      D_KL(
        p(x,y)
        ||
        p(x)p(y)
      )

  properties:
    I(X;Y) >= 0
    I(X;Y) = I(Y;X)

  independence:
    I(X;Y)=0
      IFF
    X_INDEPENDENT_Y

INFORMATION_GAIN:
  IG(Y;X) :=
    H(Y)-H(Y|X)

  equivalent:
    IG(Y;X) := I(X;Y)

  role:
    MEASURE_UNCERTAINTY_REDUCTION

CONDITIONAL_MUTUAL_INFORMATION:
  I(X;Y|Z) :=
    H(X|Z)-H(X|Y,Z)

  property:
    I(X;Y|Z) >= 0

CROSS_ENTROPY:
  H(P,Q) :=
    -SUM_x
      p(x) log_b q(x)

  relation:
    H(P,Q) :=
      H(P)+D_KL(P||Q)

  interpretation:
    EXPECTED_LOG_LOSS_USING_Q
    WHEN:
      DATA_FOLLOW_P

KL_DIVERGENCE:
  D_KL(P||Q) :=
    SUM_x
      p(x)
      log_b[
        p(x)/q(x)
      ]

  properties:
    D_KL(P||Q) >= 0
    D_KL(P||Q) != D_KL(Q||P)
    NOT_A_METRIC = TRUE

  equality:
    D_KL(P||Q)=0
    IFF:
      P=Q
    ALMOST_EVERYWHERE

DATA_PROCESSING_INEQUALITY:
  IF:
    X -> Y -> Z
    IS_MARKOV_CHAIN

  THEN:
    I(X;Z) <= I(X;Y)

  interpretation:
    PROCESSING_CANNOT_INCREASE_INFORMATION_ABOUT_ORIGINAL_VARIABLE

SOURCE:
  definition:
    STOCHASTIC_GENERATOR_OF_SYMBOLS_OR_SEQUENCES

  properties:
    ALPHABET
    DISTRIBUTION
    DEPENDENCE_STRUCTURE

SOURCE_ENTROPY:
  memoryless:
    H(X)

  process:
    ENTROPY_RATE

ENTROPY_RATE:
  H_rate :=
    LIMIT[n->infinity]
      H(X_1,...,X_n)/n
    WHEN:
      LIMIT_EXISTS

  interpretation:
    LONG_RUN_AVERAGE_INFORMATION_PER_SYMBOL

SOURCE_CODING:
  objective:
    REPRESENT_SOURCE_OUTPUT_EFFICIENTLY

  lossless_limit:
    GOVERNED_BY:
      SOURCE_ENTROPY_OR_ENTROPY_RATE
    UNDER:
      STANDARD_ASYMPTOTIC_ASSUMPTIONS

PREFIX_CODE:
  requirement:
    NO_CODEWORD_IS_PREFIX_OF_ANOTHER

  consequence:
    INSTANTANEOUS_DECODABILITY

KRAFT_INEQUALITY:
  D_ary:
    SUM_i D^(-l_i) <= 1

HUFFMAN_CODING:
  role:
    MINIMISE_EXPECTED_SYMBOL_BY_SYMBOL_PREFIX_CODE_LENGTH
    UNDER:
      KNOWN_SYMBOL_PROBABILITIES

CHANNEL:
  input:
    X

  output:
    Y

  transition:
    p(y|x)

  joint:
    p(x,y) :=
      p(x)p(y|x)

CHANNEL_INFORMATION:
  transmitted_information:
    I(X;Y)

CHANNEL_CAPACITY:
  C :=
    max_over_allowed_p(x)
      I(X;Y)

  interpretation:
    MAXIMUM_RELIABLE_INFORMATION_PER_CHANNEL_USE
    UNDER:
      STANDARD_CHANNEL_MODEL

CHANNEL_CODING_THEOREM:
  IF:
    R < C

  THEN:
    RELIABLE_COMMUNICATION_WITH_ARBITRARILY_SMALL_ERROR:
      ASYMPTOTICALLY_ACHIEVABLE
    UNDER:
      STANDARD_ASSUMPTIONS

  IF:
    R > C

  THEN:
    RELIABLE_ASYMPTOTIC_COMMUNICATION:
      NOT_ACHIEVABLE
    UNDER:
      STANDARD_MODEL

BINARY_SYMMETRIC_CHANNEL:
  crossover_probability:
    p

  capacity:
    C :=
      1-H_2(p)

BINARY_ERASURE_CHANNEL:
  erasure_probability:
    epsilon

  capacity:
    C :=
      1-epsilon

RATE_DISTORTION:
  R(D) :=
    min I(X;X_hat)

  subject_to:
    E[d(X,X_hat)] <= D

  role:
    FUNDAMENTAL_LOSSY_COMPRESSION_LIMIT

BAYESIAN_INFORMATION:
  prior:
    p(theta)

  posterior:
    p(theta|D)

  realised_gain:
    D_KL(
      p(theta|D)
      ||
      p(theta)
    )

  expected_gain:
    I(theta;D)

STATISTICAL_LEARNING:
  CROSS_ENTROPY:
    USED_AS:
      PREDICTIVE_LOG_LOSS

  KL_DIVERGENCE:
    USED_FOR:
      DISTRIBUTION_APPROXIMATION

  MUTUAL_INFORMATION:
    USED_FOR:
      DEPENDENCE_AND_FEATURE_RELEVANCE

  INFORMATION_GAIN:
    USED_FOR:
      DECISION_TREE_SPLITTING
      EXPERIMENT_SELECTION

INFORMATION_FLOW:
  SOURCE
    -> ENCODE
    -> CHANNEL_INPUT
    -> TRANSMIT
    -> CHANNEL_OUTPUT
    -> DECODE
    -> RECONSTRUCTED_INFORMATION

  losses_may_arise_from:
    - NOISE
    - COMPRESSION
    - MODEL_MISMATCH
    - QUANTISATION
    - CHANNEL_LIMITS

SEMANTIC_INFORMATION_GRAPH:
  SHANNON_ENTROPY
    -> PARENT_OF
    -> CONDITIONAL_ENTROPY

  SHANNON_ENTROPY
    -> CONNECTS_TO
    -> MUTUAL_INFORMATION

  MUTUAL_INFORMATION
    -> EQUIVALENT_CONTEXT
    -> INFORMATION_GAIN

  CROSS_ENTROPY
    -> DECOMPOSES_TO
    -> SHANNON_ENTROPY_PLUS_KL_DIVERGENCE

  KL_DIVERGENCE
    -> SPECIALISES_TO
    -> MUTUAL_INFORMATION
    WHEN:
      Q_EQUALS_PRODUCT_OF_MARGINALS

  MUTUAL_INFORMATION
    -> OPTIMISED_BY
    -> CHANNEL_CAPACITY

PROBABILITY_CONNECTION:
  INFORMATION_QUANTITIES:
    DERIVED_FROM:
      PROBABILITY_DISTRIBUTIONS

  rule:
    INVALID_PROBABILITY_MODEL
      -> INVALID_INFORMATION_QUANTITIES

STOCHASTIC_PROCESS_CONNECTION:
  process:
    X_1,X_2,...

  information_structure:
    - ENTROPY_RATE
    - CONDITIONAL_ENTROPY_RATE
    - MUTUAL_INFORMATION_RATE
    - RELATIVE_ENTROPY_RATE

MACHINE_LEARNING_CONNECTION:
  INFORMATION_THEORY
    -> SUPPORTS
    -> FEATURE_SELECTION

  INFORMATION_THEORY
    -> SUPPORTS
    -> REPRESENTATION_LEARNING

  INFORMATION_THEORY
    -> SUPPORTS
    -> GENERATIVE_MODELLING

  INFORMATION_THEORY
    -> SUPPORTS
    -> CLASSIFICATION

  INFORMATION_THEORY
    -> SUPPORTS
    -> ACTIVE_LEARNING

APPLICATIONS:
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> DATA_COMPRESSION
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> COMMUNICATION
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> STATISTICS
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> MACHINE_LEARNING
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> SIGNAL_PROCESSING
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> CRYPTOGRAPHY
  INFORMATION_THEORY_ONTOLOGY_3 -> SUPPORTS -> BAYESIAN_INFERENCE

MODEL_CONNECTIONS:
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> PROBABILITY_THEORY
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> SHANNON_ENTROPY
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> MUTUAL_INFORMATION
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> INFORMATION_GAIN
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> CROSS_ENTROPY
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> KL_DIVERGENCE
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> STOCHASTIC_PROCESSES
  INFORMATION_THEORY_ONTOLOGY_3 -> CONNECTS_TO -> BAYESIAN_STATISTICS

SEMANTIC_EQUIVALENCE:
  "Shannon entropy"
    ~= "expected self-information"

  "relative entropy"
    ~= "Kullback-Leibler divergence"

  "information gain"
    ~= "mutual information"
    IN:
      STANDARD_TARGET_FEATURE_CONTEXT

  "channel capacity"
    ~= "maximum reliable information rate"
    UNDER:
      SPECIFIED_CHANNEL_MODEL

  EXCLUDE_EQUIVALENCE:
    ENTROPY != VARIANCE
    MUTUAL_INFORMATION != CORRELATION
    MUTUAL_INFORMATION != CAUSATION
    KL_DIVERGENCE != METRIC
    CROSS_ENTROPY != ENTROPY_BY_DEFAULT
    INFORMATION_GAIN != FISHER_INFORMATION
    SHANNON_INFORMATION != THERMODYNAMIC_ENTROPY_BY_DEFAULT

VALIDATION_RULES:
  RULE_1:
    PROBABILITY_DISTRIBUTIONS:
      MUST_BE_NORMALISED

  RULE_2:
    LOG_BASE:
      MUST_BE_EXPLICIT_OR_INFERABLE_FROM_INFORMATION_UNIT

  RULE_3:
    SUPPORT_COMPATIBILITY:
      MUST_BE_CHECKED_FOR_KL_AND_CROSS_ENTROPY

  RULE_4:
    MUTUAL_INFORMATION:
      MUST_BE_NONNEGATIVE

  RULE_5:
    KL_DIVERGENCE:
      MUST_NOT_BE_TREATED_AS_SYMMETRIC_OR_METRIC

  RULE_6:
    CHANNEL_CAPACITY:
      MUST_MATCH_CHANNEL_MODEL_AND_INPUT_CONSTRAINTS

  RULE_7:
    INFORMATIONAL_DEPENDENCE:
      MUST_NOT_BE_INTERPRETED_AS_CAUSALITY_BY_DEFAULT

  RULE_8:
    SOURCE_AND_CHANNEL_CODING_RESULTS:
      MUST_PRESERVE_ASYMPTOTIC_AND_MODEL_ASSUMPTIONS

CONFIDENCE:
  ontology_identity: 0.99
  entropy_structure: 0.99
  mutual_information_structure: 0.99
  information_gain_structure: 0.99
  cross_entropy_structure: 0.99
  kl_divergence_structure: 0.99
  coding_channel_structure: 0.99

UPDATE_MODEL:
  IF:
    DISTRIBUTION_CHANGES

  THEN:
    RECOMPUTE:
      SELF_INFORMATION
      ENTROPY
      MUTUAL_INFORMATION
      CROSS_ENTROPY
      KL_DIVERGENCE

  IF:
    SOURCE_MODEL_CHANGES

  THEN:
    UPDATE:
      ALPHABET
      DISTRIBUTION
      DEPENDENCE_STRUCTURE
      ENTROPY_RATE

  IF:
    CHANNEL_MODEL_CHANGES

  THEN:
    UPDATE:
      TRANSITION_PROBABILITIES
      INPUT_CONSTRAINTS

    RECOMPUTE:
      MUTUAL_INFORMATION
      CHANNEL_CAPACITY

  IF:
    NEW_DATA_ARRIVES

  THEN:
    UPDATE:
      EMPIRICAL_DISTRIBUTIONS
      INFORMATION_ESTIMATES

    REASSESS:
      BIAS
      UNCERTAINTY
      MODEL_FIT

  PRESERVE:
    ENTITY_IDENTITIES
    PROBABILITY_MODELS
    LOG_BASE
    INFORMATION_UNIT
    SOURCE_MODEL
    CHANNEL_MODEL
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_INFORMATION_ENTITIES
    - IDENTIFY_PROBABILITY_DISTRIBUTIONS
    - VERIFY_NORMALISATION
    - IDENTIFY_LOG_BASE
    - COMPUTE_SELF_INFORMATION
    - COMPUTE_ENTROPY
    - COMPUTE_CONDITIONAL_AND_JOINT_ENTROPY
    - COMPUTE_MUTUAL_INFORMATION
    - COMPUTE_INFORMATION_GAIN
    - COMPUTE_CROSS_ENTROPY_AND_KL_IF_REQUIRED
    - IDENTIFY_SOURCE_AND_CHANNEL_STRUCTURE
    - APPLY_CODING_OR_CAPACITY_LIMITS_IF_REQUIRED
    - RETURN_SEMANTIC_INFORMATION_GRAPH

  constraints:
    - PRESERVE_PROBABILITY_NORMALISATION
    - PRESERVE_INFORMATION_UNITS
    - DISTINGUISH_ENTROPY_CROSS_ENTROPY_AND_KL
    - DO_NOT_EQUATE_MUTUAL_INFORMATION_WITH_CAUSATION
    - DO_NOT_TREAT_KL_DIVERGENCE_AS_METRIC
    - EXPOSE_SOURCE_AND_CHANNEL_ASSUMPTIONS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "information-theory-ontology-3"
  namespace: "probability-statistics/information-theory"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - INFORMATION_QUANTITIES
    - DISTRIBUTIONAL_RELATIONSHIPS
    - SOURCE_CHANNEL_STRUCTURE

END_FLOW