Entropy Ontology

flow "EntropyOntology" v4.0

# ENTROPY ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory/entropy-ontology.html"
  title: "Entropy Ontology"
  domain: "Probability and Statistics"
  category: "Information Theory"
  entity: ENTROPY
  entity_type: INFORMATION_THEORETIC_QUANTITY

DEFINE:
  ENTROPY :=
    EXPECTED_SELF_INFORMATION(
      RANDOM_VARIABLE,
      PROBABILITY_DISTRIBUTION,
      LOGARITHM_BASE
    )

PURPOSE:
  QUANTIFY:
    UNCERTAINTY
    AVERAGE_INFORMATION_CONTENT
    RANDOMNESS
    SOURCE_CODING_LIMITS

FOUNDATION:
  RANDOM_VARIABLE: X
  PROBABILITY_MASS_FUNCTION: p(x)
  SELF_INFORMATION:
    I(x) := -log_b p(x)
  ENTROPY:
    H(X) := E[I(X)]

ENTITIES:
  ENTROPY: EXPECTED_INFORMATION
  SELF_INFORMATION: OUTCOME_INFORMATION_CONTENT
  RANDOM_VARIABLE: PROBABILISTIC_VARIABLE
  PROBABILITY_DISTRIBUTION: PROBABILITY_MEASURE
  ALPHABET: SET_OF_POSSIBLE_SYMBOLS
  LOGARITHM_BASE: INFORMATION_UNIT_PARAMETER
  BIT: INFORMATION_UNIT
  NAT: INFORMATION_UNIT

RELATIONSHIPS:
  RANDOM_VARIABLE -> HAS -> PROBABILITY_DISTRIBUTION
  OUTCOME -> HAS -> SELF_INFORMATION
  SELF_INFORMATION -> EXPECTED_OVER -> DISTRIBUTION
  EXPECTED_SELF_INFORMATION -> DEFINES -> ENTROPY
  LOGARITHM_BASE -> DETERMINES -> INFORMATION_UNIT

DISCRETE_ENTROPY:
  H(X) :=
    -SUM_x p(x) log_b p(x)

  convention:
    0*log_b(0) := 0

UNITS:
  BASE_2: BIT
  BASE_e: NAT

NONNEGATIVITY:
  H(X) >= 0
  FOR:
    DISCRETE_RANDOM_VARIABLE

  equality:
    H(X)=0
    IFF:
      X_IS_DETERMINISTIC

MAXIMUM_ENTROPY_FINITE_ALPHABET:
  IF:
    ALPHABET_SIZE = m
  THEN:
    H(X) <= log_b m

  equality:
    WHEN:
      DISTRIBUTION_IS_UNIFORM

BINARY_ENTROPY:
  H_2(p) :=
    -p log_2 p
    -(1-p) log_2(1-p)

  properties:
    H_2(0)=0
    H_2(1)=0
    H_2(0.5)=1
    H_2(p)=H_2(1-p)

JOINT_ENTROPY:
  H(X,Y) :=
    -SUM_x SUM_y
      p(x,y) log_b p(x,y)

CONDITIONAL_ENTROPY:
  H(Y|X) :=
    -SUM_x SUM_y
      p(x,y) log_b p(y|x)

  identity:
    H(Y|X) :=
      H(X,Y)-H(X)

CHAIN_RULE:
  H(X,Y) :=
    H(X)+H(Y|X)

  general:
    H(X_1,...,X_n)
      =
    SUM[i=1..n]
      H(X_i | X_1,...,X_(i-1))

INDEPENDENCE:
  IF:
    X_INDEPENDENT_Y
  THEN:
    H(X,Y) := H(X)+H(Y)
    H(X|Y) := H(X)

SUBADDITIVITY:
  H(X,Y) <= H(X)+H(Y)

  equality:
    IFF:
      X_AND_Y_INDEPENDENT

MUTUAL_INFORMATION_CONNECTION:
  I(X;Y) :=
    H(X)+H(Y)-H(X,Y)

  equivalent:
    I(X;Y) :=
      H(X)-H(X|Y)

CONDITIONING_REDUCES_ENTROPY:
  H(X|Y) <= H(X)
  FOR:
    DISCRETE_RANDOM_VARIABLES

DETERMINISTIC_FUNCTION:
  IF:
    Y := f(X)
  THEN:
    H(Y|X) := 0
    H(Y) <= H(X)
    FOR:
      DISCRETE_VARIABLES

CROSS_ENTROPY_CONNECTION:
  H(P,Q) :=
    -SUM_x P(x) log_b Q(x)

  identity:
    H(P,Q) :=
      H(P)+D_KL(P||Q)

KL_DIVERGENCE_CONNECTION:
  D_KL(P||Q) :=
    H(P,Q)-H(P)

DIFFERENTIAL_ENTROPY:
  h(X) :=
    -INTEGRAL f(x) log_b f(x) dx

  properties:
    MAY_BE_NEGATIVE
    DEPENDS_ON_SCALE

  distinction:
    DIFFERENTIAL_ENTROPY != DISCRETE_ENTROPY

GAUSSIAN_ENTROPY:
  IF:
    X ~ NORMAL(mu,sigma_squared)
  THEN:
    h(X) :=
      0.5*log_b(2*pi*e*sigma_squared)

MULTIVARIATE_GAUSSIAN_ENTROPY:
  IF:
    X ~ N_d(mu,Sigma)
  THEN:
    h(X) :=
      0.5*log_b[
        (2*pi*e)^d * det(Sigma)
      ]

  require:
    Sigma_POSITIVE_DEFINITE

MAXIMUM_ENTROPY_PRINCIPLE:
  DEFINE:
    SELECT_DISTRIBUTION_MAXIMISING_ENTROPY
    SUBJECT_TO:
      SPECIFIED_CONSTRAINTS

  examples:
    FIXED_FINITE_SUPPORT:
      UNIFORM_DISTRIBUTION

    FIXED_MEAN_AND_VARIANCE_ON_REAL_LINE:
      GAUSSIAN_DISTRIBUTION

ENTROPY_RATE:
  H_rate :=
    LIMIT[n->infinity]
      H(X_1,...,X_n)/n
    WHEN_LIMIT_EXISTS

  role:
    LONG_RUN_AVERAGE_INFORMATION_PER_SYMBOL

IID_SOURCE:
  H(X_1,...,X_n) :=
    n*H(X_1)

  entropy_rate:
    H_rate := H(X_1)

MARKOV_CHAIN_ENTROPY_RATE:
  stationary_chain:
    H_rate :=
      -SUM_i pi_i
       SUM_j P_ij log_b P_ij

SOURCE_CODING_CONNECTION:
  entropy:
    FUNDAMENTAL_ASYMPTOTIC_LOSSLESS_COMPRESSION_LIMIT
    UNDER:
      STANDARD_SOURCE_CODING_ASSUMPTIONS

PREFIX_CODE_BOUND:
  expected_length:
    L

  standard_relation:
    H(X) <= L < H(X)+1
    FOR:
      APPROPRIATE_BINARY_PREFIX_CODE

HUFFMAN_CONNECTION:
  HUFFMAN_CODING:
    CONSTRUCTS_OPTIMAL_SYMBOL_BY_SYMBOL_PREFIX_CODE

INFORMATION_GAIN:
  IG(Y;X) :=
    H(Y)-H(Y|X)

  equivalent:
    I(X;Y)

NORMALISED_ENTROPY:
  H_norm(X) :=
    H(X)/log_b m
  FOR:
    FINITE_ALPHABET_SIZE_m

  range:
    0 <= H_norm <= 1

PERPLEXITY_CONNECTION:
  perplexity :=
    b^(H_b(X))

  interpretation:
    EFFECTIVE_NUMBER_OF_EQUALLY_LIKELY_OUTCOMES
    IN:
      APPROPRIATE_CONTEXT

ESTIMATION:
  plug_in:
    H_hat :=
      -SUM_x p_hat(x) log_b p_hat(x)

  issue:
    FINITE_SAMPLE_BIAS

  alternatives:
    - MILLER_MADOW
    - BAYESIAN_ESTIMATION
    - JACKKNIFE
    - K_NEAREST_NEIGHBOUR_CONTINUOUS_ESTIMATION

APPLICATIONS:
  ENTROPY -> SUPPORTS -> DATA_COMPRESSION
  ENTROPY -> SUPPORTS -> COMMUNICATION_THEORY
  ENTROPY -> SUPPORTS -> MACHINE_LEARNING
  ENTROPY -> SUPPORTS -> STATISTICS
  ENTROPY -> SUPPORTS -> SIGNAL_PROCESSING
  ENTROPY -> SUPPORTS -> CRYPTOGRAPHY

MODEL_CONNECTIONS:
  ENTROPY -> CONNECTS_TO -> INFORMATION_THEORY
  ENTROPY -> CONNECTS_TO -> SELF_INFORMATION
  ENTROPY -> CONNECTS_TO -> MUTUAL_INFORMATION
  ENTROPY -> CONNECTS_TO -> CONDITIONAL_ENTROPY
  ENTROPY -> CONNECTS_TO -> KL_DIVERGENCE
  ENTROPY -> CONNECTS_TO -> SOURCE_CODING
  ENTROPY -> CONNECTS_TO -> STOCHASTIC_PROCESSES

SEMANTIC_EQUIVALENCE:
  "entropy"
    ~= "Shannon entropy"
    IN:
      STANDARD_INFORMATION_THEORY_CONTEXT

  "Shannon entropy"
    ~= "expected self-information"

  "binary entropy"
    ~= "entropy of a Bernoulli random variable"

  EXCLUDE_EQUIVALENCE:
    ENTROPY != VARIANCE
    ENTROPY != MUTUAL_INFORMATION
    ENTROPY != CROSS_ENTROPY
    ENTROPY != KL_DIVERGENCE
    DISCRETE_ENTROPY != DIFFERENTIAL_ENTROPY
    SHANNON_ENTROPY != THERMODYNAMIC_ENTROPY_BY_DEFAULT

VALIDATION_RULES:
  RULE_1:
    PROBABILITIES:
      MUST_SATISFY:
        p(x) >= 0
        SUM_x p(x) = 1

  RULE_2:
    LOG_BASE:
      MUST_BE_EXPLICIT_OR_INFERABLE_FROM_UNIT

  RULE_3:
    ZERO_PROBABILITY:
      USE:
        0*log(0)=0

  RULE_4:
    FINITE_ALPHABET:
      MUST_SATISFY:
        0 <= H(X) <= log_b m

  RULE_5:
    CONDITIONAL_ENTROPY:
      MUST_NOT_EXCEED_UNCONDITIONAL_ENTROPY
      IN:
        STANDARD_DISCRETE_CONTEXT

  RULE_6:
    DIFFERENTIAL_ENTROPY:
      MAY_BE_NEGATIVE

  RULE_7:
    ENTROPY_ESTIMATION:
      MUST_ACCOUNT_FOR_SAMPLE_SIZE_AND_ESTIMATOR_BIAS

  RULE_8:
    ENTROPY:
      MUST_NOT_BE_INTERPRETED_AS_CAUSAL_EFFECT

CONFIDENCE:
  ontology_identity: 0.99
  entropy_definition: 0.99
  discrete_entropy_structure: 0.99
  joint_conditional_structure: 0.99
  differential_entropy_structure: 0.99
  entropy_rate_structure: 0.99
  source_coding_relationship: 0.99

UPDATE_MODEL:
  IF:
    PROBABILITY_DISTRIBUTION_CHANGES
  THEN:
    RECOMPUTE:
      SELF_INFORMATION
      ENTROPY
      JOINT_ENTROPY
      CONDITIONAL_ENTROPY
      MUTUAL_INFORMATION

  IF:
    ALPHABET_CHANGES
  THEN:
    UPDATE:
      SUPPORT
      PROBABILITIES
    REVALIDATE:
      NORMALISATION
      MAXIMUM_ENTROPY_BOUND

  IF:
    NEW_SAMPLE_DATA_ARRIVES
  THEN:
    UPDATE:
      EMPIRICAL_PROBABILITIES
      ENTROPY_ESTIMATE
    REASSESS:
      ESTIMATION_BIAS
      UNCERTAINTY

  PRESERVE:
    RANDOM_VARIABLE_IDENTITY
    ALPHABET
    LOG_BASE
    INFORMATION_UNIT
    DISTRIBUTION_PROVENANCE
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_RANDOM_VARIABLE
    - IDENTIFY_SUPPORT_OR_ALPHABET
    - IDENTIFY_PROBABILITY_DISTRIBUTION
    - VERIFY_NORMALISATION
    - IDENTIFY_LOG_BASE
    - COMPUTE_SELF_INFORMATION
    - COMPUTE_ENTROPY
    - COMPUTE_JOINT_OR_CONDITIONAL_ENTROPY_IF_REQUIRED
    - COMPUTE_ENTROPY_RATE_IF_PROCESS_DATA
    - CONNECT_TO_CODING_OR_MUTUAL_INFORMATION_IF_REQUIRED
    - ESTIMATE_FROM_DATA_IF_REQUIRED
    - RETURN_INFORMATION_UNCERTAINTY_INTERPRETATION

  constraints:
    - PRESERVE_PROBABILITY_NORMALISATION
    - EXPOSE_LOG_BASE_AND_INFORMATION_UNIT
    - DISTINGUISH_DISCRETE_FROM_DIFFERENTIAL_ENTROPY
    - DO_NOT_EQUATE_ENTROPY_WITH_VARIANCE
    - DO_NOT_EQUATE_ENTROPY_WITH_MUTUAL_INFORMATION
    - PRESERVE_SOURCE_AND_DISTRIBUTION_ASSUMPTIONS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "entropy"
  namespace: "probability-statistics/information-theory"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - SELF_INFORMATION
    - PROBABILITY_DISTRIBUTION
    - EXPECTED_INFORMATION

END_FLOW