Shannon Entropy Ontology

flow "ShannonEntropyOntology" v4.0

# SHANNON ENTROPY ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory/shannon-entropy-ontology.html"
  title: "Shannon Entropy Ontology"
  domain: "Probability and Statistics"
  category: "Information Theory"
  entity: SHANNON_ENTROPY
  entity_type: INFORMATION_THEORETIC_QUANTITY

DEFINE:
  SHANNON_ENTROPY :=
    EXPECTED_SELF_INFORMATION(
      DISCRETE_RANDOM_VARIABLE,
      PROBABILITY_MASS_FUNCTION,
      LOGARITHM_BASE
    )

PURPOSE:
  QUANTIFY:
    AVERAGE_UNCERTAINTY
    AVERAGE_INFORMATION_CONTENT
    RANDOMNESS
    LOSSLESS_SOURCE_CODING_LIMIT

FOUNDATION:
  RANDOM_VARIABLE: X
  ALPHABET: {x_1,...,x_m}
  PROBABILITY:
    p_i := P(X=x_i)
  SELF_INFORMATION:
    I(x_i) := -log_b p_i
  SHANNON_ENTROPY:
    H_b(X) := -SUM_i p_i log_b p_i

ENTITIES:
  SHANNON_ENTROPY: EXPECTED_INFORMATION
  SELF_INFORMATION: SURPRISAL
  RANDOM_VARIABLE: DISCRETE_PROBABILISTIC_VARIABLE
  ALPHABET: FINITE_OR_COUNTABLE_SYMBOL_SET
  PROBABILITY_MASS_FUNCTION: DISCRETE_DISTRIBUTION
  LOGARITHM_BASE: INFORMATION_SCALE_PARAMETER
  BIT: BINARY_INFORMATION_UNIT
  NAT: NATURAL_LOG_INFORMATION_UNIT

RELATIONSHIPS:
  RANDOM_VARIABLE -> TAKES_VALUES_IN -> ALPHABET
  RANDOM_VARIABLE -> HAS -> PROBABILITY_MASS_FUNCTION
  OUTCOME_PROBABILITY -> DETERMINES -> SELF_INFORMATION
  EXPECTATION_OF_SELF_INFORMATION -> DEFINES -> SHANNON_ENTROPY
  LOGARITHM_BASE -> DETERMINES -> INFORMATION_UNIT

FORMULA:
  H_b(X) :=
    -SUM_x p(x) log_b p(x)

  equivalent:
    H_b(X) :=
      E[-log_b p(X)]

  convention:
    0*log_b(0) := 0

INFORMATION_UNITS:
  BASE_2:
    unit: BIT

  BASE_e:
    unit: NAT

  conversion:
    H_b(X) :=
      H_c(X)/log_c(b)

NONNEGATIVITY:
  H(X) >= 0

  equality:
    H(X)=0
    IFF:
      DISTRIBUTION_IS_DEGENERATE

MAXIMUM_ENTROPY:
  finite_alphabet_size:
    m

  bound:
    H_b(X) <= log_b m

  equality:
    IFF:
      p_i = 1/m
      FOR_ALL_i

MINIMUM_ENTROPY:
  H(X) := 0

  attained_when:
    ONE_OUTCOME_HAS_PROBABILITY_ONE

CONCAVITY:
  H(p_1,...,p_m):
    CONCAVE_IN_PROBABILITY_VECTOR

SYMMETRY:
  entropy:
    INVARIANT_UNDER:
      PERMUTATION_OF_SYMBOL_LABELS

CONTINUITY:
  entropy:
    CONTINUOUS_IN:
      FINITE_PROBABILITY_VECTOR

EXPANSIBILITY:
  H(p_1,...,p_m,0)
    =
  H(p_1,...,p_m)

BINARY_ENTROPY:
  X ~ BERNOULLI(p)

  H_2(p) :=
    -p log_2 p
    -(1-p) log_2(1-p)

  properties:
    H_2(0)=0
    H_2(1)=0
    H_2(0.5)=1
    H_2(p)=H_2(1-p)

JOINT_ENTROPY:
  H(X,Y) :=
    -SUM_x SUM_y
      p(x,y) log_b p(x,y)

  bounds:
    H(X,Y) >= max(H(X),H(Y))
    H(X,Y) <= H(X)+H(Y)

CONDITIONAL_ENTROPY:
  H(Y|X) :=
    -SUM_x SUM_y
      p(x,y) log_b p(y|x)

  identity:
    H(Y|X) :=
      H(X,Y)-H(X)

  bounds:
    0 <= H(Y|X) <= H(Y)

CHAIN_RULE:
  H(X,Y) :=
    H(X)+H(Y|X)

  general:
    H(X_1,...,X_n)
      =
    SUM[i=1..n]
      H(X_i | X_1,...,X_(i-1))

INDEPENDENCE:
  IF:
    X_INDEPENDENT_Y

  THEN:
    H(X,Y) :=
      H(X)+H(Y)

    H(X|Y) :=
      H(X)

SUBADDITIVITY:
  H(X,Y) <= H(X)+H(Y)

  equality:
    IFF:
      X_AND_Y_INDEPENDENT

MUTUAL_INFORMATION_CONNECTION:
  I(X;Y) :=
    H(X)+H(Y)-H(X,Y)

  equivalent:
    I(X;Y) :=
      H(X)-H(X|Y)

DETERMINISTIC_MAPPING:
  IF:
    Y := f(X)

  THEN:
    H(Y|X) := 0
    H(Y) <= H(X)

SOURCE_ENTROPY:
  IID_SOURCE:
    H(X_1,...,X_n) :=
      n*H(X)

  entropy_rate:
    H_rate := H(X)

SOURCE_CODING_THEOREM:
  role:
    SHANNON_ENTROPY:
      DEFINES_ASYMPTOTIC_LOSSLESS_COMPRESSION_LIMIT
    FOR:
      STANDARD_DISCRETE_MEMORYLESS_SOURCE

PREFIX_CODE_BOUND:
  expected_length:
    L

  standard_binary_relation:
    H_2(X) <= L < H_2(X)+1
    FOR:
      SUITABLE_PREFIX_CODE

HUFFMAN_CODING:
  property:
    MINIMISES_EXPECTED_CODEWORD_LENGTH
    AMONG:
      SYMBOL_BY_SYMBOL_PREFIX_CODES
    UNDER:
      STANDARD_HUFFMAN_SETTING

TYPICAL_SET:
  IID_SOURCE:
    typical_sequence_probability:
      approximately:
        2^(-nH_2(X))

    typical_set_size:
      approximately:
        2^(nH_2(X))

ASYMPTOTIC_EQUIPARTITION_PROPERTY:
  IID_case:
    -1/n * log_2 p(X_1,...,X_n)
      -> H_2(X)
    UNDER:
      STANDARD_CONDITIONS

CROSS_ENTROPY_RELATION:
  H(P,Q) :=
    H(P)+D_KL(P||Q)

KL_DIVERGENCE_RELATION:
  D_KL(P||Q) :=
    H(P,Q)-H(P)

PERPLEXITY:
  PP(X) :=
    b^(H_b(X))

  interpretation:
    EFFECTIVE_NUMBER_OF_EQUALLY_LIKELY_SYMBOLS
    IN:
      APPROPRIATE_CONTEXT

NORMALISED_SHANNON_ENTROPY:
  H_norm :=
    H_b(X)/log_b m

  range:
    0 <= H_norm <= 1

  caution:
    NORMALISATION_DEFINITION_IS_CONTEXT_DEPENDENT

ESTIMATION:
  empirical_probability:
    p_hat_i := n_i/n

  plug_in_estimator:
    H_hat :=
      -SUM_i p_hat_i log_b p_hat_i

  issue:
    FINITE_SAMPLE_DOWNWARD_BIAS

  corrections:
    - MILLER_MADOW
    - JACKKNIFE
    - BAYESIAN_ESTIMATION

SHANNON_VS_DIFFERENTIAL_ENTROPY:
  SHANNON_DISCRETE:
    NONNEGATIVE

  DIFFERENTIAL:
    MAY_BE_NEGATIVE
    SCALE_DEPENDENT

  rule:
    SHANNON_ENTROPY != DIFFERENTIAL_ENTROPY

SHANNON_VS_THERMODYNAMIC_ENTROPY:
  SHANNON:
    INFORMATION_THEORETIC_QUANTITY

  THERMODYNAMIC:
    PHYSICAL_STATE_QUANTITY

  rule:
    NOT_IDENTICAL_BY_DEFAULT

SHANNON_VS_RENYI_ENTROPY:
  RENYI:
    H_alpha(X) :=
      [1/(1-alpha)]
      * log(
          SUM_x p(x)^alpha
        )

  limit:
    AS alpha -> 1:
      H_alpha -> SHANNON_ENTROPY
    UNDER:
      APPROPRIATE_CONDITIONS

SHANNON_VS_MIN_ENTROPY:
  MIN_ENTROPY:
    H_infinity(X) :=
      -log(
        max_x p(x)
      )

  distinction:
    SHANNON_ENTROPY:
      AVERAGE_INFORMATION

    MIN_ENTROPY:
      WORST_CASE_PREDICTABILITY_MEASURE

APPLICATIONS:
  SHANNON_ENTROPY -> SUPPORTS -> LOSSLESS_COMPRESSION
  SHANNON_ENTROPY -> SUPPORTS -> COMMUNICATION_THEORY
  SHANNON_ENTROPY -> SUPPORTS -> MACHINE_LEARNING
  SHANNON_ENTROPY -> SUPPORTS -> CRYPTOGRAPHY
  SHANNON_ENTROPY -> SUPPORTS -> SIGNAL_PROCESSING
  SHANNON_ENTROPY -> SUPPORTS -> STATISTICAL_INFERENCE

MODEL_CONNECTIONS:
  SHANNON_ENTROPY -> CONNECTS_TO -> INFORMATION_THEORY
  SHANNON_ENTROPY -> CONNECTS_TO -> SELF_INFORMATION
  SHANNON_ENTROPY -> CONNECTS_TO -> CONDITIONAL_ENTROPY
  SHANNON_ENTROPY -> CONNECTS_TO -> MUTUAL_INFORMATION
  SHANNON_ENTROPY -> CONNECTS_TO -> KL_DIVERGENCE
  SHANNON_ENTROPY -> CONNECTS_TO -> SOURCE_CODING
  SHANNON_ENTROPY -> CONNECTS_TO -> TYPICAL_SET

SEMANTIC_EQUIVALENCE:
  "Shannon entropy"
    ~= "expected self-information"

  "information entropy"
    ~= "Shannon entropy"
    IN:
      STANDARD_INFORMATION_THEORY_CONTEXT

  "source entropy"
    ~= "average information per source symbol"
    IN:
      DISCRETE_MEMORYLESS_SOURCE_CONTEXT

  EXCLUDE_EQUIVALENCE:
    SHANNON_ENTROPY != VARIANCE
    SHANNON_ENTROPY != CROSS_ENTROPY
    SHANNON_ENTROPY != KL_DIVERGENCE
    SHANNON_ENTROPY != MUTUAL_INFORMATION
    SHANNON_ENTROPY != DIFFERENTIAL_ENTROPY
    SHANNON_ENTROPY != THERMODYNAMIC_ENTROPY_BY_DEFAULT
    SHANNON_ENTROPY != MIN_ENTROPY

VALIDATION_RULES:
  RULE_1:
    PROBABILITY_VECTOR:
      MUST_SATISFY:
        p_i >= 0
        SUM_i p_i = 1

  RULE_2:
    LOG_BASE:
      MUST_BE_EXPLICIT_OR_INFERABLE_FROM_INFORMATION_UNIT

  RULE_3:
    ZERO_PROBABILITY:
      MUST_USE:
        0*log(0)=0

  RULE_4:
    FINITE_ALPHABET:
      MUST_SATISFY:
        0 <= H_b(X) <= log_b m

  RULE_5:
    MAXIMUM_ENTROPY:
      REQUIRES_UNIFORM_DISTRIBUTION
      ON:
        FIXED_FINITE_ALPHABET

  RULE_6:
    ENTROPY_ESTIMATE:
      MUST_DISTINGUISH_TRUE_ENTROPY_FROM_SAMPLE_ESTIMATE

  RULE_7:
    SOURCE_CODING_INTERPRETATION:
      MUST_PRESERVE_SOURCE_MODEL_AND_ASYMPTOTIC_CONTEXT

  RULE_8:
    SHANNON_ENTROPY:
      MUST_NOT_BE_INTERPRETED_AS_CAUSAL_QUANTITY

CONFIDENCE:
  ontology_identity: 0.99
  shannon_entropy_definition: 0.99
  entropy_properties: 0.99
  chain_rule_structure: 0.99
  source_coding_relationship: 0.99
  estimation_structure: 0.98
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    SYMBOL_PROBABILITIES_CHANGE
  THEN:
    RECOMPUTE:
      SELF_INFORMATION
      SHANNON_ENTROPY
      PERPLEXITY
      NORMALISED_ENTROPY

  IF:
    ALPHABET_CHANGES
  THEN:
    UPDATE:
      SYMBOL_SET
      PROBABILITY_VECTOR
    REVALIDATE:
      NORMALISATION
      MAXIMUM_ENTROPY_BOUND

  IF:
    NEW_SAMPLE_DATA_ARRIVES
  THEN:
    UPDATE:
      SYMBOL_COUNTS
      EMPIRICAL_PROBABILITIES
      ENTROPY_ESTIMATE
    REASSESS:
      ESTIMATOR_BIAS
      SUPPORT_COVERAGE

  PRESERVE:
    RANDOM_VARIABLE_IDENTITY
    SYMBOL_IDENTITIES
    LOG_BASE
    INFORMATION_UNIT
    SOURCE_MODEL
    DISTRIBUTION_PROVENANCE
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DISCRETE_RANDOM_VARIABLE
    - IDENTIFY_ALPHABET
    - IDENTIFY_PROBABILITY_MASS_FUNCTION
    - VERIFY_NORMALISATION
    - IDENTIFY_LOG_BASE
    - COMPUTE_SELF_INFORMATION
    - COMPUTE_SHANNON_ENTROPY
    - CHECK_ENTROPY_BOUNDS
    - COMPUTE_JOINT_OR_CONDITIONAL_ENTROPY_IF_REQUIRED
    - CONNECT_TO_MUTUAL_INFORMATION_IF_REQUIRED
    - APPLY_SOURCE_CODING_RESULTS_IF_REQUIRED
    - ESTIMATE_ENTROPY_FROM_DATA_IF_REQUIRED
    - RETURN_INFORMATION_UNCERTAINTY_INTERPRETATION

  constraints:
    - PRESERVE_PROBABILITY_NORMALISATION
    - EXPOSE_LOG_BASE_AND_INFORMATION_UNIT
    - DISTINGUISH_SHANNON_FROM_DIFFERENTIAL_ENTROPY
    - DO_NOT_EQUATE_SHANNON_ENTROPY_WITH_VARIANCE
    - DO_NOT_EQUATE_SHANNON_ENTROPY_WITH_MUTUAL_INFORMATION
    - PRESERVE_SOURCE_CODING_ASSUMPTIONS
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "shannon-entropy"
  namespace: "probability-statistics/information-theory"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - EXPECTED_SELF_INFORMATION
    - DISCRETE_PROBABILITY_DISTRIBUTION
    - SOURCE_CODING_LIMIT

END_FLOW