Cross Entropy Ontology

flow "CrossEntropyOntology" v4.0

# CROSS ENTROPY ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory/cross-entropy-ontology.html"
  title: "Cross Entropy Ontology"
  domain: "Probability and Statistics"
  category: "Information Theory"
  entity: CROSS_ENTROPY
  entity_type: INFORMATION_THEORETIC_QUANTITY

DEFINE:
  CROSS_ENTROPY :=
    EXPECTED_NEGATIVE_LOG_PROBABILITY(
      TRUE_DISTRIBUTION,
      MODEL_DISTRIBUTION,
      LOGARITHM_BASE
    )

PURPOSE:
  QUANTIFY:
    EXPECTED_LOG_LOSS
    CODING_INEFFICIENCY
    MODEL_MISMATCH
    PREDICTIVE_DISTRIBUTION_QUALITY

FOUNDATION:
  TRUE_DISTRIBUTION: P
  MODEL_DISTRIBUTION: Q

  CROSS_ENTROPY:
    H(P,Q) :=
      -SUM_x p(x) log_b q(x)

  expectation:
    H(P,Q) :=
      E_{X~P}[-log_b q(X)]

ENTITIES:
  CROSS_ENTROPY: EXPECTED_LOG_LOSS
  TRUE_DISTRIBUTION: DATA_GENERATING_DISTRIBUTION
  MODEL_DISTRIBUTION: APPROXIMATING_DISTRIBUTION
  SHANNON_ENTROPY: TRUE_DISTRIBUTION_ENTROPY
  KL_DIVERGENCE: DISTRIBUTIONAL_MISMATCH
  LOG_LOSS: NEGATIVE_LOG_PROBABILITY
  TARGET_LABEL: OBSERVED_CLASS
  PREDICTED_PROBABILITY: MODEL_OUTPUT

RELATIONSHIPS:
  TRUE_DISTRIBUTION -> GENERATES -> OBSERVATIONS
  MODEL_DISTRIBUTION -> ASSIGNS -> PREDICTED_PROBABILITIES
  MODEL_PROBABILITY -> DETERMINES -> LOG_LOSS
  EXPECTED_LOG_LOSS -> DEFINES -> CROSS_ENTROPY
  CROSS_ENTROPY -> DECOMPOSES_INTO -> ENTROPY_PLUS_KL_DIVERGENCE

DISCRETE_FORM:
  H(P,Q) :=
    -SUM_x p(x) log_b q(x)

  support:
    IF p(x)>0 AND q(x)=0
    THEN:
      H(P,Q) := infinity

ENTROPY_DECOMPOSITION:
  H(P,Q) :=
    H(P)
    + D_KL(P||Q)

  implication:
    H(P,Q) >= H(P)

  equality:
    IFF:
      P=Q
    ALMOST_EVERYWHERE

EXCESS_CROSS_ENTROPY:
  H(P,Q)-H(P)
    =
  D_KL(P||Q)

  interpretation:
    EXTRA_EXPECTED_LOG_LOSS_FROM_MODEL_MISMATCH

ASYMMETRY:
  generally:
    H(P,Q) != H(Q,P)

LOG_LOSS:
  single_observation:
    loss(x;Q) :=
      -log_b q(x)

  expected_loss:
    E_P[loss(X;Q)]
      =
    H(P,Q)

PROPER_SCORING_RULE:
  expected_log_loss:
    MINIMISED_BY:
      Q=P
    UNDER:
      STANDARD_PROBABILISTIC_FORECASTING_CONTEXT

BINARY_CROSS_ENTROPY:
  target:
    y IN {0,1}

  predicted_probability:
    q

  loss:
    L :=
      -[
        y log q
        + (1-y) log(1-q)
      ]

MULTICLASS_CROSS_ENTROPY:
  target_distribution:
    y_k

  predicted_distribution:
    q_k

  loss:
    L :=
      -SUM[k=1..K]
        y_k log q_k

  one_hot:
    L :=
      -log q_true_class

SOFT_LABELS:
  loss:
    L :=
      -SUM_k p_k log q_k

LABEL_SMOOTHING:
  action:
    MIX_ONE_HOT_TARGET_WITH_SMOOTH_REFERENCE_DISTRIBUTION

  caution:
    CHANGES_OPTIMISATION_TARGET_AND_LOSS_INTERPRETATION

CONTINUOUS_CROSS_ENTROPY:
  H(P,Q) :=
    -INTEGRAL
      p(x) log_b q(x) dx

  relation:
    H(P,Q) :=
      h(P)+D_KL(P||Q)
    WHEN:
      QUANTITIES_EXIST

  caution:
    DEPENDS_ON_REFERENCE_MEASURE_AND_COORDINATE_SCALE

MAXIMUM_LIKELIHOOD_CONNECTION:
  model:
    q_theta(x)

  negative_log_likelihood:
    NLL(theta) :=
      -SUM_i log q_theta(x_i)

  empirical_cross_entropy:
    H_hat :=
      -(1/n)
      * SUM_i log q_theta(x_i)

  relation:
    MINIMISING_EMPIRICAL_CROSS_ENTROPY
      EQUIVALENT_TO
    MAXIMISING_LIKELIHOOD
    UNDER:
      STANDARD_IID_MODEL

KL_MINIMISATION_CONNECTION:
  H(P,Q_theta) :=
    H(P)+D_KL(P||Q_theta)

  therefore:
    MINIMISE_CROSS_ENTROPY_OVER_Q
      EQUIVALENT_TO
    MINIMISE_FORWARD_KL_OVER_Q

NEURAL_NETWORK_CLASSIFICATION:
  softmax:
    q_k :=
      exp(z_k)
      /
      SUM_j exp(z_j)

  cross_entropy:
    L :=
      -SUM_k y_k log q_k

SOFTMAX_CROSS_ENTROPY:
  stable_form:
    L :=
      LOGSUMEXP(z)
      - z_true_class

GRADIENT:
  softmax_cross_entropy:
    dL/dz_k :=
      q_k-y_k

PERPLEXITY_CONNECTION:
  average_cross_entropy:
    H_cross

  perplexity:
    PP :=
      b^(H_cross)

  language_modelling:
    COMMON_PREDICTIVE_METRIC

CODING_INTERPRETATION:
  TRUE_SOURCE:
    P

  CODE_OPTIMISED_FOR:
    Q

  expected_code_length:
    RELATED_TO:
      H(P,Q)

  excess_length:
    RELATED_TO:
      D_KL(P||Q)

CROSS_ENTROPY_VS_ENTROPY:
  ENTROPY:
    H(P)

  CROSS_ENTROPY:
    H(P,Q)

  equality:
    WHEN:
      Q=P
    UNDER:
      STANDARD_CONDITIONS

CROSS_ENTROPY_VS_KL:
  CROSS_ENTROPY:
    ENTROPY_OF_P
    + MODEL_MISMATCH

  KL:
    MODEL_MISMATCH_ONLY

CROSS_ENTROPY_VS_MUTUAL_INFORMATION:
  CROSS_ENTROPY:
    COMPARES_DISTRIBUTIONS

  MUTUAL_INFORMATION:
    MEASURES_DEPENDENCE_BETWEEN_VARIABLES

WEIGHTED_CROSS_ENTROPY:
  loss:
    L :=
      -SUM_k w_k y_k log q_k

  requirement:
    WEIGHTS_MUST_BE_EXPLICIT

FOCAL_LOSS_CONNECTION:
  common_form:
    FL :=
      -(1-p_t)^gamma
      log(p_t)

  distinction:
    FOCAL_LOSS != STANDARD_CROSS_ENTROPY

CALIBRATION:
  cross_entropy:
    SENSITIVE_TO_PROBABILITY_QUALITY

  overconfident_wrong_prediction:
    INCURS_LARGE_LOG_LOSS

  caution:
    CALIBRATION_SHOULD_ALSO_BE_ASSESSED_SEPARATELY

EMPIRICAL_ESTIMATION:
  estimator:
    H_hat(P,Q) :=
      -(1/n)
      * SUM_i log_b q(x_i)

  convergence:
    H_hat(P,Q)
      -> H(P,Q)
    UNDER:
      APPROPRIATE_CONDITIONS

GENERALISATION:
  training_cross_entropy:
    COMPUTED_ON_TRAINING_DATA

  validation_cross_entropy:
    COMPUTED_ON_HELD_OUT_DATA

  caution:
    LOW_TRAINING_LOSS_DOES_NOT_GUARANTEE_LOW_POPULATION_CROSS_ENTROPY

ZERO_PROBABILITY_PROBLEM:
  IF:
    TRUE_EVENT_OCCURS
    AND:
      q(x)=0

  THEN:
    loss := infinity

  practical_responses:
    - NUMERICAL_CLIPPING
    - SMOOTHING
    - REGULARISATION

  caution:
    CLIPPING_CHANGES_EXACT_OBJECTIVE

APPLICATIONS:
  CROSS_ENTROPY -> SUPPORTS -> CLASSIFICATION
  CROSS_ENTROPY -> SUPPORTS -> LANGUAGE_MODELLING
  CROSS_ENTROPY -> SUPPORTS -> MAXIMUM_LIKELIHOOD_ESTIMATION
  CROSS_ENTROPY -> SUPPORTS -> DISTRIBUTION_APPROXIMATION
  CROSS_ENTROPY -> SUPPORTS -> DATA_COMPRESSION
  CROSS_ENTROPY -> SUPPORTS -> PROBABILISTIC_FORECASTING
  CROSS_ENTROPY -> SUPPORTS -> DEEP_LEARNING

MODEL_CONNECTIONS:
  CROSS_ENTROPY -> CONNECTS_TO -> INFORMATION_THEORY
  CROSS_ENTROPY -> CONNECTS_TO -> SHANNON_ENTROPY
  CROSS_ENTROPY -> CONNECTS_TO -> KL_DIVERGENCE
  CROSS_ENTROPY -> CONNECTS_TO -> LOG_LOSS
  CROSS_ENTROPY -> CONNECTS_TO -> MAXIMUM_LIKELIHOOD
  CROSS_ENTROPY -> CONNECTS_TO -> PERPLEXITY
  CROSS_ENTROPY -> CONNECTS_TO -> PROPER_SCORING_RULES

SEMANTIC_EQUIVALENCE:
  "cross entropy"
    ~= "expected negative log probability assigned by Q to outcomes from P"

  "categorical cross entropy"
    ~= "multiclass logarithmic loss"
    IN:
      CLASSIFICATION_CONTEXT

  "binary cross entropy"
    ~= "binary logarithmic loss"

  "empirical cross entropy"
    ~= "average negative log likelihood"
    UNDER:
      STANDARD_IID_MODEL_CONTEXT

  EXCLUDE_EQUIVALENCE:
    CROSS_ENTROPY != SHANNON_ENTROPY_BY_DEFAULT
    CROSS_ENTROPY != KL_DIVERGENCE
    CROSS_ENTROPY != MUTUAL_INFORMATION
    CROSS_ENTROPY != CLASSIFICATION_ERROR_RATE
    WEIGHTED_CROSS_ENTROPY != ORDINARY_CROSS_ENTROPY
    FOCAL_LOSS != STANDARD_CROSS_ENTROPY

VALIDATION_RULES:
  RULE_1:
    P_AND_Q_MUST_BE_VALID_PROBABILITY_DISTRIBUTIONS

  RULE_2:
    LOG_BASE_MUST_BE_EXPLICIT_OR_INFERABLE_FROM_UNIT

  RULE_3:
    IF_P_POSITIVE_AND_Q_ZERO:
      CROSS_ENTROPY_IS_INFINITE

  RULE_4:
    CROSS_ENTROPY_MUST_SATISFY_ENTROPY_PLUS_KL_DECOMPOSITION
    WHEN:
      QUANTITIES_DEFINED

  RULE_5:
    EMPIRICAL_CROSS_ENTROPY_MUST_BE_DISTINGUISHED_FROM_POPULATION_CROSS_ENTROPY

  RULE_6:
    CLASSIFICATION_TARGET_MUST_MATCH_BINARY_MULTICLASS_OR_SOFT_LABEL_FORMULATION

  RULE_7:
    WEIGHTED_LOSS_MUST_DOCUMENT_WEIGHTS_AND_NORMALISATION

  RULE_8:
    NUMERICAL_IMPLEMENTATION_SHOULD_AVOID_DIRECT_LOG_ZERO_AND_UNSTABLE_SOFTMAX

CONFIDENCE:
  ontology_identity: 0.99
  cross_entropy_definition: 0.99
  entropy_kl_decomposition: 0.99
  log_loss_relationship: 0.99
  maximum_likelihood_relationship: 0.99
  classification_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    TRUE_DISTRIBUTION_CHANGES
  THEN:
    UPDATE:
      P
    RECOMPUTE:
      ENTROPY
      CROSS_ENTROPY
      KL_DIVERGENCE

  IF:
    MODEL_DISTRIBUTION_CHANGES
  THEN:
    UPDATE:
      Q
    RECOMPUTE:
      CROSS_ENTROPY
      KL_DIVERGENCE
      EXPECTED_LOG_LOSS

  IF:
    NEW_SAMPLE_DATA_ARRIVES
  THEN:
    UPDATE:
      EMPIRICAL_CROSS_ENTROPY
      NEGATIVE_LOG_LIKELIHOOD
    REASSESS:
      GENERALISATION
      UNCERTAINTY

  PRESERVE:
    TRUE_DISTRIBUTION_IDENTITY
    MODEL_DISTRIBUTION_IDENTITY
    LOG_BASE
    INFORMATION_UNIT
    LOSS_WEIGHTING
    MODEL_VERSION
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_TRUE_DISTRIBUTION_P
    - IDENTIFY_MODEL_DISTRIBUTION_Q
    - VERIFY_NORMALISATION
    - VERIFY_SUPPORT_COMPATIBILITY
    - IDENTIFY_LOG_BASE
    - COMPUTE_SHANNON_ENTROPY_OF_P
    - COMPUTE_CROSS_ENTROPY
    - COMPUTE_KL_DIVERGENCE
    - CONNECT_TO_LOG_LOSS_OR_LIKELIHOOD_IF_REQUIRED
    - APPLY_CLASSIFICATION_FORM_IF_REQUIRED
    - ESTIMATE_FROM_DATA_IF_REQUIRED
    - RETURN_MODEL_MISMATCH_AND_LOG_LOSS_INTERPRETATION

  constraints:
    - PRESERVE_TRUE_AND_MODEL_DISTRIBUTION_IDENTITIES
    - PRESERVE_PROBABILITY_NORMALISATION
    - EXPOSE_LOG_BASE_AND_INFORMATION_UNIT
    - DISTINGUISH_CROSS_ENTROPY_FROM_ENTROPY
    - DISTINGUISH_CROSS_ENTROPY_FROM_KL_DIVERGENCE
    - DISTINGUISH_EMPIRICAL_FROM_POPULATION_CROSS_ENTROPY
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "cross-entropy"
  namespace: "probability-statistics/information-theory"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - TRUE_DISTRIBUTION
    - MODEL_DISTRIBUTION
    - EXPECTED_NEGATIVE_LOG_PROBABILITY

END_FLOW