Information Gain Ontology

flow "InformationGainOntology" v4.0

# INFORMATION GAIN ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/information-theory/information-gain-ontology.html"
  title: "Information Gain Ontology"
  domain: "Probability and Statistics"
  category: "Information Theory"
  entity: INFORMATION_GAIN
  entity_type: INFORMATION_THEORETIC_QUANTITY

DEFINE:
  INFORMATION_GAIN :=
    REDUCTION_IN_UNCERTAINTY(
      PRIOR_ENTROPY,
      CONDITIONAL_ENTROPY,
      OBSERVATION_OR_FEATURE
    )

PURPOSE:
  QUANTIFY:
    INFORMATION_PROVIDED_BY_OBSERVATION
    INFORMATION_PROVIDED_BY_FEATURE
    INFORMATION_PROVIDED_BY_EXPERIMENT
    REDUCTION_IN_TARGET_UNCERTAINTY

FOUNDATION:
  TARGET_VARIABLE: Y
  INFORMATION_SOURCE: X
  PRIOR_UNCERTAINTY: H(Y)
  RESIDUAL_UNCERTAINTY: H(Y|X)

  INFORMATION_GAIN:
    IG(Y;X) :=
      H(Y)-H(Y|X)

  equivalence:
    IG(Y;X) := I(X;Y)

ENTITIES:
  INFORMATION_GAIN: UNCERTAINTY_REDUCTION
  TARGET_VARIABLE: VARIABLE_OF_INTEREST
  FEATURE: INFORMATION_SOURCE
  OBSERVATION: EVIDENCE
  PRIOR_ENTROPY: PRE_OBSERVATION_UNCERTAINTY
  CONDITIONAL_ENTROPY: RESIDUAL_UNCERTAINTY
  MUTUAL_INFORMATION: SHARED_INFORMATION
  PRIOR_DISTRIBUTION: PRE_DATA_BELIEF
  POSTERIOR_DISTRIBUTION: UPDATED_BELIEF
  KL_DIVERGENCE: DISTRIBUTIONAL_INFORMATION_CHANGE
  DECISION_TREE_SPLIT: PARTITION_OPERATION

RELATIONSHIPS:
  TARGET_VARIABLE -> HAS -> PRIOR_ENTROPY
  FEATURE -> CONDITIONS -> TARGET_VARIABLE
  CONDITIONING -> PRODUCES -> CONDITIONAL_ENTROPY
  PRIOR_ENTROPY_MINUS_CONDITIONAL_ENTROPY -> DEFINES -> INFORMATION_GAIN
  INFORMATION_GAIN -> EQUIVALENT_TO -> MUTUAL_INFORMATION
  DATA -> UPDATES -> PRIOR_DISTRIBUTION
  PRIOR_TO_POSTERIOR_CHANGE -> QUANTIFIED_BY -> KL_DIVERGENCE

DISCRETE_FORM:
  IG(Y;X) :=
    H(Y)
    - SUM_x p(x) H(Y|X=x)

MUTUAL_INFORMATION_FORM:
  IG(Y;X) :=
    I(X;Y)

  I(X;Y) :=
    SUM_x SUM_y
      p(x,y)
      log_b[
        p(x,y)/(p(x)p(y))
      ]

NONNEGATIVITY:
  IG(Y;X) >= 0

  equality:
    IG(Y;X)=0
    IFF:
      X_AND_Y_INDEPENDENT

UPPER_BOUND:
  IG(Y;X) <= H(Y)

  equality:
    WHEN:
      H(Y|X)=0

CONDITIONAL_INFORMATION_GAIN:
  IG(Y;X|Z) :=
    H(Y|Z)-H(Y|X,Z)

  equivalent:
    I(X;Y|Z)

  role:
    ADDITIONAL_INFORMATION_FROM_X
    AFTER:
      Z_IS_KNOWN

CHAIN_RULE:
  I(Y;X_1,...,X_n)
    =
  SUM[i=1..n]
    I(
      Y;
      X_i
      |
      X_1,...,X_(i-1)
    )

REDUNDANCY:
  IF:
    X_PROVIDES_INFORMATION_ALREADY_IN_Z

  THEN:
    I(Y;X|Z):
      MAY_BE_SMALL

SYNERGY:
  concept:
    FEATURE_COMBINATION_MAY_PROVIDE_INFORMATION
    NOT_AVAILABLE_FROM_FEATURES_INDIVIDUALLY

  caution:
    FORMAL_DEFINITION_DEPENDS_ON_INFORMATION_DECOMPOSITION_FRAMEWORK

DECISION_TREE_INFORMATION_GAIN:
  before_split:
    H(Y)

  after_split:
    H(Y|X)

  gain:
    IG(Y;X) :=
      H(Y)-H(Y|X)

  selection:
    PREFER_HIGH_INFORMATION_GAIN
    SUBJECT_TO:
      TREE_ALGORITHM_CONSTRAINTS

ENTROPY_AFTER_SPLIT:
  H(Y|X) :=
    SUM_x
      P(X=x)
      H(Y|X=x)

  weighting:
    CHILD_NODE_PROBABILITY_OR_SAMPLE_PROPORTION

DECISION_TREE_BIAS:
  raw_information_gain:
    MAY_FAVOUR_HIGH_CARDINALITY_FEATURES

  responses:
    - GAIN_RATIO
    - REGULARISATION
    - MINIMUM_LEAF_SIZE
    - VALIDATION

GAIN_RATIO:
  common_form:
    GAIN_RATIO :=
      IG(Y;X)/H(X)

  purpose:
    PENALISE_HIGH_CARDINALITY_SPLITS

  caution:
    ZERO_OR_SMALL_DENOMINATOR_REQUIRES_HANDLING

FEATURE_SELECTION:
  relevance:
    I(X;Y)

  caution:
    CONSIDER:
      REDUNDANCY
      SAMPLE_SIZE
      ESTIMATION_BIAS
      MULTIPLE_TESTING

CONDITIONAL_FEATURE_SELECTION:
  incremental_value:
    I(X_new;Y | X_selected)

  role:
    MEASURE_NEW_INFORMATION_NOT_ALREADY_PRESENT

BAYESIAN_INFORMATION_GAIN:
  parameter:
    theta

  prior:
    p(theta)

  data:
    D

  posterior:
    p(theta|D)

  realised_gain:
    D_KL(
      p(theta|D)
      ||
      p(theta)
    )

EXPECTED_INFORMATION_GAIN:
  design:
    d

  future_data:
    Y

  EIG(d) :=
    E_Y[
      D_KL(
        p(theta|Y,d)
        ||
        p(theta)
      )
    ]

  equivalent:
    I(theta;Y | d)
    UNDER:
      BAYESIAN_EXPERIMENT_DESIGN_FORMULATION

ACTIVE_LEARNING:
  objective:
    SELECT_NEXT_QUERY
    TO:
      MAXIMISE_EXPECTED_INFORMATION_GAIN

HYPOTHESIS_INFORMATION_GAIN:
  prior:
    p(H)

  posterior:
    p(H|D)

  gain:
    D_KL(
      p(H|D)
      ||
      p(H)
    )

EXPECTED_POSTERIOR_ENTROPY:
  relation:
    EXPECTED_INFORMATION_GAIN :=
      H(theta)
      - E[H(theta|Y)]

  equivalent:
    I(theta;Y)

EXPERIMENTAL_DESIGN:
  candidate_design:
    d

  criterion:
    d_star :=
      argmax_d EIG(d)

  additional_constraints:
    - COST
    - ETHICS
    - FEASIBILITY
    - ROBUSTNESS

KL_DIVERGENCE_CONNECTION:
  realised_bayesian_gain:
    D_KL(POSTERIOR||PRIOR)

  distinction:
    REALISED_GAIN != EXPECTED_GAIN

LOG_LIKELIHOOD_RATIO_CONNECTION:
  evidence:
    log[
      p(data|H_1)
      /
      p(data|H_0)
    ]

  caution:
    INFORMATION_GAIN != LOG_LIKELIHOOD_RATIO_BY_DEFAULT

FISHER_INFORMATION_DISTINCTION:
  INFORMATION_GAIN:
    UNCERTAINTY_REDUCTION_OR_BELIEF_CHANGE

  FISHER_INFORMATION:
    LOCAL_EXPECTED_PARAMETER_INFORMATION

  rule:
    INFORMATION_GAIN != FISHER_INFORMATION

DATA_PROCESSING:
  IF:
    Y -> X -> Z
    IS_MARKOV_CHAIN

  THEN:
    I(Y;Z) <= I(Y;X)

CONTINUOUS_VARIABLES:
  information_gain:
    MAY_USE:
      MUTUAL_INFORMATION
      KL_DIVERGENCE

  caution:
    DIFFERENTIAL_ENTROPY_MAY_BE_NEGATIVE
    BUT:
      MUTUAL_INFORMATION_REMAINS_NONNEGATIVE

ESTIMATION_DISCRETE:
  estimator:
    IG_hat :=
      H_hat(Y)
      - H_hat(Y|X)

  issue:
    FINITE_SAMPLE_BIAS

ESTIMATION_CONTINUOUS:
  methods:
    - DISCRETISATION
    - KERNEL_DENSITY
    - K_NEAREST_NEIGHBOUR
    - PARAMETRIC_MODELS
    - NEURAL_MUTUAL_INFORMATION_ESTIMATORS

PERMUTATION_BASELINE:
  flow:
    PERMUTE_TARGET_OR_FEATURE
      -> BREAK_DEPENDENCE
      -> RECOMPUTE_INFORMATION_GAIN
      -> FORM_NULL_DISTRIBUTION

NORMALISED_INFORMATION_GAIN:
  possible_form:
    IG_norm :=
      I(X;Y)/H(Y)

  range:
    0 <= IG_norm <= 1
    WHEN:
      H(Y)>0
      AND:
      STANDARD_DISCRETE_CONTEXT

  caution:
    MULTIPLE_NORMALISATIONS_EXIST

CAUSALITY_DISTINCTION:
  INFORMATION_GAIN:
    MEASURES_INFORMATIONAL_OR_STATISTICAL_DEPENDENCE

  DOES_NOT_BY_ITSELF:
    ESTABLISH_CAUSAL_EFFECT
    ESTABLISH_CAUSAL_DIRECTION

APPLICATIONS:
  INFORMATION_GAIN -> SUPPORTS -> DECISION_TREES
  INFORMATION_GAIN -> SUPPORTS -> FEATURE_SELECTION
  INFORMATION_GAIN -> SUPPORTS -> ACTIVE_LEARNING
  INFORMATION_GAIN -> SUPPORTS -> BAYESIAN_EXPERIMENT_DESIGN
  INFORMATION_GAIN -> SUPPORTS -> SENSOR_SELECTION
  INFORMATION_GAIN -> SUPPORTS -> HYPOTHESIS_DISCRIMINATION
  INFORMATION_GAIN -> SUPPORTS -> REPRESENTATION_LEARNING

MODEL_CONNECTIONS:
  INFORMATION_GAIN -> CONNECTS_TO -> INFORMATION_THEORY
  INFORMATION_GAIN -> CONNECTS_TO -> SHANNON_ENTROPY
  INFORMATION_GAIN -> CONNECTS_TO -> CONDITIONAL_ENTROPY
  INFORMATION_GAIN -> CONNECTS_TO -> MUTUAL_INFORMATION
  INFORMATION_GAIN -> CONNECTS_TO -> KL_DIVERGENCE
  INFORMATION_GAIN -> CONNECTS_TO -> BAYESIAN_STATISTICS
  INFORMATION_GAIN -> CONNECTS_TO -> EXPERIMENTAL_DESIGN

SEMANTIC_EQUIVALENCE:
  "information gain"
    ~= "entropy reduction"
    IN:
      STANDARD_TARGET_FEATURE_CONTEXT

  "information gain"
    ~= "mutual information"
    WHEN:
      IG(Y;X) := H(Y)-H(Y|X)

  "expected information gain"
    ~= "expected posterior-to-prior KL divergence"
    IN:
      BAYESIAN_EXPERIMENT_DESIGN

  EXCLUDE_EQUIVALENCE:
    INFORMATION_GAIN != FISHER_INFORMATION
    INFORMATION_GAIN != LOG_LIKELIHOOD_RATIO_BY_DEFAULT
    INFORMATION_GAIN != CAUSAL_EFFECT
    INFORMATION_GAIN != GAIN_RATIO
    REALISED_INFORMATION_GAIN != EXPECTED_INFORMATION_GAIN
    CONDITIONAL_INFORMATION_GAIN != UNCONDITIONAL_INFORMATION_GAIN

VALIDATION_RULES:
  RULE_1:
    TARGET_AND_INFORMATION_SOURCE_MUST_BE_EXPLICITLY_DEFINED

  RULE_2:
    PROBABILITY_DISTRIBUTIONS_MUST_BE_NORMALISED

  RULE_3:
    ENTROPY_BASE_MUST_BE_CONSISTENT

  RULE_4:
    INFORMATION_GAIN_MUST_BE_NONNEGATIVE
    WHEN:
      DEFINED_AS_MUTUAL_INFORMATION

  RULE_5:
    DECISION_TREE_GAIN_MUST_USE_WEIGHTED_CHILD_ENTROPY

  RULE_6:
    BAYESIAN_REALISED_GAIN_MUST_SPECIFY_PRIOR_POSTERIOR_AND_DATA

  RULE_7:
    EXPECTED_INFORMATION_GAIN_MUST_AVERAGE_OVER_PREDICTIVE_DATA_DISTRIBUTION

  RULE_8:
    INFORMATION_GAIN_MUST_NOT_BE_INTERPRETED_AS_CAUSAL_EFFECT_WITHOUT_CAUSAL_DESIGN

CONFIDENCE:
  ontology_identity: 0.99
  information_gain_definition: 0.99
  mutual_information_equivalence: 0.99
  decision_tree_structure: 0.99
  bayesian_information_gain: 0.99
  experimental_design_relationship: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    TARGET_DISTRIBUTION_CHANGES
  THEN:
    RECOMPUTE:
      PRIOR_ENTROPY
      CONDITIONAL_ENTROPY
      INFORMATION_GAIN

  IF:
    NEW_FEATURE_OR_OBSERVATION_ADDED
  THEN:
    UPDATE:
      JOINT_DISTRIBUTION
    COMPUTE:
      UNCONDITIONAL_INFORMATION_GAIN
      CONDITIONAL_INFORMATION_GAIN

  IF:
    BAYESIAN_DATA_ARRIVES
  THEN:
    UPDATE:
      POSTERIOR_DISTRIBUTION
    COMPUTE:
      REALISED_INFORMATION_GAIN

  IF:
    EXPERIMENT_DESIGN_CHANGES
  THEN:
    UPDATE:
      PREDICTIVE_DISTRIBUTION
    RECOMPUTE:
      EXPECTED_INFORMATION_GAIN

  PRESERVE:
    TARGET_IDENTITY
    FEATURE_IDENTITIES
    PRIOR_DISTRIBUTION
    POSTERIOR_DISTRIBUTION
    LOG_BASE
    ESTIMATION_METHOD
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_TARGET_VARIABLE
    - IDENTIFY_INFORMATION_SOURCE
    - IDENTIFY_PROBABILITY_DISTRIBUTIONS
    - COMPUTE_PRIOR_ENTROPY
    - COMPUTE_CONDITIONAL_ENTROPY
    - COMPUTE_INFORMATION_GAIN
    - CONNECT_TO_MUTUAL_INFORMATION
    - COMPUTE_CONDITIONAL_GAIN_IF_PRIOR_FEATURES_EXIST
    - APPLY_DECISION_TREE_OR_FEATURE_SELECTION_CONTEXT_IF_REQUIRED
    - APPLY_BAYESIAN_POSTERIOR_PRIOR_KL_IF_REQUIRED
    - COMPUTE_EXPECTED_INFORMATION_GAIN_IF_DESIGN_PROBLEM
    - RETURN_UNCERTAINTY_REDUCTION_INTERPRETATION

  constraints:
    - PRESERVE_TARGET_AND_FEATURE_IDENTITIES
    - PRESERVE_PROBABILITY_NORMALISATION
    - EXPOSE_LOG_BASE_AND_INFORMATION_UNIT
    - DISTINGUISH_REALISED_FROM_EXPECTED_INFORMATION_GAIN
    - DISTINGUISH_INFORMATION_GAIN_FROM_FISHER_INFORMATION
    - DO_NOT_INTERPRET_INFORMATION_GAIN_AS_CAUSAL_EFFECT
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "information-gain"
  namespace: "probability-statistics/information-theory"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - PRIOR_ENTROPY
    - CONDITIONAL_ENTROPY
    - UNCERTAINTY_REDUCTION

END_FLOW