Discriminant Analysis Ontology

flow "DiscriminantAnalysisOntology" v4.0

# ============================================================
# DISCRIMINANT ANALYSIS ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)
# ============================================================

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/multivariate-statistics/discriminant-analysis-ontology.html"
  title: "Discriminant Analysis Ontology"
  domain: "Probability and Statistics"
  category: "Multivariate Statistics"
  entity: DISCRIMINANT_ANALYSIS
  entity_type: SUPERVISED_MULTIVARIATE_CLASSIFICATION_METHOD

DEFINE DISCRIMINANT_ANALYSIS:
  DISCRIMINANT_ANALYSIS :=
    SUPERVISED_STATISTICAL_METHOD(
      FEATURE_VECTOR,
      KNOWN_CLASS_LABELS,
      CLASS_CONDITIONAL_DISTRIBUTIONS,
      DISCRIMINANT_FUNCTIONS
    )

  PURPOSE:
    CLASSIFY:
      OBSERVATIONS
    INTO:
      PREDEFINED_GROUPS
    USING:
      MULTIVARIATE_FEATURES

CORE_STRUCTURE:
  FEATURE_VECTOR:
    x := [x_1, x_2, ..., x_p]^T

  CLASS_VARIABLE:
    G in {1, 2, ..., K}

  CLASS_PRIOR:
    pi_k := P(G = k)

  CLASS_CONDITIONAL_DENSITY:
    f_k(x) := p(x | G = k)

  POSTERIOR_CLASS_PROBABILITY:
    P(G=k | x)
      proportional_to
    pi_k * f_k(x)

  CLASSIFICATION_RULE:
    ASSIGN(x)
      :=
    argmax_k P(G=k | x)

ENTITIES:
  OBSERVATION:
    type: DATA_ENTITY

  FEATURE:
    type: PREDICTOR_VARIABLE

  FEATURE_VECTOR:
    type: MULTIVARIATE_VECTOR

  CLASS:
    type: PREDEFINED_CATEGORY

  CLASS_LABEL:
    type: SUPERVISED_TARGET

  CLASS_PRIOR:
    type: PROBABILITY

  CLASS_MEAN_VECTOR:
    type: LOCATION_PARAMETER
    symbol: mu_k

  CLASS_COVARIANCE_MATRIX:
    type: DEPENDENCE_MATRIX
    symbol: Sigma_k

  POOLED_COVARIANCE_MATRIX:
    type: COMMON_DEPENDENCE_MATRIX
    symbol: Sigma

  DISCRIMINANT_FUNCTION:
    type: CLASSIFICATION_SCORE_FUNCTION

  DECISION_BOUNDARY:
    type: CLASSIFICATION_BOUNDARY

RELATIONSHIPS:
  DISCRIMINANT_ANALYSIS
    -> USES
    -> LABELLED_DATA

  OBSERVATION
    -> DESCRIBED_BY
    -> FEATURE_VECTOR

  OBSERVATION
    -> HAS_KNOWN_OR_PREDICTED
    -> CLASS_LABEL

  CLASS
    -> HAS
    -> CLASS_PRIOR

  CLASS
    -> HAS
    -> CLASS_MEAN_VECTOR

  CLASS
    -> HAS
    -> CLASS_CONDITIONAL_DISTRIBUTION

  CLASS_CONDITIONAL_DISTRIBUTION
    -> MAY_HAVE
    -> CLASS_COVARIANCE_MATRIX

  DISCRIMINANT_FUNCTION
    -> PRODUCES
    -> CLASS_SCORE

  CLASS_SCORES
    -> DETERMINE
    -> PREDICTED_CLASS

BAYES_CLASSIFICATION:
  BAYES_THEOREM:
    P(G=k | x)
      =
    [
      pi_k f_k(x)
    ]
    /
    [
      SUM[j=1..K]
        pi_j f_j(x)
    ]

  DECISION:
    CHOOSE_CLASS:
      k_star :=
        argmax_k
        P(G=k | x)

  EQUIVALENT_FOR_CLASSIFICATION:
    MAXIMISE:
      log(pi_k)
      + log(f_k(x))

LINEAR_DISCRIMINANT_ANALYSIS:
  alias: LDA

  ASSUMPTION:
    X | G=k
      ~
    N_p(mu_k, Sigma)

  KEY_STRUCTURE:
    ALL_CLASSES_SHARE:
      COMMON_COVARIANCE_MATRIX
      Sigma

  DISCRIMINANT_FUNCTION:
    delta_k(x) :=
      x^T Sigma^(-1) mu_k
      -
      0.5 mu_k^T Sigma^(-1) mu_k
      +
      log(pi_k)

  CLASSIFICATION:
    ASSIGN(x) :=
      argmax_k delta_k(x)

  DECISION_BOUNDARY:
    BETWEEN_CLASS_j_AND_CLASS_k:
      LINEAR_IN(x)

  REQUIRE:
    APPROPRIATE_ESTIMATE_OF:
      COMMON_COVARIANCE_MATRIX

QUADRATIC_DISCRIMINANT_ANALYSIS:
  alias: QDA

  ASSUMPTION:
    X | G=k
      ~
    N_p(mu_k, Sigma_k)

  KEY_STRUCTURE:
    EACH_CLASS_MAY_HAVE:
      DISTINCT_COVARIANCE_MATRIX

  DISCRIMINANT_FUNCTION:
    delta_k(x) :=
      -0.5 log(|Sigma_k|)
      -0.5 (x-mu_k)^T Sigma_k^(-1) (x-mu_k)
      + log(pi_k)

  CLASSIFICATION:
    ASSIGN(x) :=
      argmax_k delta_k(x)

  DECISION_BOUNDARY:
    GENERALLY:
      QUADRATIC_IN(x)

LDA_VS_QDA:
  LDA:
    covariance:
      COMMON_ACROSS_CLASSES
    boundary:
      LINEAR
    parameter_complexity:
      LOWER

  QDA:
    covariance:
      CLASS_SPECIFIC
    boundary:
      QUADRATIC
    parameter_complexity:
      HIGHER

  TRADEOFF:
    LDA:
      MORE_RESTRICTIVE
      BUT:
        MAY_ESTIMATE_MORE_STABLY_WITH_LIMITED_DATA

    QDA:
      MORE_FLEXIBLE
      BUT:
        REQUIRES_MORE_DATA_FOR_COVARIANCE_ESTIMATION

CLASS_PRIORS:
  definition:
    pi_k := P(G=k)

  sources:
    - POPULATION_KNOWLEDGE
    - TRAINING_CLASS_FREQUENCIES
    - EXPLICIT_ANALYST_SPECIFICATION

  effect:
    CLASS_PRIORS
      -> SHIFT
      -> CLASSIFICATION_DECISIONS

  REQUIRE:
    PRIOR_SELECTION:
      SHOULD_BE_REPORTED

COVARIANCE_ESTIMATION:
  LDA:
    estimate:
      POOLED_WITHIN_CLASS_COVARIANCE

  QDA:
    estimate:
      SEPARATE_COVARIANCE_PER_CLASS

  RISKS:
    - SINGULARITY
    - INSTABILITY
    - HIGH_DIMENSIONALITY
    - SMALL_CLASS_SAMPLE_SIZE

  responses:
    - REGULARISATION
    - SHRINKAGE
    - DIMENSIONALITY_REDUCTION
    - FEATURE_SELECTION

DISCRIMINANT_DIRECTIONS:
  IN_CLASSICAL_LDA_DIMENSION_REDUCTION:
    objective:
      MAXIMISE:
        BETWEEN_CLASS_SEPARATION
      RELATIVE_TO:
        WITHIN_CLASS_VARIATION

  BETWEEN_CLASS_SCATTER:
    S_B

  WITHIN_CLASS_SCATTER:
    S_W

  GENERAL_EIGENPROBLEM:
    S_B v :=
      lambda S_W v

  RESULT:
    DISCRIMINANT_AXES

  MAXIMUM_NUMBER_OF_NONZERO_CLASSICAL_AXES:
    min(p, K-1)

FISHER_DISCRIMINANT:
  TWO_CLASS_CONTEXT:
    direction:
      w
      proportional_to
      S_W^(-1)
      (mu_1 - mu_2)

  objective:
    MAXIMISE:
      PROJECTED_BETWEEN_CLASS_SEPARATION
      /
      PROJECTED_WITHIN_CLASS_VARIANCE

  connection:
    FISHER_LINEAR_DISCRIMINANT
      -> CONNECTS_TO
      -> LDA

DECISION_BOUNDARIES:
  DEFINE:
    SET_OF_POINTS:
      WHERE:
        delta_j(x) = delta_k(x)

  LDA:
    boundary_type:
      HYPERPLANE

  QDA:
    boundary_type:
      QUADRATIC_SURFACE

CLASSIFICATION_OUTPUT:
  possible_outputs:
    - PREDICTED_CLASS
    - DISCRIMINANT_SCORE
    - POSTERIOR_CLASS_PROBABILITY
    - DISCRIMINANT_COORDINATES

  REQUIRE:
    DISTINGUISH:
      CLASS_SCORE
      FROM:
        CALIBRATED_PROBABILITY
    UNLESS:
      MODEL_SUPPORTS_PROBABILISTIC_INTERPRETATION

DATA_PREPARATION:
  CHECK:
    - MISSING_VALUES
    - OUTLIERS
    - FEATURE_TYPES
    - CLASS_BALANCE
    - CLASS_SAMPLE_SIZE
    - MULTICOLLINEARITY
    - COVARIANCE_SINGULARITY
    - DISTRIBUTIONAL_ASSUMPTIONS

  OPTIONAL:
    - IMPUTATION
    - CENTERING
    - STANDARDISATION
    - FEATURE_SELECTION
    - DIMENSIONALITY_REDUCTION

  NOTE:
    STANDARDISATION:
      MAY_BE_USEFUL
      FOR:
        NUMERICAL_STABILITY
        INTERPRETATION
        REGULARISED_VARIANTS
      BUT:
        CLASSICAL_LDA_CLASSIFICATION_IS_AFFINE_INVARIANT
        UNDER:
          CONSISTENT_NONSINGULAR_LINEAR_TRANSFORMATION

ASSUMPTIONS:
  CLASSICAL_LDA:
    - MULTIVARIATE_NORMAL_CLASS_CONDITIONAL_DISTRIBUTIONS
    - COMMON_WITHIN_CLASS_COVARIANCE
    - INDEPENDENT_OBSERVATIONS
    - APPROPRIATE_COVARIANCE_ESTIMATION

  CLASSICAL_QDA:
    - MULTIVARIATE_NORMAL_CLASS_CONDITIONAL_DISTRIBUTIONS
    - CLASS_SPECIFIC_COVARIANCES
    - INDEPENDENT_OBSERVATIONS
    - SUFFICIENT_DATA_PER_CLASS

  CAUTION:
    CLASSIFICATION_PERFORMANCE:
      MAY_REMAIN_USEFUL
      EVEN_WHEN:
        NORMALITY_IS_APPROXIMATE
    BUT:
      INFERENTIAL_AND_PROBABILISTIC_INTERPRETATION:
        DEPENDS_ON_MODEL_ASSUMPTIONS

MODEL_EVALUATION:
  USE:
    - TRAIN_TEST_SPLIT
    - CROSS_VALIDATION
    - CONFUSION_MATRIX
    - CLASS_SPECIFIC_RECALL
    - PRECISION
    - SPECIFICITY
    - ROC_ANALYSIS_WHEN_APPROPRIATE
    - LOG_LOSS_FOR_PROBABILISTIC_OUTPUT_WHEN_APPROPRIATE

  CAUTION:
    OVERALL_ACCURACY:
      MAY_BE_MISLEADING
      UNDER:
        CLASS_IMBALANCE

  REQUIRE:
    EVALUATE:
      OUT_OF_SAMPLE_PERFORMANCE

MISCLASSIFICATION_COST:
  DEFAULT:
    MAY_ASSUME:
      EQUAL_COSTS

  GENERAL_DECISION_THEORY:
    CHOOSE_ACTION:
      MINIMISING:
        EXPECTED_LOSS

  IF:
    MISCLASSIFICATION_COSTS_DIFFER

  THEN:
    CLASSIFICATION_RULE:
      SHOULD_ACCOUNT_FOR:
        COST_MATRIX
        POSTERIOR_PROBABILITIES

REGULARISED_DISCRIMINANT_ANALYSIS:
  purpose:
    STABILISE:
      COVARIANCE_ESTIMATION

  useful_when:
    - HIGH_DIMENSIONAL_DATA
    - SMALL_SAMPLE_SIZE
    - STRONG_MULTICOLLINEARITY

  methods:
    - SHRINKAGE_LDA
    - REGULARISED_DISCRIMINANT_ANALYSIS

HIGH_DIMENSIONAL_CONTEXT:
  IF:
    p LARGE_RELATIVE_TO n

  THEN:
    SAMPLE_COVARIANCE:
      MAY_BE:
        SINGULAR_OR_UNSTABLE

  responses:
    - SHRINKAGE
    - REGULARISATION
    - FEATURE_SELECTION
    - PCA_PREPROCESSING
    - SPARSE_DISCRIMINANT_METHODS

DISCRIMINANT_ANALYSIS_VS_LOGISTIC_REGRESSION:
  DISCRIMINANT_ANALYSIS:
    models:
      p(x | class)
      AND:
      class_prior

    derives:
      p(class | x)

  LOGISTIC_REGRESSION:
    models:
      p(class | x)
    DIRECTLY

  RULE:
    DISCRIMINANT_ANALYSIS != LOGISTIC_REGRESSION

DISCRIMINANT_ANALYSIS_VS_CLUSTERING:
  DISCRIMINANT_ANALYSIS:
    SUPERVISED = TRUE
    requires:
      KNOWN_TRAINING_CLASS_LABELS

  CLUSTERING:
    SUPERVISED = FALSE
    discovers:
      GROUP_STRUCTURE_WITHOUT_KNOWN_CLASS_LABELS

  RULE:
    DISCRIMINANT_ANALYSIS != CLUSTER_ANALYSIS

APPLICATIONS:
  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> CLASSIFICATION

  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> PATTERN_RECOGNITION

  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> MEDICAL_CLASSIFICATION

  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> BIOSTATISTICS

  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> CUSTOMER_CLASSIFICATION

  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> QUALITY_CONTROL

  DISCRIMINANT_ANALYSIS
    -> SUPPORTS
    -> MACHINE_LEARNING

MODEL_CONNECTIONS:
  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> MULTIVARIATE_STATISTICS

  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> MULTIVARIATE_NORMAL_DISTRIBUTION

  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> COVARIANCE_MATRIX

  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> BAYES_CLASSIFICATION

  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> CLASSIFICATION

  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> MACHINE_LEARNING

  DISCRIMINANT_ANALYSIS
    -> CONNECTS_TO
    -> DIMENSIONALITY_REDUCTION

SEMANTIC_EQUIVALENCE:
  "discriminant analysis"
    ~= "supervised multivariate discriminant classification"

  "LDA"
    ~= "linear discriminant analysis"

  "QDA"
    ~= "quadratic discriminant analysis"

  "discriminant function"
    ~= "class scoring function"

  "class prior"
    ~= "prior probability of class membership"

  EXCLUDE_EQUIVALENCE:
    DISCRIMINANT_ANALYSIS != CLUSTER_ANALYSIS
    LDA != QDA
    LDA != LOGISTIC_REGRESSION
    CLASS_LABEL != CLUSTER_LABEL
    DISCRIMINANT_SCORE != POSTERIOR_PROBABILITY_BY_DEFAULT

VALIDATION_RULES:
  RULE_1:
    TRAINING_DATA:
      MUST_INCLUDE:
        CLASS_LABELS

  RULE_2:
    FEATURE_DIMENSIONS:
      MUST_BE:
        CONSISTENT_ACROSS_OBSERVATIONS

  RULE_3:
    LDA:
      COMMON_COVARIANCE_ASSUMPTION:
        MUST_BE_ASSESSED
      WHEN:
        MODEL_INTERPRETATION_DEPENDS_ON_CLASSICAL_ASSUMPTIONS

  RULE_4:
    QDA:
      CLASS_SPECIFIC_COVARIANCE_ESTIMATES:
        REQUIRE:
          SUFFICIENT_DATA_AND_RANK

  RULE_5:
    MATRIX_INVERSION:
      MUST_CHECK:
        COVARIANCE_RANK
        NUMERICAL_STABILITY

  RULE_6:
    CLASS_PRIORS:
      MUST_BE:
        EXPLICIT_OR_DOCUMENTED

  RULE_7:
    PERFORMANCE:
      MUST_BE:
        EVALUATED_OUT_OF_SAMPLE

  RULE_8:
    CLASS_IMBALANCE:
      MUST_BE:
        CONSIDERED_IN_METRIC_INTERPRETATION

CONFIDENCE:
  ontology_identity: 0.99
  discriminant_definition: 0.99
  lda_structure: 0.99
  qda_structure: 0.99
  bayes_classification_relationship: 0.99
  covariance_relationship: 0.99
  supervised_classification_distinction: 0.99

UPDATE_MODEL:
  IF:
    NEW_LABELLED_OBSERVATIONS_ARRIVE

  THEN:
    UPDATE:
      CLASS_PRIORS
      CLASS_MEANS
      COVARIANCE_ESTIMATES

    REFIT:
      DISCRIMINANT_FUNCTIONS

    REEVALUATE:
      OUT_OF_SAMPLE_PERFORMANCE

  IF:
    NEW_FEATURES_ARRIVE

  THEN:
    EXPAND:
      FEATURE_VECTOR

    REESTIMATE:
      CLASS_MEANS
      COVARIANCE_STRUCTURE

    REVALIDATE:
      MATRIX_RANK
      SAMPLE_SIZE
      MODEL_ASSUMPTIONS

  IF:
    CLASS_DEFINITIONS_CHANGE

  THEN:
    REBUILD:
      CLASS_LABEL_STRUCTURE
      PRIORS
      CLASS_CONDITIONAL_MODELS

  PRESERVE:
    FEATURE_IDENTITIES
    FEATURE_UNITS
    CLASS_IDENTITIES
    PRIOR_SPECIFICATION
    MODEL_TYPE
    TRAINING_DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_FEATURES
    - IDENTIFY_KNOWN_CLASSES
    - VALIDATE_LABELLED_DATA
    - ESTIMATE_OR_SET_CLASS_PRIORS
    - ESTIMATE_CLASS_MEANS
    - SELECT_LDA_QDA_OR_REGULARISED_MODEL
    - ESTIMATE_COVARIANCE_STRUCTURE
    - VALIDATE_MATRIX_RANK
    - COMPUTE_DISCRIMINANT_FUNCTIONS
    - COMPUTE_POSTERIOR_CLASS_PROBABILITIES_IF_MODELLED
    - ASSIGN_CLASSES
    - EVALUATE_OUT_OF_SAMPLE_PERFORMANCE
    - RETURN_CLASSIFICATION_AND_UNCERTAINTY

  constraints:
    - PRESERVE_SUPERVISED_CLASS_LABEL_STRUCTURE
    - DO_NOT_EQUATE_DISCRIMINANT_ANALYSIS_WITH_CLUSTERING
    - EXPOSE_CLASS_PRIORS
    - EXPOSE_COVARIANCE_ASSUMPTIONS
    - CHECK_MATRIX_RANK_BEFORE_INVERSION
    - DISTINGUISH_LDA_FROM_QDA
    - REPORT_OUT_OF_SAMPLE_EVALUATION

FLOW_HASH:
  canonical_entity: "discriminant-analysis"
  namespace: "probability-statistics/multivariate-statistics"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - CLASS_CONDITIONAL_STRUCTURE
    - DISCRIMINANT_FUNCTION
    - CLASSIFICATION_RULE

END_FLOW