Principal Component Analysis

flow "PrincipalComponentAnalysis" v4.0

# ============================================================
# PRINCIPAL COMPONENT ANALYSIS
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)
# ============================================================

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/multivariate-statistics/principal-component-analysis.html"
  title: "Principal Component Analysis"
  domain: "Probability and Statistics"
  category: "Multivariate Statistics"
  entity: PRINCIPAL_COMPONENT_ANALYSIS
  aliases:
    - PCA
  entity_type: DIMENSIONALITY_REDUCTION_METHOD

DEFINE PRINCIPAL_COMPONENT_ANALYSIS:
  PRINCIPAL_COMPONENT_ANALYSIS :=
    LINEAR_TRANSFORMATION(
      CORRELATED_VARIABLES
      -> ORTHOGONAL_PRINCIPAL_COMPONENTS
    )

  OBJECTIVE:
    REPRESENT_MAXIMUM_VARIANCE
    USING:
      FEWER_DIMENSIONS

CORE_STRUCTURE:
  DATA_MATRIX:
    X:
      dimensions: n x p

  CENTERED_DATA:
    X_c :=
      X - COLUMN_MEANS(X)

  COVARIANCE_MATRIX:
    S :=
      (1/(n-1))
      * X_c^T X_c

  EIGEN_DECOMPOSITION:
    S v_k :=
      lambda_k v_k

  ORDER:
    lambda_1 >= lambda_2 >= ... >= lambda_p >= 0

  PRINCIPAL_COMPONENT_SCORE:
    z_k :=
      X_c v_k

ENTITIES:
  OBSERVATION:
    type: DATA_VECTOR

  VARIABLE:
    type: STATISTICAL_VARIABLE

  DATA_MATRIX:
    type: MATRIX

  CENTERED_DATA:
    type: TRANSFORMED_DATA_MATRIX

  STANDARDISED_DATA:
    type: TRANSFORMED_DATA_MATRIX

  COVARIANCE_MATRIX:
    type: STATISTICAL_MATRIX

  CORRELATION_MATRIX:
    type: STANDARDISED_STATISTICAL_MATRIX

  EIGENVALUE:
    type: VARIANCE_QUANTITY
    symbol: lambda_k

  EIGENVECTOR:
    type: PRINCIPAL_DIRECTION
    symbol: v_k

  PRINCIPAL_COMPONENT:
    type: LINEAR_COMBINATION

  COMPONENT_SCORE:
    type: PROJECTED_OBSERVATION

  LOADING:
    type: VARIABLE_COMPONENT_RELATION

RELATIONSHIPS:
  PRINCIPAL_COMPONENT_ANALYSIS
    -> OPERATES_ON
    -> MULTIVARIATE_DATA

  MULTIVARIATE_DATA
    -> CENTERED_TO
    -> CENTERED_DATA

  CENTERED_DATA
    -> DEFINES
    -> COVARIANCE_MATRIX

  COVARIANCE_MATRIX
    -> EIGEN_DECOMPOSES_INTO
    -> EIGENVALUES

  COVARIANCE_MATRIX
    -> EIGEN_DECOMPOSES_INTO
    -> EIGENVECTORS

  EIGENVECTOR
    -> DEFINES
    -> PRINCIPAL_DIRECTION

  DATA
    -> PROJECTED_ONTO
    -> PRINCIPAL_DIRECTION

  PROJECTION
    -> PRODUCES
    -> COMPONENT_SCORE

  EIGENVALUE
    -> REPRESENTS
    -> COMPONENT_VARIANCE

DATA_PREPARATION:
  INPUT:
    X

  CENTER:
    FOR_EACH_VARIABLE_j:
      x_ij :=
        x_ij - x_bar_j

  OPTIONAL_STANDARDISATION:
    FOR_EACH_VARIABLE_j:
      z_ij :=
        (x_ij - x_bar_j)
        /
        s_j

  USE_STANDARDISATION_WHEN:
    - VARIABLE_SCALES_DIFFER_MATERIALLY
    - VARIABLE_UNITS_DIFFER
    - RELATIVE_VARIATION_IS_MORE_RELEVANT_THAN_ABSOLUTE_VARIATION

  RESULT:
    ANALYSIS_MATRIX

COVARIANCE_VS_CORRELATION:
  COVARIANCE_BASED_PCA:
    use_when:
      VARIABLES_HAVE_COMPARABLE_SCALES
      AND:
      ABSOLUTE_VARIANCE_IS_MEANINGFUL

  CORRELATION_BASED_PCA:
    equivalent_to:
      PCA_ON_STANDARDISED_VARIABLES

    use_when:
      VARIABLE_SCALES_OR_UNITS_DIFFER

EIGEN_STRUCTURE:
  FOR_EACH_COMPONENT_k:
    S v_k :=
      lambda_k v_k

  REQUIRE:
    ||v_k|| = 1

  ORTHOGONALITY:
    FOR:
      j != k

    v_j^T v_k = 0

  VARIANCE:
    Var(PC_k) :=
      lambda_k

  TOTAL_VARIANCE:
    SUM[k=1..p] lambda_k
      =
    trace(S)

PRINCIPAL_COMPONENT:
  FOR_OBSERVATION_VECTOR:
    x

  CENTERED_VECTOR:
    x_c :=
      x - x_bar

  COMPONENT_k:
    PC_k :=
      v_k^T x_c

  INTERPRETATION:
    PRINCIPAL_COMPONENT:
      LINEAR_COMBINATION_OF
      ORIGINAL_VARIABLES

FIRST_COMPONENT:
  PC_1:
    direction:
      v_1

    objective:
      MAXIMISE:
        Var(v_1^T X)

    subject_to:
      ||v_1|| = 1

SUBSEQUENT_COMPONENTS:
  FOR:
    k > 1

  objective:
    MAXIMISE:
      REMAINING_VARIANCE

  subject_to:
    ||v_k|| = 1

    v_k:
      ORTHOGONAL_TO:
        v_1, ..., v_(k-1)

EXPLAINED_VARIANCE:
  COMPONENT_k:
    explained_variance :=
      lambda_k

  EXPLAINED_VARIANCE_RATIO:
    EVR_k :=
      lambda_k
      /
      SUM[j=1..p] lambda_j

  CUMULATIVE_EXPLAINED_VARIANCE:
    CEV_m :=
      SUM[k=1..m] lambda_k
      /
      SUM[j=1..p] lambda_j

COMPONENT_SELECTION:
  methods:
    - CUMULATIVE_EXPLAINED_VARIANCE
    - SCREE_PLOT
    - KAISER_CRITERION_FOR_CORRELATION_PCA
    - CROSS_VALIDATION_WHEN_APPROPRIATE
    - DOMAIN_INTERPRETABILITY

  RULE:
    COMPONENT_COUNT:
      SHOULD_NOT_BE_SELECTED
      FROM:
        A_SINGLE_UNIVERSAL_THRESHOLD

LOADINGS:
  GENERAL_ROLE:
    QUANTIFY:
      RELATIONSHIP_BETWEEN
      ORIGINAL_VARIABLES
      AND
      PRINCIPAL_COMPONENTS

  NOTE:
    LOADING_DEFINITION:
      MAY_VARY_BY_CONVENTION

  POSSIBLE_REPRESENTATION:
    loading_jk :=
      v_jk * sqrt(lambda_k)

  REQUIRE:
    REPORT_CONVENTION_USED

SCORES:
  SCORE_MATRIX:
    Z :=
      X_c V

  where:
    V :=
      [v_1, v_2, ..., v_m]

  dimensions:
    n x m

  purpose:
    REPRESENT_OBSERVATIONS
    IN:
      PRINCIPAL_COMPONENT_SPACE

DIMENSIONALITY_REDUCTION:
  SELECT:
    m < p

  PROJECTION:
    Z_m :=
      X_c V_m

  APPROXIMATE_RECONSTRUCTION:
    X_hat_c :=
      Z_m V_m^T

  ORIGINAL_SCALE_RECONSTRUCTION:
    X_hat :=
      X_hat_c
      + COLUMN_MEANS(X)

  property:
    PCA_SUBSPACE:
      MINIMISES_SQUARED_RECONSTRUCTION_ERROR
      AMONG:
        m_DIMENSIONAL_LINEAR_SUBSPACES

SINGULAR_VALUE_DECOMPOSITION:
  CENTERED_DATA:
    X_c :=
      U D V^T

  PCA_DIRECTIONS:
    columns_of:
      V

  SAMPLE_COVARIANCE_EIGENVALUES:
    lambda_k :=
      d_k^2 / (n-1)

  SCORES:
    X_c V :=
      U D

  advantage:
    SVD:
      OFTEN_COMPUTATIONALLY_STABLE
      AND:
      AVOIDS_EXPLICIT_COVARIANCE_MATRIX_FORMATION

GEOMETRIC_INTERPRETATION:
  ORIGINAL_COORDINATE_SYSTEM:
    axes:
      ORIGINAL_VARIABLES

  PCA:
    ROTATES_OR_PROJECTS_COORDINATE_SYSTEM
    TO:
      ORTHOGONAL_DIRECTIONS_OF_MAXIMUM_VARIANCE

  FIRST_AXIS:
    MAXIMUM_VARIANCE_DIRECTION

  SECOND_AXIS:
    NEXT_MAXIMUM_VARIANCE_DIRECTION
    ORTHOGONAL_TO:
      FIRST_AXIS

  CONTINUE:
    UNTIL:
      p_COMPONENTS

PROPERTIES:
  LINEAR:
    TRUE

  UNSUPERVISED:
    TRUE

  COMPONENTS:
    ORTHOGONAL = TRUE

  COMPONENT_SCORES:
    UNCOVARIATED_IN_SAMPLE
    UNDER_STANDARD_PCA_CONSTRUCTION

  ORDERED_BY:
    EXPLAINED_VARIANCE

  FULL_COMPONENT_SET:
    PRESERVES:
      TOTAL_VARIANCE
      FULL_LINEAR_INFORMATION
    SUBJECT_TO:
      NUMERICAL_PRECISION

LIMITATIONS:
  PCA:
    CAPTURES:
      LINEAR_VARIANCE_STRUCTURE

  PCA_DOES_NOT_AUTOMATICALLY_CAPTURE:
    - NONLINEAR_STRUCTURE
    - CAUSAL_RELATIONSHIPS
    - CLASS_SEPARATION
    - SEMANTIC_IMPORTANCE

  HIGH_VARIANCE:
    DOES_NOT_NECESSARILY_MEAN:
      HIGH_PREDICTIVE_IMPORTANCE

  OUTLIERS:
    MAY_STRONGLY_AFFECT:
      MEANS
      COVARIANCE
      PRINCIPAL_DIRECTIONS

  SCALING:
    CAN_STRONGLY_CHANGE:
      PCA_RESULTS

PCA_VS_FACTOR_ANALYSIS:
  PCA:
    objective:
      REPRESENT_TOTAL_VARIANCE
      USING:
        ORTHOGONAL_COMPONENTS

  FACTOR_ANALYSIS:
    objective:
      MODEL_SHARED_COVARIANCE
      USING:
        LATENT_FACTORS
        + UNIQUE_ERROR

  RULE:
    PCA != FACTOR_ANALYSIS

PCA_VS_FEATURE_SELECTION:
  PCA:
    CREATES:
      NEW_COMPOSITE_FEATURES

  FEATURE_SELECTION:
    RETAINS:
      SUBSET_OF_ORIGINAL_FEATURES

  RULE:
    DIMENSION_REDUCTION_BY_PCA
      !=
    FEATURE_SELECTION

APPLICATIONS:
  PCA
    -> SUPPORTS
    -> DATA_VISUALISATION

  PCA
    -> SUPPORTS
    -> DIMENSIONALITY_REDUCTION

  PCA
    -> SUPPORTS
    -> NOISE_REDUCTION

  PCA
    -> SUPPORTS
    -> FEATURE_EXTRACTION

  PCA
    -> SUPPORTS
    -> MULTICOLLINEARITY_REDUCTION

  PCA
    -> SUPPORTS
    -> DATA_COMPRESSION

  PCA
    -> SUPPORTS
    -> EXPLORATORY_DATA_ANALYSIS

  PCA
    -> SUPPORTS
    -> MACHINE_LEARNING_PREPROCESSING

MODEL_CONNECTIONS:
  PCA
    -> CONNECTS_TO
    -> MULTIVARIATE_STATISTICS

  PCA
    -> CONNECTS_TO
    -> COVARIANCE_MATRIX

  PCA
    -> CONNECTS_TO
    -> CORRELATION_MATRIX

  PCA
    -> CONNECTS_TO
    -> EIGENVALUES

  PCA
    -> CONNECTS_TO
    -> EIGENVECTORS

  PCA
    -> CONNECTS_TO
    -> SINGULAR_VALUE_DECOMPOSITION

  PCA
    -> CONNECTS_TO
    -> LINEAR_ALGEBRA

  PCA
    -> CONNECTS_TO
    -> DIMENSIONALITY_REDUCTION

SEMANTIC_EQUIVALENCE:
  "principal component analysis"
    ~= "PCA"

  "principal component"
    ~= "variance-maximising orthogonal linear component"

  "principal direction"
    ~= "PCA eigenvector direction"

  "explained variance"
    ~= "variance represented by a principal component"

  EXCLUDE_EQUIVALENCE:
    PCA != FACTOR_ANALYSIS
    PCA != FEATURE_SELECTION
    PCA != INDEPENDENT_COMPONENT_ANALYSIS
    PRINCIPAL_COMPONENT != ORIGINAL_VARIABLE
    HIGH_VARIANCE != CAUSAL_IMPORTANCE

VALIDATION_RULES:
  RULE_1:
    INPUT_DATA:
      MUST_HAVE:
        CONSISTENT_OBSERVATION_AND_VARIABLE_DIMENSIONS

  RULE_2:
    DATA:
      SHOULD_BE_CENTERED
      BEFORE:
        STANDARD_COVARIANCE_PCA

  RULE_3:
    SCALING_DECISION:
      MUST_BE_EXPLICIT
      WHEN:
        VARIABLE_SCALES_DIFFER

  RULE_4:
    COVARIANCE_MATRIX:
      MUST_BE:
        SYMMETRIC
        POSITIVE_SEMIDEFINITE

  RULE_5:
    EIGENVECTORS:
      MUST_BE:
        ORTHONORMAL
      FOR:
        SYMMETRIC_COVARIANCE_MATRIX_EIGENBASIS

  RULE_6:
    EIGENVALUES:
      MUST_BE:
        NONNEGATIVE
      UP_TO:
        NUMERICAL_TOLERANCE

  RULE_7:
    COMPONENT_INTERPRETATION:
      MUST_PRESERVE:
        VARIABLE_LOADING_SIGNS_AND_MAGNITUDES
      WHILE_RECOGNISING:
        EIGENVECTOR_SIGN_INDETERMINACY

  RULE_8:
    REDUCED_REPRESENTATION:
      MUST_REPORT:
        NUMBER_OF_COMPONENTS
        EXPLAINED_VARIANCE

CONFIDENCE:
  ontology_identity: 0.99
  pca_definition: 0.99
  eigenstructure: 0.99
  explained_variance_structure: 0.99
  svd_relationship: 0.99
  dimensionality_reduction_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_OBSERVATIONS_ARRIVE

  THEN:
    UPDATE:
      COLUMN_MEANS
      CENTERED_DATA
      COVARIANCE_MATRIX

    RECOMPUTE:
      EIGENVALUES
      EIGENVECTORS
      COMPONENT_SCORES
      EXPLAINED_VARIANCE

  IF:
    NEW_VARIABLES_ARRIVE

  THEN:
    EXPAND:
      DATA_MATRIX
      COVARIANCE_MATRIX

    RECOMPUTE:
      PCA_MODEL

  IF:
    SCALING_METHOD_CHANGES

  THEN:
    RECOMPUTE:
      TRANSFORMED_DATA
      COVARIANCE_OR_CORRELATION_MATRIX
      PRINCIPAL_COMPONENTS

  PRESERVE:
    VARIABLE_IDENTITIES
    VARIABLE_UNITS
    CENTERING_PARAMETERS
    SCALING_PARAMETERS
    COMPONENT_ORDER
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DATA_MATRIX
    - VALIDATE_DIMENSIONS
    - CHECK_MISSING_VALUES_AND_OUTLIERS
    - DETERMINE_CENTERING
    - DETERMINE_STANDARDISATION
    - COMPUTE_COVARIANCE_OR_CORRELATION_MATRIX
    - COMPUTE_EIGENVALUES_AND_EIGENVECTORS
    - ORDER_COMPONENTS_BY_VARIANCE
    - COMPUTE_COMPONENT_SCORES
    - COMPUTE_EXPLAINED_VARIANCE
    - SELECT_COMPONENT_COUNT_IF_REQUIRED
    - INTERPRET_LOADINGS_AND_SCORES
    - RETURN_REDUCED_REPRESENTATION

  constraints:
    - CENTER_DATA_FOR_STANDARD_PCA
    - EXPOSE_SCALING_DECISION
    - PRESERVE_COMPONENT_ORDER
    - DO_NOT_EQUATE_PCA_WITH_FACTOR_ANALYSIS
    - DO_NOT_INFER_CAUSATION_FROM_COMPONENT_LOADINGS
    - ACCOUNT_FOR_EIGENVECTOR_SIGN_INDETERMINACY
    - REPORT_EXPLAINED_VARIANCE_FOR_REDUCED_MODELS

FLOW_HASH:
  canonical_entity: "principal-component-analysis"
  namespace: "probability-statistics/multivariate-statistics"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - COVARIANCE_EIGENSTRUCTURE
    - PRINCIPAL_COMPONENTS
    - EXPLAINED_VARIANCE

END_FLOW