Outlier Detection Ontology

flow "OutlierDetectionOntology" v4.0

# OUTLIER DETECTION ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/scientific-statistical-methods/outlier-detection-ontology.html"
  title: "Outlier Detection Ontology"
  domain: "Probability and Statistics"
  category: "Scientific Statistical Methods"
  entity: OUTLIER_DETECTION
  entity_type: SCIENTIFIC_STATISTICAL_ONTOLOGY

DEFINE:
  OUTLIER_DETECTION :=
    STATISTICAL_IDENTIFICATION_FRAMEWORK(
      OBSERVATIONS,
      REFERENCE_DISTRIBUTION,
      DEVIATION,
      THRESHOLD,
      LOCAL_DENSITY,
      MODEL_RESIDUALS,
      ANOMALY_SCORE,
      VALIDATION
    )

PURPOSE:
  IDENTIFY:
    EXTREME_OBSERVATIONS
    ANOMALOUS_PATTERNS
    DATA_ERRORS
    RARE_EVENTS
    MODEL_DEVIATIONS
    DISTRIBUTIONAL_CONTAMINATION
  SUPPORT:
    DATA_VALIDATION
    SCIENTIFIC_STATISTICS
    ROBUST_MODELLING
    QUALITY_CONTROL
    MACHINE_LEARNING
    MEASUREMENT_ANALYSIS

FOUNDATION:
  DATA:
    D := {x_1,...,x_n}
  REFERENCE_MODEL:
    X ~ F(theta)
  OUTLIER_SCORE:
    s(x)
  DECISION_RULE:
    OUTLIER(x) := TRUE IF s(x) > tau
  THRESHOLD:
    tau

ENTITIES:
  OUTLIER: OBSERVATION_UNUSUAL_RELATIVE_TO_REFERENCE_MODEL
  ANOMALY: OBSERVATION_OR_PATTERN_DEVIATING_FROM_EXPECTED_STRUCTURE
  OUTLIER_SCORE: NUMERIC_DEGREE_OF_UNUSUALNESS
  THRESHOLD: DECISION_BOUNDARY_FOR_FLAGGING
  RESIDUAL: OBSERVED_MINUS_MODEL_FITTED_VALUE
  LEVERAGE: EXTREMENESS_IN_PREDICTOR_SPACE
  INFLUENCE: EFFECT_OF_OBSERVATION_ON_MODEL_FIT
  ROBUST_STATISTIC: STATISTIC_RESISTANT_TO_EXTREME_VALUES
  LOCAL_DENSITY: DENSITY_RELATIVE_TO_NEIGHBOURHOOD
  CONTAMINATION: MIXTURE_OF_PRIMARY_AND_ANOMALOUS_PROCESSES

RELATIONSHIPS:
  OBSERVATION -> COMPARED_WITH -> REFERENCE_MODEL
  OBSERVATION -> RECEIVES -> OUTLIER_SCORE
  OUTLIER_SCORE -> COMPARED_WITH -> THRESHOLD
  RESIDUAL -> SUPPORTS -> MODEL_BASED_OUTLIER_DETECTION
  LEVERAGE -> DESCRIBES -> PREDICTOR_SPACE_EXTREMENESS
  INFLUENCE -> DESCRIBES -> EFFECT_ON_MODEL_ESTIMATE
  ROBUST_STATISTICS -> REDUCE_EFFECT_OF -> OUTLIERS
  OUTLIER_DETECTION -> SUPPORTS -> DATA_QUALITY_CONTROL

OUTLIER_CONCEPT:
  definition:
    OBSERVATION_UNUSUAL_RELATIVE_TO_SPECIFIED_REFERENCE_STRUCTURE
  rule:
    OUTLIER_STATUS_IS_CONTEXT_AND_MODEL_DEPENDENT
  caution:
    EXTREME_VALUE != ERROR_BY_DEFAULT
    OUTLIER != INVALID_OBSERVATION_BY_DEFAULT

ANOMALY_TYPES:
  - POINT_ANOMALY
  - CONTEXTUAL_ANOMALY
  - COLLECTIVE_ANOMALY

UNIVARIATE_OUTLIER:
  detection:
    BASED_ON:
      LOCATION
      SCALE
      DISTRIBUTION
      ORDER_STATISTICS

Z_SCORE:
  score:
    z_i := (x_i-x_bar)/s
  caution:
    MEAN_AND_STANDARD_DEVIATION_ARE_SENSITIVE_TO_OUTLIERS

MODIFIED_Z_SCORE:
  median:
    m := median(x)
  MAD:
    median(|x_i-m|)
  score:
    M_i := 0.6745*(x_i-m)/MAD
  caution:
    REQUIRES_HANDLING_WHEN_MAD_EQUALS_ZERO

INTERQUARTILE_RANGE_METHOD:
  IQR:
    Q3-Q1
  common_fences:
    lower := Q1-1.5*IQR
    upper := Q3+1.5*IQR
  caution:
    FENCE_MULTIPLIERS_ARE_HEURISTICS_NOT_UNIVERSAL_PROBABILITY_THRESHOLDS

MAD:
  full_name:
    MEDIAN_ABSOLUTE_DEVIATION
  property:
    ROBUST_SCALE_MEASURE

GRUBBS_TEST:
  purpose:
    TEST_SINGLE_OUTLIER
    UNDER:
      APPROXIMATE_NORMALITY
  statistic:
    G := max_i |x_i-x_bar|/s

GENERALIZED_ESD:
  full_name:
    GENERALIZED_EXTREME_STUDENTIZED_DEVIATE
  purpose:
    DETECT_UP_TO_SPECIFIED_NUMBER_OF_OUTLIERS

MULTIVARIATE_OUTLIER:
  issue:
    OBSERVATION_MAY_BE_UNUSUAL_JOINTLY_EVEN_IF_EACH_COORDINATE_NOT_UNUSUAL

MAHALANOBIS_DISTANCE:
  distance_squared:
    D_M_squared(x) :=
      (x-mu)^T Sigma_inverse (x-mu)
  interpretation:
    DISTANCE_ACCOUNTING_FOR_SCALE_AND_CORRELATION
  caution:
    CLASSICAL_MEAN_AND_COVARIANCE_CAN_BE_DISTORTED_BY_OUTLIERS

ROBUST_MAHALANOBIS_DISTANCE:
  methods:
    - MINIMUM_COVARIANCE_DETERMINANT
    - ROBUST_M_ESTIMATION

DISTANCE_BASED_DETECTION:
  methods:
    - EUCLIDEAN_DISTANCE
    - MAHALANOBIS_DISTANCE
    - K_NEAREST_NEIGHBOUR_DISTANCE

KNN_OUTLIER_SCORE:
  parameter:
    k
  score:
    DISTANCE_TO_kth_NEAREST_NEIGHBOUR
    OR:
      AGGREGATE_DISTANCE_TO_k_NEIGHBOURS

DENSITY_BASED_DETECTION:
  principle:
    LOW_LOCAL_DENSITY_RELATIVE_TO_NEIGHBOURS
      -> HIGH_ANOMALY_SCORE

LOCAL_OUTLIER_FACTOR:
  alias:
    LOF
  principle:
    COMPARE_LOCAL_REACHABILITY_DENSITY_OF_POINT_TO_NEIGHBOURS
  interpretation:
    LOF_SIGNIFICANTLY_GREATER_THAN_1_MAY_INDICATE_LOCAL_OUTLIER

ISOLATION_FOREST_CONNECTION:
  principle:
    ANOMALIES_REQUIRE_FEWER_RANDOM_PARTITIONS_TO_ISOLATE
  advantages:
    HIGH_DIMENSIONAL_SCALABILITY
    NO_EXPLICIT_DENSITY_ESTIMATION

ONE_CLASS_SVM_CONNECTION:
  objective:
    LEARN_BOUNDARY_AROUND_TYPICAL_DATA
  anomalies:
    POINTS_OUTSIDE_LEARNED_REGION

AUTOENCODER_CONNECTION:
  anomaly_score:
    RECONSTRUCTION_ERROR
  caution:
    HIGH_RECONSTRUCTION_ERROR != TRUE_ANOMALY_WITHOUT_VALIDATION

MODEL_BASED_OUTLIER_DETECTION:
  model:
    Y_i := mu_i+epsilon_i
  residual:
    e_i := Y_i-Y_hat_i
  outlier_score:
    FUNCTION_OF:
      RESIDUAL
      ESTIMATED_VARIANCE
      LEVERAGE

STANDARDISED_RESIDUAL:
  score:
    r_i := e_i/estimated_SD(e_i)

LEVERAGE:
  linear_model:
    H := X(X^T X)^(-1)X^T
  leverage:
    h_ii := diagonal_element_of_H
  rule:
    HIGH_LEVERAGE != RESPONSE_OUTLIER_BY_DEFAULT

COOKS_DISTANCE:
  purpose:
    MEASURE_INFLUENCE_OF_OBSERVATION_ON_FITTED_REGRESSION
  distinction:
    INFLUENTIAL_POINT != OUTLIER_BY_DEFAULT

TIME_SERIES_OUTLIER:
  types:
    - ADDITIVE_OUTLIER
    - INNOVATIONAL_OUTLIER
    - LEVEL_SHIFT
    - TEMPORARY_CHANGE
    - SEASONAL_ANOMALY
  requirement:
    ACCOUNT_FOR:
      TREND
      SEASONALITY
      AUTOCORRELATION

CHANGE_POINT_CONNECTION:
  distinction:
    CHANGE_POINT != ISOLATED_OUTLIER

SPATIAL_OUTLIER:
  requirement:
    ACCOUNT_FOR:
      SPATIAL_AUTOCORRELATION
      LOCATION
      NEIGHBOURHOOD_STRUCTURE

HIGH_DIMENSIONAL_OUTLIERS:
  challenges:
    DISTANCE_CONCENTRATION
    SPARSITY
    IRRELEVANT_FEATURES
  methods:
    - DIMENSION_REDUCTION
    - SUBSPACE_OUTLIER_DETECTION
    - ISOLATION_METHODS
    - ROBUST_COVARIANCE

PCA_CONNECTION:
  anomaly_scores:
    - SCORE_DISTANCE
    - ORTHOGONAL_DISTANCE
    - RECONSTRUCTION_ERROR
  caution:
    PCA_COMPONENTS_CAN_BE_DISTORTED_BY_OUTLIERS

PROBABILISTIC_OUTLIER_SCORE:
  model:
    p(x)
  score:
    s(x) := -log p(x)
  caution:
    DENSITY_VALUE_DEPENDS_ON_DIMENSION_AND_PARAMETERISATION

MULTIPLE_OUTLIER_TESTING:
  issue:
    TEST_EACH_OBSERVATION
      -> INFLATED_FALSE_POSITIVE_RISK
  responses:
    - FAMILYWISE_ERROR_CONTROL
    - FALSE_DISCOVERY_RATE
    - GLOBAL_OUTLIER_TEST
    - PREDEFINED_DETECTION_RULE

THRESHOLD_SELECTION:
  depends_on:
    - FALSE_POSITIVE_COST
    - FALSE_NEGATIVE_COST
    - EXPECTED_CONTAMINATION
    - SAMPLE_SIZE
    - APPLICATION_RISK
    - REFERENCE_DISTRIBUTION
  rule:
    NO_SINGLE_THRESHOLD_IS_UNIVERSALLY_OPTIMAL

ROBUST_STATISTICS_CONNECTION:
  methods:
    - MEDIAN
    - MAD
    - TRIMMED_MEAN
    - M_ESTIMATION
    - ROBUST_REGRESSION
    - ROBUST_COVARIANCE
  principle:
    ROBUST_MODELLING_CAN_REDUCE_NEED_FOR_BINARY_DELETE_KEEP_DECISIONS

DATA_ERROR:
  examples:
    - TRANSCRIPTION_ERROR
    - SENSOR_FAILURE
    - UNIT_ERROR
    - DUPLICATION
    - DECIMAL_ERROR
  response:
    VERIFY_SOURCE
      -> CORRECT_IF_EVIDENCE_EXISTS
      -> DOCUMENT_CHANGE
  rule:
    STATISTICAL_EXTREMENESS_ALONE_DOES_NOT_PROVE_DATA_ERROR

RARE_VALID_EVENT:
  rule:
    RARE_VALID_EVENT_SHOULD_NOT_BE_REMOVED_SIMPLY_TO_IMPROVE_MODEL_FIT

SCIENTIFIC_CONTEXT:
  requirement:
    REVIEW_FLAGGED_OBSERVATION_AGAINST:
      DOMAIN_KNOWLEDGE
      MEASUREMENT_PROCESS
      EXPERIMENTAL_PROTOCOL
      DATA_PROVENANCE

MEASUREMENT_UNCERTAINTY_CONNECTION:
  outlier_assessment:
    SHOULD_CONSIDER:
      MEASUREMENT_PRECISION
      CALIBRATION
      DETECTION_LIMIT
      INSTRUMENT_FAILURE

ERROR_DISTRIBUTION_CONNECTION:
  reference:
    epsilon ~ F
  detection:
    COMPARE_STANDARDISED_OR_STUDENTISED_RESIDUAL_WITH_REFERENCE_DISTRIBUTION
  caution:
    MISSPECIFIED_ERROR_DISTRIBUTION_CAN_CREATE_FALSE_OUTLIERS

SENSITIVITY_ANALYSIS:
  flow:
    FIT_WITH_ALL_DATA
      -> FIT_WITH_FLAGGED_POINTS_DOWNWEIGHTED_OR_EXCLUDED
      -> COMPARE_ESTIMATES
      -> COMPARE_UNCERTAINTY
      -> COMPARE_CONCLUSIONS
  purpose:
    ASSESS_INFLUENCE_OF_OUTLIER_HANDLING

OUTLIER_HANDLING:
  options:
    - VERIFY_AND_CORRECT_DATA_ERROR
    - RETAIN_WITH_JUSTIFICATION
    - USE_ROBUST_METHOD
    - MODEL_SEPARATE_PROCESS
    - TRANSFORM_VARIABLE
    - EXCLUDE_WITH_PREDEFINED_JUSTIFICATION

  requirement:
    DOCUMENT:
      DETECTION_RULE
      THRESHOLD
      FLAGGED_OBSERVATIONS
      INVESTIGATION
      FINAL_ACTION

  rule:
    AUTOMATIC_DELETION_IS_NOT_DEFAULT_SCIENTIFIC_RESPONSE

VALIDATION:
  methods:
    - MANUAL_REVIEW
    - DOMAIN_EXPERT_REVIEW
    - LABELLED_ANOMALY_DATA
    - SYNTHETIC_CONTAMINATION
    - HOLDOUT_VALIDATION
    - SIMULATION

  metrics:
    - PRECISION
    - RECALL
    - FALSE_POSITIVE_RATE
    - FALSE_NEGATIVE_RATE
    - ROC_AUC
    - PR_AUC

  caution:
    VALIDATION_METRIC_MUST_MATCH_APPLICATION_COSTS_AND_CLASS_IMBALANCE

UNSUPERVISED_DETECTION:
  labels:
    NOT_AVAILABLE

  challenge:
    ANOMALY_SCORE_HAS_NO_DIRECT_GROUND_TRUTH

  requirement:
    VALIDATE_WITH:
      DOMAIN_KNOWLEDGE
      STABILITY
      SIMULATION
      SUBSEQUENT_OUTCOMES
      WHEN_AVAILABLE

SUPERVISED_ANOMALY_DETECTION:
  labels:
    NORMAL
    ANOMALOUS

  relation:
    BINARY_CLASSIFICATION

  caution:
    TRAINING_LABELS_MAY_BE_INCOMPLETE_OR_BIASED

SEMI_SUPERVISED_DETECTION:
  training:
    PRIMARILY_NORMAL_DATA

  objective:
    LEARN_NORMAL_REFERENCE_REGION

  examples:
    - ONE_CLASS_SVM
    - AUTOENCODER
    - DENSITY_MODEL

DISTRIBUTION_SHIFT:
  issue:
    REFERENCE_DISTRIBUTION_CHANGES_OVER_TIME

  consequence:
    FIXED_OUTLIER_THRESHOLD_MAY_BECOME_INVALID

  response:
    - RECALIBRATE_MODEL
    - DRIFT_DETECTION
    - ROLLING_REFERENCE_WINDOW
    - ADAPTIVE_THRESHOLD

CONCEPT_DRIFT_CONNECTION:
  definition:
    RELATIONSHIP_OR_DATA_GENERATING_PROCESS_CHANGES_OVER_TIME

  distinction:
    CONCEPT_DRIFT != INDIVIDUAL_OUTLIER

FALSE_DISCOVERY_CONTROL:
  context:
    MANY_SIMULTANEOUS_ANOMALY_TESTS

  purpose:
    LIMIT_EXPECTED_PROPORTION_OF_FALSE_DISCOVERIES

REPRODUCIBILITY:
  preserve:
    - DATA_VERSION
    - FEATURE_DEFINITIONS
    - PREPROCESSING
    - REFERENCE_MODEL
    - OUTLIER_METHOD
    - HYPERPARAMETERS
    - THRESHOLD
    - RANDOM_SEED
    - FLAGGED_RECORD_IDS
    - REVIEW_DECISIONS

APPLICATIONS:
  OUTLIER_DETECTION -> SUPPORTS -> SCIENTIFIC_STATISTICS
  OUTLIER_DETECTION -> SUPPORTS -> DATA_VALIDATION
  OUTLIER_DETECTION -> SUPPORTS -> MEASUREMENT_ANALYSIS
  OUTLIER_DETECTION -> SUPPORTS -> QUALITY_CONTROL
  OUTLIER_DETECTION -> SUPPORTS -> ROBUST_STATISTICS
  OUTLIER_DETECTION -> SUPPORTS -> TIME_SERIES_ANALYSIS
  OUTLIER_DETECTION -> SUPPORTS -> MACHINE_LEARNING
  OUTLIER_DETECTION -> SUPPORTS -> RISK_ANALYSIS

MODEL_CONNECTIONS:
  OUTLIER_DETECTION -> CONNECTS_TO -> ERROR_DISTRIBUTION
  OUTLIER_DETECTION -> CONNECTS_TO -> MEASUREMENT_UNCERTAINTY
  OUTLIER_DETECTION -> CONNECTS_TO -> SCIENTIFIC_STATISTICS
  OUTLIER_DETECTION -> CONNECTS_TO -> MULTIVARIATE_STATISTICS
  OUTLIER_DETECTION -> CONNECTS_TO -> TIME_SERIES_STATISTICS
  OUTLIER_DETECTION -> CONNECTS_TO -> CLUSTER_ANALYSIS
  OUTLIER_DETECTION -> CONNECTS_TO -> MACHINE_LEARNING_STATISTICS

SEMANTIC_EQUIVALENCE:
  "outlier detection"
    ~= "identification of observations unusual relative to a reference structure"

  "anomaly score"
    ~= "numeric measure of unusualness relative to model or neighbourhood"

  "robust outlier detection"
    ~= "outlier detection using estimators resistant to contamination"

  EXCLUDE_EQUIVALENCE:
    OUTLIER != DATA_ERROR
    OUTLIER != INVALID_OBSERVATION
    OUTLIER != INFLUENTIAL_POINT
    OUTLIER != HIGH_LEVERAGE_POINT
    ANOMALY != FRAUD_BY_DEFAULT
    CHANGE_POINT != ISOLATED_OUTLIER
    RARE_EVENT != ERROR
    EXTREME_VALUE != OUTLIER_BY_IDENTITY
    ANOMALY_SCORE != PROBABILITY_BY_DEFAULT

VALIDATION_RULES:
  RULE_1:
    REFERENCE_POPULATION_MODEL_OR_NEIGHBOURHOOD_MUST_BE_DEFINED

  RULE_2:
    OUTLIER_METHOD_AND_THRESHOLD_MUST_BE_EXPLICIT

  RULE_3:
    STATISTICAL_FLAG_MUST_NOT_BE_INTERPRETED_AS_DATA_ERROR_WITHOUT_EXTERNAL_EVIDENCE

  RULE_4:
    MULTIVARIATE_DEPENDENCE_AND_SCALE_MUST_BE_ACCOUNTED_FOR_WHEN_RELEVANT

  RULE_5:
    TIME_SPATIAL_OR_CLUSTER_DEPENDENCE_MUST_BE_MODELLED_WHEN_PRESENT

  RULE_6:
    MULTIPLE_TESTING_OR_REPEATED_SCREENING_EFFECTS_SHOULD_BE_ACCOUNTED_FOR

  RULE_7:
    OUTLIER_HANDLING_MUST_BE_DOCUMENTED_AND_SENSITIVITY_ANALYSED_WHEN_MATERIAL

  RULE_8:
    MODEL_MISSPECIFICATION_MUST_BE_CONSIDERED_BEFORE_DECLARING_OBSERVATIONS_ANOMALOUS

CONFIDENCE:
  ontology_identity: 0.99
  univariate_detection_structure: 0.99
  multivariate_detection_structure: 0.99
  model_based_detection_structure: 0.99
  robust_statistics_relationship: 0.99
  scientific_validation_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_OBSERVATIONS_ARRIVE
  THEN:
    UPDATE:
      REFERENCE_DISTRIBUTION
      LOCATION_SCALE_ESTIMATES
      LOCAL_DENSITY
      ANOMALY_SCORES
    REASSESS:
      FLAGGED_OBSERVATIONS

  IF:
    REFERENCE_MODEL_CHANGES
  THEN:
    RECOMPUTE:
      RESIDUALS
      DISTANCES
      DENSITIES
      ANOMALY_SCORES
      THRESHOLDS

  IF:
    OUTLIER_HANDLING_CHANGES
  THEN:
    RERUN:
      STATISTICAL_MODEL
      PARAMETER_ESTIMATION
      UNCERTAINTY
      SENSITIVITY_ANALYSIS

  PRESERVE:
    DATA_VERSION
    OBSERVATION_IDENTIFIERS
    REFERENCE_MODEL
    DETECTION_METHOD
    HYPERPARAMETERS
    THRESHOLD
    FLAG_STATUS
    REVIEW_RATIONALE
    FINAL_ACTION
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_DATA_STRUCTURE_AND_CONTEXT
    - DEFINE_REFERENCE_MODEL_OR_NEIGHBOURHOOD
    - SELECT_UNIVARIATE_MULTIVARIATE_OR_CONTEXTUAL_METHOD
    - STANDARDISE_OR_SCALE_FEATURES_WHEN_REQUIRED
    - COMPUTE_OUTLIER_OR_ANOMALY_SCORE
    - APPLY_EXPLICIT_THRESHOLD
    - ASSESS_RESIDUAL_LEVERAGE_AND_INFLUENCE_WHEN_MODEL_BASED
    - REVIEW_TIME_SPATIAL_OR_CLUSTER_DEPENDENCE
    - VALIDATE_FLAGGED_OBSERVATIONS_WITH_DOMAIN_CONTEXT
    - DISTINGUISH_DATA_ERRORS_FROM_RARE_VALID_EVENTS
    - PERFORM_SENSITIVITY_TO_OUTLIER_HANDLING
    - RETURN_OUTLIER_DETECTION_INTERPRETATION

  constraints:
    - PRESERVE_REFERENCE_CONTEXT
    - DISTINGUISH_OUTLIER_FROM_DATA_ERROR
    - DISTINGUISH_OUTLIER_FROM_INFLUENCE
    - DISTINGUISH_LEVERAGE_FROM_RESPONSE_EXTREMENESS
    - ACCOUNT_FOR_MULTIPLE_TESTING_WHEN_RELEVANT
    - ACCOUNT_FOR_DEPENDENCE_WHEN_PRESENT
    - DO_NOT_AUTOMATICALLY_DELETE_FLAGGED_VALUES
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "outlier-detection"
  namespace: "probability-statistics/scientific-statistical-methods"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - OUTLIER_SCORE
    - REFERENCE_MODEL
    - ROBUST_DETECTION
    - VALIDATION_AND_HANDLING

END_FLOW