Missing Data Ontology

FLOW ALGEBRA

Ontology: Missing Data Ontology
Domain: Metadata Ontologies > Data Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/data-metadata/missing-data-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "MissingDataOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: DataMetadata
  subject: MissingData
  purpose:
    - represent absent, unavailable, unobserved, or unknown data values
    - distinguish missingness from null, zero, false, empty, and invalid values
    - describe causes, mechanisms, scope, and treatment of missing data
    - support validation, analysis, imputation, quality assessment, and provenance
    - enable machine-readable reasoning about incomplete datasets

# ---------- CORE ENTITY ----------
entity MissingData:
  type: DataQualityMetadata
  describes: MissingValueState
  may_include:
    - missing_value_identifier
    - missingness_type
    - missingness_reason
    - missingness_mechanism
    - affected_field
    - affected_record
    - affected_dataset
    - expected_value_type
    - missing_since
    - detection_method
    - quality_flag
    - imputation_status
    - remediation
    - provenance

entity MissingValueState:
  type: DataState
  indicates: ExpectedValueUnavailable

entity MissingnessReason:
  type: CausalMetadata
  examples:
    - not_collected
    - not_observed
    - unavailable
    - lost
    - withheld
    - not_applicable
    - sensor_failure
    - processing_failure
    - unknown_reason

entity MissingnessMechanism:
  type: StatisticalMetadata
  may_be:
    - MCAR
    - MAR
    - MNAR
    - UnknownMechanism

# ---------- RELATIONSHIPS ----------
relationships:
  MissingData -> DESCRIBES -> MissingValueState
  MissingValueState -> AFFECTS -> DataField
  MissingValueState -> MAY_AFFECT -> DataRecord
  MissingValueState -> MAY_AFFECT -> Dataset
  MissingValueState -> HAS_REASON -> MissingnessReason
  MissingValueState -> MAY_HAVE_MECHANISM -> MissingnessMechanism
  MissingValueState -> EXPECTED_TYPE -> DataType
  MissingValueState -> MAY_HAVE -> QualityFlag
  MissingValueState -> MAY_BE_TREATED_BY -> ImputationMethod
  MissingData -> MAY_REFERENCE -> Provenance

# ---------- FUNDAMENTAL DISTINCTIONS ----------
rule missing_not_zero:
  MissingData != ZeroValue

rule missing_not_false:
  MissingData != FalseValue

rule missing_not_empty:
  MissingData != EmptyString

rule missing_not_null:
  MissingData != NullValue
  unless source semantics explicitly use null as missing-data marker

rule missing_not_invalid:
  MissingData != InvalidValue

rule missing_not_not_applicable:
  MissingData != NotApplicable
  unless domain explicitly classifies not-applicable as a missingness state

# ---------- REPRESENTATION ----------
entity MissingValueMarker:
  type: RepresentationMetadata
  examples:
    - null
    - NA
    - N/A
    - NaN
    - blank
    - sentinel_value
    - omitted_field

rule marker_semantics:
  MissingValueMarker MUST be interpreted using dataset or field metadata

rule sentinel:
  sentinel value MUST NOT be treated as genuine measurement
  when explicitly defined as missing marker

rule blank:
  blank textual field MAY represent:
    - missing
    - empty_string
    - whitespace
  and MUST NOT be resolved without context

# ---------- EXPECTEDNESS ----------
rule expected_value:
  value is missing only when value is expected or semantically applicable

rule optional_field:
  absent optional field != necessarily MissingData

rule not_applicable:
  IF field does_not_apply to entity
  THEN classify as NotApplicable
  rather_than MissingData
  unless domain convention states otherwise

# ---------- MISSINGNESS MECHANISMS ----------
rule MCAR:
  MissingCompletelyAtRandom means probability_of_missingness
  is independent_of observed and unobserved data
  under declared statistical model

rule MAR:
  MissingAtRandom means missingness MAY depend on observed data
  but not on missing value itself
  under declared statistical model

rule MNAR:
  MissingNotAtRandom means missingness MAY depend on unobserved value
  or unobserved factors

rule mechanism_evidence:
  MissingnessMechanism MUST NOT be asserted with certainty
  without supporting statistical or domain evidence

# ---------- DETECTION ----------
operation detect_missing(dataset):
  inspect:
    - schema_required_fields
    - missing_value_markers
    - nullability
    - expected_record_structure
    - temporal_sequence
    - spatial_sequence
    - domain_constraints

  output:
    - missing_value_states
    - affected_scope
    - reason_if_known
    - confidence

rule required_field_missing:
  IF required DataField absent
  THEN infer MissingValueState
  WITH confidence = 1.00

rule sequence_gap:
  IF regular sequence expects value at position T
  AND no observation exists
  THEN infer MissingData candidate
  WITH confidence dependent_on sequence certainty

# ---------- SCOPE ----------
entity MissingDataScope:
  type: MetadataScope
  may_be:
    - value
    - field
    - record
    - dataset
    - temporal_interval
    - spatial_region

rule scope:
  MissingData assertion SHOULD identify affected scope

rule aggregation:
  dataset missingness rate SHOULD be derived from explicitly defined expected population

# ---------- QUALITY ----------
rule completeness:
  MissingData MAY reduce DataCompleteness

rule completeness_rate:
  IF expected_count > 0
  THEN completeness = observed_count / expected_count

rule missing_rate:
  IF expected_count > 0
  THEN missing_rate = missing_count / expected_count

rule completeness_relation:
  under same population definition:
    completeness + missing_rate = 1
  when all expected values are classified as observed or missing

# ---------- IMPUTATION ----------
entity ImputationMethod:
  type: DataProcessingMethod
  examples:
    - mean_imputation
    - median_imputation
    - mode_imputation
    - interpolation
    - forward_fill
    - backward_fill
    - regression_imputation
    - multiple_imputation
    - model_based_imputation

entity ImputedValue:
  type: DerivedDataValue
  derived_from: MissingValueState

rule imputation_not_observation:
  ImputedValue != ObservedValue

rule imputation_trace:
  ImputedValue MUST preserve:
    - original_missing_state
    - imputation_method
    - parameters
    - source_data
    - timestamp
    - provenance

rule uncertainty:
  imputed values SHOULD preserve or estimate uncertainty
  where analytically relevant

# ---------- DELETION ----------
entity MissingDataDeletion:
  type: DataProcessingMethod
  examples:
    - listwise_deletion
    - pairwise_deletion
    - field_exclusion

rule deletion_trace:
  deletion due to missing data SHOULD be recorded
  where it affects analysis or reproducibility

# ---------- TEMPORAL MISSINGNESS ----------
rule temporal_gap:
  IF TimeSeries has regular expected sampling interval
  AND expected observation absent
  THEN classify temporal missing interval

rule missing_interval_not_zero:
  missing temporal observation MUST NOT be replaced with zero
  unless domain model explicitly defines zero

# ---------- PROVENANCE ----------
rule provenance:
  MissingData SHOULD preserve available:
    - source
    - detection_method
    - reason
    - processing_history
    - imputation_history
    - remediation_history

rule source_missingness:
  missingness inherited from source dataset SHOULD remain distinguishable
  from missingness introduced during transformation

# ---------- VALIDATION ----------
validation:
  require:
    - affected_data_context

  validate_if_present:
    - missing_value_marker
    - reason
    - mechanism
    - scope
    - expected_type
    - imputation_status
    - provenance

  reject:
    - zero_classified_as_missing_without explicit semantics
    - false_classified_as_missing_without explicit semantics
    - empty_string_classified_as_missing_without context
    - observed_value_and_missing_state simultaneously asserted without version/context
    - imputed_value_presented_as directly_observed
    - unsupported_missingness_mechanism_claim

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "missing data" ~= "missing values" WHEN referring to absent expected values
  "missingness" ~= "missing data state"
  "NA" ~= "missing value marker" ONLY WHEN dataset semantics define NA as missing
  "data gap" ~= "missing data" WHEN expected data is absent
  "imputed value" ~= "estimated replacement value" WHEN produced by imputation

  PRESERVE distinctions:
    missing != null
    missing != zero
    missing != false
    missing != empty_string
    missing != invalid
    missing != not_applicable
    imputed != observed
    missing_marker != missing_semantics
    source_missingness != transformation_missingness

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_missing_metadata:
    score: 1.00

  required_field_absent:
    score: 1.00

  explicit_missing_marker:
    score: 0.99

  regular_sequence_gap:
    score: 0.95

  inferred_missing_from_blank:
    score: <= 0.70

  inferred_missingness_mechanism:
    score: <= 0.70

  ambiguous_missing_state:
    score: <= 0.60
    action: preserve candidate interpretations

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_missing_detection:
    - identify affected scope
    - preserve original representation
    - classify reason where known
    - assign confidence
    - update completeness metadata
    - record provenance

  on_value_recovery:
    - preserve previous MissingValueState
    - insert recovered value
    - distinguish recovered_from imputed
    - update quality metrics
    - record provenance

  on_imputation:
    - preserve MissingValueState
    - create ImputedValue
    - record method and parameters
    - update analysis-ready representation
    - preserve uncertainty and provenance

  on_marker_definition_change:
    - preserve previous marker semantics
    - re-evaluate affected values
    - update missingness classifications
    - record provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - missing_data_entities
    - relationships
    - distinction_rules
    - mechanism_rules
    - detection_rules
    - imputation_rules
    - validation
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - data metadata ontology
    - missing-data reasoning model
    - data completeness structure
    - imputation provenance model

  operations:
    - identify missing values
    - interpret missing markers
    - distinguish missing from null, zero, false, and empty
    - classify missingness reasons
    - evaluate missingness mechanisms
    - calculate missingness and completeness
    - detect temporal gaps
    - trace imputation
    - distinguish imputed and observed values
    - preserve provenance
    - expose uncertainty

  constraints:
    - do_not_treat_zero_as_missing_without evidence
    - do_not_treat_false_as_missing_without evidence
    - do_not_treat_blank_as_missing_without context
    - do_not_present_imputed_values_as observed
    - do_not_assert_MCAR_MAR_MNAR_without support
    - preserve_original_missing_state_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - dataset
    - field
    - record
    - missing_value_marker
    - missingness_reason
    - missingness_mechanism
    - temporal_constraint
    - imputation_status

  resolve:
    1. identify expected data structure
    2. inspect missing-value semantics
    3. detect MissingValueStates
    4. resolve affected scope
    5. resolve reason where available
    6. evaluate mechanism evidence
    7. inspect quality impact
    8. inspect imputation or remediation
    9. trace provenance
    10. return confidence

  output:
    - missing_data_state
    - affected_scope
    - missingness_reason
    - missingness_mechanism
    - completeness_impact
    - imputation_state
    - provenance
    - confidence

# ---------- INFERENCE ----------
inference:
  IF required field has no value
  THEN infer MissingData
  WITH confidence = 1.00

  IF explicit marker is defined as missing
  AND field contains marker
  THEN infer MissingData
  WITH confidence = 0.99

  IF regular TimeSeries lacks expected observation
  THEN infer temporal MissingData
  WITH confidence <= 0.95

  IF missing value is replaced using ImputationMethod
  THEN classify replacement as ImputedValue
  NOT ObservedValue

  IF missingness semantics remain ambiguous
  THEN preserve alternative interpretations
  AND reduce confidence

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: MissingData
  missingness_structure: machine_interpretable
  missing_markers: context_aware
  missingness_reason: preserved_where_known
  mechanism: evidence_qualified
  quality_impact: reportable
  imputation_history: traceable
  observed_imputed_distinction: preserved
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow