Data Quality Ontology

FLOW ALGEBRA

Ontology: Data Quality Ontology
Domain: Metadata Ontologies > Data Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/data-metadata/data-quality-ontology-2.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "DataQualityOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: DataMetadata
  subject: DataQuality
  purpose:
    - represent quality characteristics of data resources
    - define measurable dimensions, metrics, rules, and quality states
    - support validation, comparison, monitoring, and remediation
    - preserve quality evidence, scope, method, and provenance
    - enable machine-readable reasoning about fitness for data use

# ---------- CORE ENTITY ----------
entity DataQuality:
  type: QualityMetadata
  describes: DataResource
  may_include:
    - quality_identifier
    - quality_dimension
    - quality_metric
    - quality_score
    - quality_rule
    - quality_issue
    - quality_flag
    - assessment_method
    - assessment_time
    - scope
    - threshold
    - evidence
    - remediation
    - provenance

entity QualityDimension:
  type: QualityCharacteristic
  examples:
    - completeness
    - validity
    - consistency
    - accuracy
    - uniqueness
    - timeliness
    - integrity
    - conformity
    - precision
    - accessibility

entity QualityMetric:
  type: MeasurementDefinition
  measures: QualityDimension

entity QualityScore:
  type: QuantitativeQualityValue
  may_include:
    - value
    - scale
    - unit
    - threshold
    - confidence

entity QualityRule:
  type: ValidationRule
  evaluates: DataResource

entity QualityIssue:
  type: DataQualityFinding
  indicates: QualityProblem

# ---------- RELATIONSHIPS ----------
relationships:
  DataQuality -> DESCRIBES -> DataResource
  DataQuality -> HAS_DIMENSION -> QualityDimension
  QualityDimension -> MEASURED_BY -> QualityMetric
  QualityMetric -> PRODUCES -> QualityScore
  DataResource -> EVALUATED_BY -> QualityRule
  QualityRule -> MAY_DETECT -> QualityIssue
  QualityIssue -> MAY_HAVE -> QualityFlag
  QualityIssue -> MAY_REQUIRE -> RemediationAction
  DataQuality -> HAS_SCOPE -> QualityScope
  DataQuality -> MAY_REFERENCE -> Evidence
  DataQuality -> MAY_REFERENCE -> Provenance

# ---------- QUALITY SCOPE ----------
entity QualityScope:
  type: MetadataScope
  may_be:
    - dataset
    - table
    - record
    - field
    - value
    - temporal_window
    - spatial_region

rule scope:
  every quality assertion SHOULD identify applicable QualityScope

rule scope_inheritance:
  dataset-level quality MUST NOT automatically be assigned to every record or value
  unless assessment semantics justify inheritance

# ---------- COMPLETENESS ----------
rule completeness:
  completeness measures presence of expected data

operation completeness_score(observed, expected):
  require:
    - expected >= 0
    - observed >= 0
  IF expected > 0
  THEN score = observed / expected

rule missing_not_invalid:
  missing data != invalid data

rule completeness_context:
  expected population MUST be defined
  for deterministic completeness calculation

# ---------- VALIDITY ----------
rule validity:
  validity measures conformance to declared:
    - datatype
    - schema
    - value_domain
    - constraint
    - business_or_scientific_rule

rule invalid_value:
  value violating mandatory constraint SHOULD be classified invalid
  with violated rule preserved

# ---------- CONSISTENCY ----------
rule consistency:
  consistency measures absence of contradictory representations or assertions

rule cross_field_consistency:
  IF fields have dependency constraints
  THEN evaluate records against dependency rules

rule cross_dataset_consistency:
  comparison across datasets requires compatible semantics and reference context

# ---------- ACCURACY ----------
rule accuracy:
  accuracy describes closeness to accepted, reference, or true value
  where such reference is available

rule accuracy_evidence:
  accuracy claim SHOULD identify reference or assessment method

rule precision_not_accuracy:
  precision != accuracy

# ---------- UNIQUENESS ----------
rule uniqueness:
  uniqueness measures absence of unintended duplicate entities or values

rule duplicate:
  duplicate detection MUST define identity or matching criteria

rule repeated_value:
  repeated value != duplicate entity
  unless uniqueness constraint requires unique value

# ---------- TIMELINESS ----------
rule timeliness:
  timeliness evaluates whether data is sufficiently current for declared use

rule age:
  data_age = assessment_time - relevant_data_time

rule timeliness_threshold:
  IF maximum acceptable age is defined
  THEN compare data_age with threshold

# ---------- INTEGRITY ----------
rule integrity:
  integrity includes preservation of:
    - referential_relationships
    - structural_constraints
    - entity_identity
    - required_dependencies

rule referential_integrity:
  foreign or linked identifier SHOULD resolve
  where relationship requires existing target

# ---------- CONFORMITY ----------
rule conformity:
  conformity evaluates adherence to:
    - schema
    - standard
    - vocabulary
    - format
    - protocol

rule semantic_conformity:
  syntactic conformity != semantic conformity

# ---------- QUALITY METRICS ----------
rule metric_definition:
  QualityMetric SHOULD define:
    - measured_dimension
    - calculation_method
    - scale
    - direction
    - valid_range

rule metric_direction:
  metric MUST declare whether higher or lower values indicate better quality
  where not intrinsically obvious

rule metric_comparison:
  QualityScores MAY be compared only when metric semantics are compatible

# ---------- THRESHOLDS ----------
entity QualityThreshold:
  type: DecisionConstraint
  may_include:
    - minimum_acceptable
    - maximum_acceptable
    - warning_level
    - critical_level

rule threshold:
  threshold evaluation MUST preserve metric and scope

rule threshold_not_truth:
  passing quality threshold != universally high-quality data
  outside declared purpose and scope

# ---------- QUALITY FLAGS ----------
quality_flags:
  - valid
  - warning
  - suspect
  - invalid
  - incomplete
  - stale
  - duplicate
  - inconsistent
  - unresolved

rule quality_flag:
  QualityFlag SHOULD be linked to reason, rule, or metric

# ---------- EVIDENCE ----------
entity QualityEvidence:
  type: Evidence
  may_include:
    - validation_result
    - reference_data
    - audit_result
    - comparison_result
    - test_result

rule evidence:
  quality assertion SHOULD preserve supporting evidence where available

# ---------- REMEDIATION ----------
entity RemediationAction:
  type: DataProcessingAction
  examples:
    - correct_value
    - fill_missing_value
    - remove_duplicate
    - standardise_format
    - resolve_identifier
    - update_stale_data
    - flag_for_review

rule remediation_trace:
  remediation MUST preserve:
    - original_state
    - action
    - agent_or_process
    - timestamp
    - resulting_state

rule correction_not_erasure:
  corrected data SHOULD retain provenance of original value
  where auditability requires

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_data_resource
    - quality_dimension_or_rule

  validate_if_present:
    - quality_metric
    - quality_score
    - quality_scope
    - threshold
    - assessment_method
    - assessment_time
    - evidence
    - remediation
    - provenance

  reject:
    - score_outside_declared_scale
    - undefined_metric_semantics
    - contradictory_quality_state
    - quality_claim_without_applicable_scope
    - unsupported_accuracy_claim
    - incompatible_metric_comparison

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "data quality" ~= "quality of data"
  "quality metric" ~= "data quality metric"
  "quality rule" ~= "data validation rule" WHEN rule evaluates quality
  "quality issue" ~= "data quality problem"
  "quality score" ~= "data quality score"
  "data cleansing" ~= "data remediation" WHEN correcting quality issues

  PRESERVE distinctions:
    completeness != validity
    validity != accuracy
    accuracy != precision
    uniqueness != repeated_value
    timeliness != temporal_accuracy
    syntactic_conformity != semantic_conformity
    quality_score != quality_dimension
    quality_flag != quality_metric
    remediation != original_observation

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_quality_metadata:
    score: 1.00

  deterministic_validation_result:
    score: 1.00

  reference_based_accuracy_assessment:
    score: 0.95

  calculated_quality_metric:
    score: 0.95

  inferred_quality_issue:
    score: 0.80

  heuristic_duplicate_detection:
    score: <= 0.80

  ambiguous_quality_context:
    score: <= 0.60
    action: preserve uncertainty

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_data_change:
    - identify affected QualityScopes
    - rerun relevant QualityRules
    - recalculate QualityMetrics
    - update QualityFlags
    - preserve assessment history

  on_quality_rule_change:
    - preserve previous rule version
    - reassess affected DataResources
    - distinguish old and new assessment results
    - update provenance

  on_remediation:
    - preserve original data state
    - apply RemediationAction
    - revalidate affected data
    - record before_and_after quality state
    - update provenance

  on_threshold_change:
    - preserve previous threshold
    - reevaluate QualityScores
    - update flags
    - preserve assessment history

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - quality_entities
    - relationships
    - dimension_rules
    - metric_rules
    - validation_rules
    - remediation_rules
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - data metadata ontology
    - data quality assessment model
    - validation and remediation structure
    - quality monitoring model

  operations:
    - identify quality dimensions
    - evaluate quality rules
    - calculate quality metrics
    - interpret quality scores
    - apply thresholds
    - detect quality issues
    - assign quality flags
    - trace evidence
    - recommend or record remediation
    - preserve assessment history
    - expose uncertainty

  constraints:
    - do_not_equate_missing_with_invalid
    - do_not_equate_precision_with_accuracy
    - do_not_assume_duplicate_from_repeated_value_alone
    - do_not_compare_incompatible_quality_metrics
    - do_not_hide_original_values_after_remediation
    - preserve_scope_method_evidence_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - data_resource
    - quality_dimension
    - quality_metric
    - quality_rule
    - quality_threshold
    - quality_flag
    - scope
    - assessment_time

  resolve:
    1. identify DataResource
    2. retrieve DataQuality metadata
    3. resolve QualityScope
    4. resolve QualityDimension
    5. execute or inspect QualityMetric
    6. evaluate QualityRules
    7. compare against thresholds
    8. retrieve QualityIssues and flags
    9. inspect evidence and remediation
    10. return confidence and provenance

  output:
    - data_resource
    - quality_dimension
    - quality_metric
    - quality_score
    - quality_state
    - issues
    - evidence
    - remediation
    - scope
    - provenance
    - confidence

# ---------- INFERENCE ----------
inference:
  IF required value is absent
  THEN infer completeness_issue
  WITH confidence = 1.00
  when requirement is explicit

  IF value violates explicit mandatory constraint
  THEN infer validity_issue
  WITH confidence = 1.00

  IF entity violates explicit uniqueness constraint
  THEN infer uniqueness_issue
  WITH confidence = 1.00

  IF data age exceeds declared timeliness threshold
  THEN infer stale_data_issue
  WITH confidence = 0.95

  IF quality evidence is incomplete or conflicting
  THEN reduce confidence
  AND preserve unresolved quality state

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: DataQuality
  quality_structure: machine_interpretable
  dimensions: explicit
  metrics: calculable
  scope: preserved
  thresholds: evaluable
  issues: traceable
  remediation: auditable
  assessment_history: preserved
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow