Data Validation Ontology

FLOW ALGEBRA

Ontology: Data Validation Ontology
Domain: Metadata Ontologies > Data Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/data-metadata/data-validation-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "DataValidationOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: DataMetadata
  subject: DataValidation
  purpose:
    - represent processes and rules used to validate data
    - determine conformance to schemas, datatypes, domains, and constraints
    - identify validation errors, warnings, and unresolved states
    - preserve validation evidence, scope, method, and provenance
    - enable machine-readable reasoning about data validity

# ---------- CORE ENTITY ----------
entity DataValidation:
  type: DataQualityMetadata
  evaluates: DataResource
  may_include:
    - validation_identifier
    - validation_rule
    - validation_constraint
    - validation_method
    - validation_scope
    - validation_result
    - validation_status
    - severity
    - error
    - warning
    - evidence
    - assessment_time
    - validator
    - remediation
    - provenance

entity ValidationRule:
  type: ConstraintRule
  evaluates: DataResource

entity ValidationConstraint:
  type: DataConstraint
  may_be:
    - datatype_constraint
    - required_value_constraint
    - range_constraint
    - pattern_constraint
    - enumeration_constraint
    - uniqueness_constraint
    - cardinality_constraint
    - referential_constraint
    - schema_constraint
    - semantic_constraint

entity ValidationResult:
  type: AssessmentResult
  produced_by: ValidationProcess
  may_include:
    - status
    - violations
    - warnings
    - evidence
    - timestamp

# ---------- RELATIONSHIPS ----------
relationships:
  DataValidation -> EVALUATES -> DataResource
  DataValidation -> USES_RULE -> ValidationRule
  ValidationRule -> ENFORCES -> ValidationConstraint
  ValidationProcess -> APPLIES -> ValidationRule
  ValidationProcess -> PRODUCES -> ValidationResult
  ValidationResult -> MAY_CONTAIN -> ValidationError
  ValidationResult -> MAY_CONTAIN -> ValidationWarning
  ValidationResult -> HAS_SCOPE -> ValidationScope
  ValidationError -> MAY_REQUIRE -> RemediationAction
  DataValidation -> MAY_REFERENCE -> ValidationEvidence
  DataValidation -> MAY_REFERENCE -> Provenance

# ---------- VALIDATION SCOPE ----------
entity ValidationScope:
  type: MetadataScope
  may_be:
    - dataset
    - table
    - record
    - field
    - value
    - relationship
    - file
    - stream
    - temporal_window

rule scope:
  ValidationResult SHOULD identify ValidationScope

rule scope_inheritance:
  valid dataset-level result MUST NOT automatically imply every individual value is valid
  unless validation method explicitly covered all values

# ---------- VALIDATION STATUS ----------
validation_status:
  - valid
  - invalid
  - warning
  - partially_valid
  - not_validated
  - validation_failed
  - unresolved

rule status:
  ValidationStatus MUST reflect assessed scope and applied rules

rule not_validated:
  NotValidated != Valid

rule warning:
  Warning != Invalid
  unless rule policy explicitly escalates warning to failure

# ---------- DATATYPE VALIDATION ----------
rule datatype:
  DataValue SHOULD conform to declared DataType

rule parseability:
  lexical representation MUST be parseable
  according_to DataType lexical rules

rule datatype_failure:
  IF value cannot be interpreted as declared DataType
  THEN produce ValidationError

# ---------- REQUIRED VALUES ----------
rule required:
  IF DataField is required
  THEN required value MUST be present

rule nullable:
  nullable field MAY contain null
  according_to schema semantics

rule missing_vs_null:
  missing != null
  and validation MUST apply declared field semantics

# ---------- RANGE ----------
entity RangeConstraint:
  subtype_of: ValidationConstraint
  may_include:
    - minimum
    - maximum
    - inclusive_minimum
    - inclusive_maximum

rule range:
  IF RangeConstraint exists
  THEN comparable DataValue MUST satisfy bounds

# ---------- PATTERN ----------
entity PatternConstraint:
  subtype_of: ValidationConstraint
  defines: AcceptedLexicalPattern

rule pattern:
  textual DataValue MUST satisfy declared PatternConstraint
  where constraint is mandatory

# ---------- ENUMERATION ----------
entity EnumerationConstraint:
  subtype_of: ValidationConstraint
  defines: AllowedValueSet

rule enumeration:
  DataValue MUST belong to AllowedValueSet
  when enumeration is mandatory

# ---------- UNIQUENESS ----------
rule uniqueness:
  IF field or identifier is declared unique
  THEN duplicate values within declared scope are invalid

rule duplicate_context:
  repeated value != validation error
  unless uniqueness constraint applies

# ---------- CARDINALITY ----------
rule cardinality:
  relationship or field occurrence count MUST satisfy declared minimum and maximum cardinality

rule cardinality_min:
  IF count < minimum
  THEN ValidationError

rule cardinality_max:
  IF maximum exists AND count > maximum
  THEN ValidationError

# ---------- REFERENTIAL VALIDATION ----------
rule reference:
  IF mandatory relationship references target identifier
  THEN target SHOULD resolve

rule orphan:
  unresolved mandatory reference produces ValidationError

rule external_reference:
  externally managed reference MAY remain unresolved locally
  if validation policy permits external resolution

# ---------- SCHEMA VALIDATION ----------
rule schema:
  DataResource declaring DataSchema SHOULD conform to schema structure

rule unknown_field:
  undeclared field MAY be:
    - invalid
    - warning
    - permitted_extension
  according_to schema policy

rule schema_version:
  validation MUST use applicable schema version

# ---------- SEMANTIC VALIDATION ----------
rule semantic_constraint:
  DataValue or relationship MAY be validated against:
    - ontology
    - controlled_vocabulary
    - domain_rule
    - logical_constraint

rule semantic_identifier:
  semantic validation SHOULD prefer stable identifiers over labels

rule syntactic_semantic_distinction:
  syntactically_valid != semantically_valid

# ---------- CROSS-FIELD VALIDATION ----------
rule dependency:
  IF field B depends on field A
  THEN evaluate dependency constraint

rule conditional_required:
  IF condition C is true
  THEN field X MAY become required

rule contradiction:
  mutually incompatible field values SHOULD produce semantic validation issue

# ---------- VALIDATION METHOD ----------
entity ValidationMethod:
  type: ProcessingMethod
  may_be:
    - schema_validation
    - rule_validation
    - statistical_validation
    - semantic_validation
    - referential_validation
    - checksum_validation
    - manual_review

rule method:
  DataValidation SHOULD identify ValidationMethod

rule deterministic_validation:
  deterministic rule result SHOULD be reproducible from same input and rule version

# ---------- SEVERITY ----------
validation_severity:
  - info
  - warning
  - error
  - critical

rule severity:
  each validation issue SHOULD have severity
  where prioritisation is required

# ---------- ERRORS / WARNINGS ----------
entity ValidationError:
  type: DataQualityIssue
  indicates: MandatoryValidationFailure

entity ValidationWarning:
  type: DataQualityIssue
  indicates: NonFatalValidationConcern

rule issue_trace:
  validation issue SHOULD preserve:
    - affected_resource
    - affected_path
    - violated_rule
    - observed_value
    - expected_constraint
    - severity
    - evidence

# ---------- EVIDENCE ----------
entity ValidationEvidence:
  type: Evidence
  may_include:
    - rule_output
    - parser_output
    - schema_result
    - reference_resolution
    - comparison_result

rule evidence:
  ValidationResult SHOULD preserve evidence sufficient to explain result

# ---------- REMEDIATION ----------
entity RemediationAction:
  type: DataProcessingAction
  examples:
    - correct_value
    - add_required_value
    - remove_duplicate
    - repair_reference
    - convert_datatype
    - standardise_format
    - map_vocabulary_term
    - manual_review

rule remediation:
  remediation SHOULD preserve:
    - original_value
    - correction
    - agent_or_process
    - timestamp
    - reason
    - provenance

rule revalidation:
  remediated data SHOULD be revalidated

# ---------- VALIDATION OF VALIDATION ----------
validation:
  require:
    - identifiable_data_resource
    - validation_rule_or_method

  validate_if_present:
    - validation_scope
    - rule_version
    - schema_version
    - result
    - severity
    - evidence
    - validator
    - assessment_time

  reject:
    - valid_status_with_known_mandatory_failure
    - invalid_status_without_applicable_failure where deterministic rules are complete
    - contradictory_validation_results_without context
    - validation_against_wrong_schema_version
    - unsupported_semantic_validation_claim

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "data validation" ~= "validation of data"
  "validation rule" ~= "data validation rule"
  "validation constraint" ~= "data constraint" WHEN used for validation
  "validation error" ~= "data validation failure"
  "validation warning" ~= "data validation warning"
  "schema validation" ~= "schema conformance validation"

  PRESERVE distinctions:
    validation != verification
    valid != accurate
    valid != complete
    warning != error
    not_validated != valid
    syntactic_validity != semantic_validity
    schema_validity != factual_truth
    remediation != original_data
    validation_rule != validation_result

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_validation_metadata:
    score: 1.00

  deterministic_rule_result:
    score: 1.00

  schema_validator_result:
    score: 0.99

  referential_resolution_result:
    score: 0.98

  semantic_rule_result:
    score: 0.95

  heuristic_validation:
    score: <= 0.80

  ambiguous_validation_context:
    score: <= 0.60
    action: preserve uncertainty

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_data_change:
    - identify affected ValidationRules
    - rerun validation
    - preserve previous ValidationResult
    - update issues and status
    - record provenance

  on_rule_change:
    - preserve previous rule version
    - revalidate affected DataResources
    - distinguish results by rule version
    - update provenance

  on_schema_change:
    - preserve previous schema version
    - select applicable schema
    - revalidate affected resources
    - record migration issues
    - update provenance

  on_remediation:
    - preserve original invalid state
    - apply correction
    - rerun applicable validation
    - record before_and_after results
    - update provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - validation_entities
    - relationships
    - datatype_rules
    - schema_rules
    - semantic_rules
    - referential_rules
    - remediation_rules
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - data metadata ontology
    - validation rule model
    - schema conformance structure
    - semantic validation model
    - remediation and audit structure

  operations:
    - identify validation rules
    - determine validation scope
    - validate datatypes
    - validate required values
    - validate ranges and patterns
    - validate enumerations
    - validate uniqueness and cardinality
    - resolve references
    - validate schema conformance
    - evaluate semantic constraints
    - classify errors and warnings
    - support remediation
    - preserve validation history
    - expose uncertainty

  constraints:
    - do_not_equate_valid_with accurate
    - do_not_equate_not_validated_with valid
    - do_not_hide_validation_errors
    - do_not_ignore_schema_version
    - do_not_overwrite_original_invalid_data_without provenance
    - preserve_rules_evidence_scope_and_history

# ---------- QUERY MODEL ----------
query:
  input:
    - data_resource
    - validation_rule
    - validation_method
    - validation_status
    - severity
    - schema
    - field
    - assessment_time

  resolve:
    1. identify DataResource
    2. retrieve DataValidation metadata
    3. determine ValidationScope
    4. resolve applicable ValidationRules
    5. resolve schema and rule versions
    6. evaluate constraints
    7. collect ValidationErrors and Warnings
    8. inspect evidence
    9. inspect remediation
    10. return status, confidence, and provenance

  output:
    - data_resource
    - validation_scope
    - applied_rules
    - validation_status
    - errors
    - warnings
    - evidence
    - remediation
    - provenance
    - confidence

# ---------- INFERENCE ----------
inference:
  IF value violates explicit mandatory datatype constraint
  THEN infer ValidationError
  WITH confidence = 1.00

  IF required field is absent
  THEN infer ValidationError
  WITH confidence = 1.00

  IF mandatory reference does not resolve
  THEN infer ReferentialValidationError
  WITH confidence = 1.00

  IF all applicable mandatory rules pass
  THEN infer validation_status = valid
  within assessed scope

  IF validation evidence is incomplete
  THEN do_not infer full validity
  AND preserve unresolved state

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: DataValidation
  validation_structure: machine_interpretable
  rules: explicit
  constraints: machine_evaluable
  scope: preserved
  status: explicit
  errors_and_warnings: traceable
  evidence: preserved
  remediation: auditable
  validation_history: preserved
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow