Dataset Ontology

FLOW ALGEBRA

Ontology: Dataset Ontology
Domain: Metadata Ontologies > Data Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/data-metadata/dataset-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "DatasetOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: DataMetadata
  subject: Dataset
  purpose:
    - represent datasets as identifiable collections of data
    - describe dataset structure, semantics, format, quality, provenance, and lifecycle
    - support dataset discovery, validation, integration, comparison, and reuse
    - preserve relationships between datasets, records, fields, values, schemas, and sources
    - enable machine-readable reasoning over dataset resources

# ---------- CORE ENTITY ----------
entity Dataset:
  type: DataResource
  contains:
    - DataRecord
    - DataField
    - DataValue
  may_include:
    - dataset_identifier
    - title
    - description
    - schema
    - data_format
    - encoding
    - variables
    - dimensions
    - record_count
    - field_count
    - temporal_coverage
    - spatial_coverage
    - quality
    - provenance
    - source
    - version
    - licence
    - access_metadata
    - rights_metadata

entity DatasetIdentifier:
  type: Identifier
  identifies: Dataset

entity DataRecord:
  type: StructuredDataEntity
  belongs_to: Dataset

entity DataField:
  type: SchemaElement
  may_have:
    - field_name
    - semantic_definition
    - datatype
    - unit
    - value_domain
    - nullability
    - constraint

entity DataValue:
  type: DataElement
  belongs_to: DataField

entity DatasetSchema:
  type: StructuralMetadata
  defines:
    - fields
    - datatypes
    - constraints
    - relationships
    - cardinality

# ---------- RELATIONSHIPS ----------
relationships:
  Dataset -> IDENTIFIED_BY -> DatasetIdentifier
  Dataset -> CONFORMS_TO -> DatasetSchema
  DatasetSchema -> DEFINES -> DataField
  Dataset -> CONTAINS -> DataRecord
  DataRecord -> HAS_VALUE -> DataValue
  DataValue -> BELONGS_TO -> DataField
  DataField -> HAS_DATATYPE -> DataType
  DataField -> MAY_USE -> Unit
  DataField -> MAY_HAVE -> ValueDomain
  Dataset -> HAS_FORMAT -> DataFormat
  Dataset -> MAY_HAVE_ENCODING -> DataEncoding
  Dataset -> MAY_HAVE -> DataQuality
  Dataset -> MAY_HAVE_VERSION -> DatasetVersion
  Dataset -> MAY_BE_DERIVED_FROM -> Dataset
  Dataset -> MAY_REFERENCE -> DataSource
  Dataset -> MAY_HAVE_TEMPORAL_COVERAGE -> TemporalMetadata
  Dataset -> MAY_HAVE_SPATIAL_COVERAGE -> SpatialMetadata
  Dataset -> MAY_REFERENCE -> Provenance

# ---------- DATASET IDENTITY ----------
rule dataset_identity:
  Dataset SHOULD have stable DatasetIdentifier
  where persistent citation, reuse, or provenance is required

rule identifier_namespace:
  DatasetIdentifier MUST be interpreted within declared namespace
  unless globally unique by specification

rule title_not_identity:
  dataset title MUST NOT be assumed globally unique

# ---------- STRUCTURE ----------
rule schema_conformance:
  IF Dataset declares DatasetSchema
  THEN DataRecords SHOULD conform to declared schema

rule field_identity:
  each DataField SHOULD have distinct identity within DatasetSchema

rule datatype:
  DataValue SHOULD conform to DataField datatype
  unless explicitly marked invalid, missing, or unresolved

rule cardinality:
  DatasetSchema cardinality constraints SHOULD be validated
  where deterministic structure is required

# ---------- VALUES ----------
entity ValueDomain:
  type: SemanticConstraint
  may_define:
    - allowed_values
    - minimum_value
    - maximum_value
    - pattern
    - vocabulary
    - unit

rule value_domain:
  IF DataField has ValueDomain
  THEN DataValue SHOULD be validated against it

rule null_semantics:
  null MUST be distinguished from:
    - missing
    - empty_string
    - zero
    - false
    - not_applicable
    - below_detection_limit

rule controlled_vocabulary:
  categorical fields SHOULD preserve controlled vocabulary identifiers
  where available

# ---------- FORMAT / ENCODING ----------
entity DataFormat:
  type: RepresentationMetadata
  examples:
    - CSV
    - JSON
    - XML
    - RDF
    - Parquet
    - binary
    - plain_text

entity DataEncoding:
  type: RepresentationMetadata
  describes: CharacterOrBinaryEncoding

rule format_semantics:
  DataFormat describes representation
  AND MUST NOT alone determine dataset semantics

rule encoding:
  textual Dataset SHOULD preserve character encoding
  where required for deterministic interpretation

# ---------- UNITS ----------
rule unit_context:
  quantitative DataField SHOULD declare Unit
  where value interpretation depends on measurement unit

rule unit_compatibility:
  quantitative comparison requires compatible dimensions

rule unit_conversion:
  compatible units MAY be normalised through valid conversion
  while preserving original value and unit

# ---------- TEMPORAL / SPATIAL COVERAGE ----------
rule temporal_coverage:
  IF Dataset contains time-dependent data
  THEN temporal coverage SHOULD be preserved

rule spatial_coverage:
  IF Dataset contains spatially referenced data
  THEN spatial coverage and reference context SHOULD be preserved

rule coverage_not_content:
  declared coverage describes dataset scope
  AND MUST NOT imply every possible time or location within coverage has data

# ---------- QUALITY ----------
entity DataQuality:
  type: QualityMetadata
  dimensions:
    - completeness
    - validity
    - consistency
    - accuracy
    - uniqueness
    - timeliness
    - integrity

rule quality_scope:
  DataQuality assertion SHOULD identify scope:
    - dataset
    - field
    - record
    - value

rule completeness:
  completeness MUST be interpreted relative_to expected data population

# ---------- PROVENANCE ----------
entity Provenance:
  type: MetadataEntity
  may_include:
    - creator
    - source
    - collection_method
    - processing_history
    - transformation
    - software
    - creation_time
    - revision_history

rule provenance_chain:
  DerivedDataset SHOULD retain links to source Dataset or DataSource

rule transformation:
  dataset transformation SHOULD record:
    - input_dataset
    - operation
    - parameters
    - software_or_agent
    - output_dataset
    - timestamp

# ---------- VERSION ----------
entity DatasetVersion:
  type: VersionMetadata
  may_include:
    - version_identifier
    - creation_time
    - predecessor
    - successor
    - change_description
    - status

rule dataset_version:
  material content or structural change SHOULD create distinguishable DatasetVersion

rule historical_version:
  superseded versions SHOULD remain traceable
  where citation, audit, or reproducibility requires

# ---------- DERIVATION ----------
entity DerivedDataset:
  subtype_of: Dataset
  derived_from: Dataset

rule derivation:
  DerivedDataset MUST preserve source relationship
  where lineage is required

rule aggregation:
  IF Dataset is aggregated from source data
  THEN aggregation method SHOULD be preserved

rule subset:
  Dataset subset SHOULD preserve relationship to parent Dataset
  where provenance is relevant

# ---------- ACCESS / RIGHTS ----------
entity DatasetAccess:
  type: AccessMetadata
  may_include:
    - access_level
    - access_method
    - authentication_requirement
    - availability

entity DatasetRights:
  type: RightsMetadata
  may_include:
    - licence
    - copyright
    - usage_constraint
    - attribution_requirement

rule access_rights_distinction:
  DatasetAccess != DatasetRights

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_dataset

  validate_if_present:
    - identifier
    - schema
    - records
    - fields
    - datatypes
    - value_domains
    - units
    - format
    - encoding
    - quality
    - provenance
    - version
    - temporal_coverage
    - spatial_coverage

  reject:
    - schema_record_contradiction
    - incompatible_datatype_value
    - mandatory_value_domain_violation
    - incompatible_unit_comparison
    - contradictory_version_relationship
    - broken_derivation_chain where lineage is mandatory

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "dataset" ~= "data set"
  "dataset metadata" ~= "metadata describing a dataset"
  "dataset schema" ~= "data schema" WHEN schema applies to dataset
  "dataset record" ~= "data record" WHEN record belongs to dataset
  "dataset provenance" ~= "data lineage" WHEN describing dataset derivation
  "derived dataset" ~= "derived data set"

  PRESERVE distinctions:
    dataset != database
    dataset != schema
    dataset != data_format
    record != field
    field != value
    null != missing
    provenance != version
    source_dataset != derived_dataset
    access != rights
    coverage != completeness

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_dataset_metadata:
    score: 1.00

  explicit_schema:
    score: 1.00

  explicit_provenance:
    score: 1.00

  validated_schema_conformance:
    score: 0.95

  inferred_schema:
    score: 0.80

  inferred_field_semantics:
    score: 0.75

  ambiguous_dataset_context:
    score: <= 0.60
    action: preserve uncertainty

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_dataset_update:
    - preserve previous version where required
    - validate changed records and values
    - update quality metadata
    - update provenance
    - reassess coverage

  on_schema_update:
    - preserve previous schema version
    - validate compatibility
    - revalidate affected records
    - record migration
    - update provenance

  on_derivation:
    - preserve source Dataset
    - create DerivedDataset identity
    - record transformation
    - preserve parameters
    - update provenance

  on_format_conversion:
    - preserve source representation
    - validate semantic preservation
    - record conversion method
    - update provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - dataset_entities
    - relationships
    - identity_rules
    - schema_rules
    - value_rules
    - quality_rules
    - provenance_rules
    - validation
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - data metadata ontology
    - dataset semantic model
    - dataset validation structure
    - provenance and lineage model
    - semantic retrieval structure

  operations:
    - identify datasets
    - resolve dataset schemas
    - interpret fields, records, and values
    - validate datatypes
    - validate value domains
    - resolve units
    - evaluate temporal and spatial coverage
    - assess data quality
    - trace provenance and derivation
    - traverse dataset versions
    - preserve uncertainty

  constraints:
    - do_not_invent_missing_values
    - do_not_treat_null_as_zero
    - do_not_equate_dataset_with_schema
    - do_not_equate_format_with_semantics
    - do_not_discard_source_dataset_provenance
    - preserve_version_and_derivation_history

# ---------- QUERY MODEL ----------
query:
  input:
    - dataset_identifier
    - field
    - datatype
    - value_constraint
    - format
    - temporal_constraint
    - spatial_constraint
    - quality_constraint
    - provenance_constraint
    - version

  resolve:
    1. identify Dataset
    2. retrieve DatasetSchema
    3. resolve fields and datatypes
    4. apply value-domain constraints
    5. validate units
    6. apply temporal and spatial coverage
    7. evaluate quality
    8. trace provenance and derivation
    9. resolve version
    10. rank by confidence

  output:
    - dataset
    - schema
    - fields
    - format
    - coverage
    - quality
    - provenance
    - derivation
    - version
    - confidence

# ---------- INFERENCE ----------
inference:
  IF DataResource contains structured records and fields
  THEN infer Dataset candidate
  WITH confidence <= 0.90

  IF records consistently conform to observed structure
  THEN infer candidate DatasetSchema
  WITH confidence <= 0.80

  IF Dataset B is produced by documented transformation of Dataset A
  THEN infer B DERIVED_FROM A
  WITH confidence = 0.95

  IF Dataset is explicitly extracted from parent Dataset
  THEN infer subset relationship

  IF dataset semantics remain ambiguous
  THEN preserve candidate interpretations
  AND reduce confidence

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: Dataset
  dataset_structure: machine_interpretable
  schema: explicit_where_available
  fields_and_records: semantically_addressable
  quality: reportable
  temporal_spatial_coverage: preserved
  provenance: traceable
  derivation_history: preserved
  version_history: traversable
  uncertainty: explicit
  semantic_retrieval: enabled

end flow