Data Provenance Ontology

FLOW ALGEBRA

Ontology: Data Provenance Ontology
Domain: Metadata Ontologies > Data Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/data-metadata/data-provenance-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "DataProvenanceOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: DataMetadata
  subject: DataProvenance
  purpose:
    - represent the origin, history, custody, transformation, and derivation of data
    - preserve relationships between source data, processes, agents, and derived resources
    - support traceability, reproducibility, validation, auditing, and trust assessment
    - distinguish source assertions from derived or transformed data
    - enable machine-readable reasoning over data lineage

# ---------- CORE ENTITY ----------
entity DataProvenance:
  type: ProvenanceMetadata
  describes: DataResource
  may_include:
    - provenance_identifier
    - source
    - creator
    - contributor
    - creation_time
    - acquisition_method
    - processing_activity
    - transformation
    - derivation
    - custody
    - version
    - software
    - agent
    - timestamp
    - evidence
    - provenance_chain

entity DataResource:
  type: InformationResource
  may_be:
    - Dataset
    - DataFile
    - Database
    - Record
    - DataObject
    - DerivedDataset
    - DataStream

entity ProvenanceEntity:
  type: Entity
  participates_in: ProvenanceRelationship

entity ProvenanceActivity:
  type: Process
  acts_on: DataResource
  may_produce: DataResource

entity ProvenanceAgent:
  type: Agent
  may_be:
    - Person
    - Organisation
    - SoftwareAgent
    - Instrument
    - AutomatedProcess

# ---------- RELATIONSHIPS ----------
relationships:
  DataProvenance -> DESCRIBES -> DataResource
  DataResource -> WAS_GENERATED_BY -> ProvenanceActivity
  ProvenanceActivity -> USED -> DataResource
  ProvenanceActivity -> WAS_ASSOCIATED_WITH -> ProvenanceAgent
  DataResource -> WAS_ATTRIBUTED_TO -> ProvenanceAgent
  DataResource -> WAS_DERIVED_FROM -> DataResource
  DataResource -> MAY_BE_VERSION_OF -> DataResource
  DataResource -> MAY_BE_REVISION_OF -> DataResource
  DataResource -> MAY_HAVE_SOURCE -> DataSource
  ProvenanceActivity -> MAY_HAVE_TIME -> Timestamp
  DataProvenance -> MAY_REFERENCE -> ProvenanceEvidence

# ---------- SOURCE ----------
entity DataSource:
  type: ProvenanceEntity
  may_be:
    - original_dataset
    - observation
    - measurement
    - database
    - file
    - API
    - instrument
    - publication
    - external_provider

rule source_identity:
  DataSource SHOULD be identifiable sufficiently for traceability

rule source_not_creator:
  DataSource != Creator
  unless same entity performs both roles

rule original_source:
  "original" MUST be interpreted relative_to known provenance chain
  and MUST NOT imply absolute first creation without evidence

# ---------- CREATION ----------
rule creation:
  newly created DataResource SHOULD preserve:
    - creator_or_agent
    - creation_time
    - creation_activity
    - source_inputs_if_any

rule generated_data:
  DataResource generated by process SHOULD link to generating ProvenanceActivity

# ---------- DERIVATION ----------
entity Derivation:
  type: ProvenanceRelationship
  relates:
    - source_resource
    - derived_resource

rule derivation:
  IF DataResource B is produced using DataResource A
  THEN B WAS_DERIVED_FROM A

rule transitive_lineage:
  IF C derived_from B
  AND B derived_from A
  THEN C has_ancestor_source A

rule derivation_not_identity:
  DerivedResource != SourceResource
  unless transformation explicitly preserves same resource identity/version semantics

# ---------- TRANSFORMATION ----------
entity DataTransformation:
  subtype_of: ProvenanceActivity
  may_include:
    - operation
    - parameters
    - software
    - software_version
    - input
    - output
    - timestamp
    - agent

rule transformation_trace:
  DataTransformation SHOULD preserve:
    - input_resources
    - operation
    - parameters
    - executing_agent_or_software
    - output_resources
    - time

rule transformation_loss:
  IF transformation is lossy
  THEN loss SHOULD be recorded where relevant

# ---------- PROCESSING HISTORY ----------
entity ProcessingHistory:
  type: OrderedProvenanceSequence
  contains: ProvenanceActivity

rule activity_order:
  processing activities SHOULD preserve temporal or dependency order

rule processing_chain:
  output of activity MAY become input to subsequent activity

rule incomplete_history:
  missing provenance activity SHOULD be represented as unknown gap
  rather_than invented process

# ---------- AGENTS ----------
rule agent_identity:
  ProvenanceAgent SHOULD have stable identity where accountability is required

rule agent_role:
  role SHOULD be associated with specific ProvenanceActivity

agent_roles:
  - creator
  - contributor
  - processor
  - curator
  - publisher
  - custodian
  - operator
  - software_agent

# ---------- SOFTWARE ----------
entity SoftwareAgent:
  subtype_of: ProvenanceAgent
  may_include:
    - software_name
    - version
    - build
    - configuration
    - execution_environment

rule software_version:
  software version SHOULD be preserved
  where processing reproducibility depends on implementation

rule configuration:
  material software configuration SHOULD be preserved

# ---------- TEMPORAL PROVENANCE ----------
rule activity_time:
  ProvenanceActivity SHOULD preserve:
    - start_time
    - end_time
  OR event timestamp where available

rule temporal_order:
  generating activity MUST NOT occur after dependent activity
  unless temporal semantics explicitly permit asynchronous or delayed recording

# ---------- VERSION PROVENANCE ----------
entity DataVersion:
  type: VersionMetadata
  may_include:
    - version_identifier
    - predecessor
    - successor
    - creation_time
    - change_description

rule revision:
  IF DataResource is materially revised
  THEN preserve relationship to previous DataVersion

rule supersession:
  superseded version SHOULD remain historically traceable

rule restoration:
  restoration of old content SHOULD create new provenance event
  rather_than erase intervening history

# ---------- CUSTODY ----------
entity CustodyEvent:
  type: ProvenanceActivity
  may_include:
    - from_agent
    - to_agent
    - timestamp
    - location
    - condition

rule custody:
  chain_of_custody SHOULD preserve ordered transfers
  where accountability or evidential integrity requires

rule custody_gap:
  unknown custody interval SHOULD remain explicit

# ---------- EVIDENCE ----------
entity ProvenanceEvidence:
  type: Evidence
  may_include:
    - audit_log
    - transaction_log
    - signature
    - certificate
    - checksum
    - source_record
    - workflow_record

rule evidence:
  provenance assertion SHOULD reference evidence where available

rule evidence_not_provenance:
  evidence supports provenance assertion
  but MUST NOT be conflated with provenance relationship itself

# ---------- PROVENANCE GRAPH ----------
entity ProvenanceGraph:
  type: DirectedGraph
  nodes:
    - DataResource
    - ProvenanceActivity
    - ProvenanceAgent
  edges:
    - used
    - generated
    - derived_from
    - attributed_to
    - associated_with

rule graph_acyclic_derivation:
  derivation relationships SHOULD avoid unsupported cycles

rule graph_traversal:
  provenance graph SHOULD support upstream and downstream lineage traversal

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_data_resource

  validate_if_present:
    - source
    - creator
    - activity
    - agent
    - transformation
    - derivation
    - timestamps
    - version
    - custody
    - evidence

  reject:
    - self_derivation_without explicit version semantics
    - impossible_activity_order
    - contradictory_agent_attribution
    - broken_required_lineage
    - circular_derivation_without justified semantics
    - provenance_claim_without identifiable referent

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "data provenance" ~= "data lineage" WHEN describing origin and derivation
  "provenance history" ~= "processing history" WHEN processing forms provenance chain
  "derived from" ~= "was derived from"
  "data origin" ~= "source provenance"
  "chain of custody" ~= "custody history"

  PRESERVE distinctions:
    provenance != metadata_source
    source != creator
    activity != agent
    derivation != identity
    version != provenance
    evidence != provenance_relationship
    custody != ownership
    processing_history != raw_data
    restored_version != original_historical_event

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_provenance_record:
    score: 1.00

  signed_or_audited_provenance:
    score: 0.99

  explicit_derivation_link:
    score: 1.00

  deterministic_workflow_trace:
    score: 0.98

  inferred_transitive_lineage:
    score: 0.95

  inferred_source_from_context:
    score: 0.75

  incomplete_provenance_chain:
    score: <= 0.60
    action: preserve gaps

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_new_resource:
    - assign identity
    - record generating activity
    - record source inputs
    - record agent
    - record creation time
    - update provenance graph

  on_transformation:
    - preserve input resources
    - create transformation activity
    - record method and parameters
    - create or identify output resource
    - link output to inputs
    - update provenance graph

  on_revision:
    - preserve previous DataVersion
    - create new version
    - record revision activity
    - record change description
    - update lineage

  on_provenance_correction:
    - preserve previous provenance assertion
    - add corrected assertion
    - record correction reason and agent
    - preserve audit history

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - provenance_entities
    - relationships
    - source_rules
    - derivation_rules
    - transformation_rules
    - version_rules
    - custody_rules
    - validation
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - data metadata ontology
    - provenance graph model
    - data lineage structure
    - reproducibility and audit model

  operations:
    - identify data sources
    - resolve generating activities
    - resolve agents
    - trace derivation
    - reconstruct processing history
    - traverse upstream lineage
    - traverse downstream lineage
    - inspect software and parameters
    - reconstruct version history
    - inspect custody
    - evaluate provenance evidence
    - preserve unknown gaps
    - expose uncertainty

  constraints:
    - do_not_invent_missing_provenance
    - do_not_equate_source_with_creator
    - do_not_discard_superseded_versions
    - do_not_hide_lossy_transformations
    - do_not_collapse_agents_activities_and_entities
    - preserve_lineage_evidence_and_history

# ---------- QUERY MODEL ----------
query:
  input:
    - data_resource
    - source
    - agent
    - activity
    - transformation
    - version
    - time_constraint
    - provenance_depth
    - direction

  resolve:
    1. identify DataResource
    2. retrieve DataProvenance
    3. resolve source entities
    4. resolve generating and processing activities
    5. resolve associated agents
    6. traverse derivation relationships
    7. reconstruct version history
    8. inspect custody and evidence
    9. preserve unknown gaps
    10. return confidence

  output:
    - data_resource
    - sources
    - activities
    - agents
    - derivation_chain
    - processing_history
    - version_history
    - custody_history
    - evidence
    - confidence

# ---------- INFERENCE ----------
inference:
  IF resource B is generated by activity using resource A
  THEN infer B WAS_DERIVED_FROM A
  WITH confidence = 0.98

  IF C derived_from B
  AND B derived_from A
  THEN infer A is upstream ancestor of C
  WITH confidence = 0.95

  IF explicit workflow log identifies inputs and outputs
  THEN construct provenance edges
  WITH confidence = 0.98

  IF provenance chain has unresolved interval
  THEN preserve provenance_gap
  AND do_not invent missing activity

  IF restored content originates from historical version
  THEN link restoration activity to historical source version
  AND preserve intervening history

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: DataProvenance
  provenance_structure: machine_interpretable
  sources: identifiable
  agents: traceable
  activities: ordered_where_possible
  derivation_chain: traversable
  processing_history: preserved
  version_history: preserved
  custody_history: preserved_where_applicable
  evidence: traceable
  provenance_gaps: explicit
  uncertainty: explicit
  semantic_retrieval: enabled

end flow