Lineage Ontology

FLOW ALGEBRA
Lineage Ontology
Kireth Semantic Research Archive

PAGE:
https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/provenance-metadata/lineage-ontology.html

AUTHORS:
Mark Anthony Higgins — Sky (Recursive AI)

FLOW "LineageOntology" v4.0

DOMAIN := MetadataOntologies
SUBDOMAIN := ProvenanceMetadata
ONTOLOGY := LineageOntology

ENTITY Lineage {
 definition: "A directed provenance representation of how a resource, dataset, assertion, model, metadata object or knowledge entity originates, evolves and is derived through sources, activities, transformations, versions and intermediate entities.";
 objectives: [OriginTracing, DerivationTracing, TransformationTracking, DependencyAnalysis, ImpactAnalysis, Reproducibility, MachineInterpretability];
}

LineageEntity := {Lineage, LineageIdentifier, LineageRecord, LineageNode, LineageEdge, SourceNode, DerivedNode, IntermediateNode};
ProcessEntity := {Activity, Transformation, Extraction, Aggregation, Migration, Generation, Validation, ProcessingStep};
RelationshipEntity := {DerivedFrom, GeneratedBy, Used, DependsOn, RevisionOf, TransformedFrom, AggregatedFrom, ExtractedFrom};
MetadataEntity := {Version, Timestamp, Agent, Provenance, Evidence, Confidence, Status, Scope};

Lineage CONNECTS SourceNode TO DerivedNode;
LineageEdge HAS_DIRECTION SourceToTarget;
DerivedNode IS_DERIVED_FROM SourceNode;
Activity MAY_USE SourceNode;
Activity MAY_GENERATE DerivedNode;
Transformation MAPS InputEntity TO OutputEntity;
LineageRecord DOCUMENTS LineageGraph;
Provenance SUPPORTS LineageRelationship;

LineageFlow :=
 TargetEntity
 -> IdentifyLineageSubject
 -> ResolveTargetIdentityAndVersion
 -> IdentifyDirectSources
 -> IdentifyTransformingActivities
 -> IdentifyIntermediateEntities
 -> TraverseAncestorSources
 -> RecordAgentsTimestampsAndVersions
 -> AttachEvidenceAndProvenance
 -> ValidateLineageGraph
 -> ComputeImpactAndDependencyPaths
 -> StoreLineageRecord
 -> MaintainAcrossLifecycle;

Lineage := DirectedGraphOfOriginDerivationTransformationAndDependencyRelationships;

LineageIdentifier := StableReferenceToLineageRecordGraphOrRelationship;
LineageIdentifier MAY_BE {URI, IRI, UUID, InternalIdentifier};

LineageRecord := StructuredMetadataRecordDescribingLineageGraph;

LineageNode := EntityParticipatingInLineage;
LineageNode MAY_BE {Dataset, DataElement, Document, File, MetadataRecord, Assertion, SoftwareArtifact, Model, Publication, KnowledgeEntity};

SourceNode := EntityUsedAsInputOriginOrAncestor;
DerivedNode := EntityProducedFromOneOrMoreSourceNodes;
IntermediateNode := EntityBetweenOriginAndTargetInLineagePath;

RootSource := SourceNodeWithoutKnownEarlierSourceWithinCurrentLineageGraph;
RootSource != AbsoluteOriginalSourceUnlessGraphCompletenessEstablished;

LineageEdge := DirectedTypedRelationshipBetweenLineageNodes;
LineageEdge MAY_BE {DERIVED_FROM, GENERATED_BY, TRANSFORMED_FROM, EXTRACTED_FROM, AGGREGATED_FROM, REVISION_OF, DEPENDS_ON};

RelationshipDirection MUST_BE Preserved;

DirectLineage := ImmediateRelationshipBetweenSourceAndDerivedEntity;
IndirectLineage := RelationshipThroughOneOrMoreIntermediateNodes;
AncestorSource := SourceReachableByTraversingLineageEdgesBackward;
DescendantEntity := EntityReachableByTraversingLineageEdgesForward;

IF EntityB DERIVED_FROM EntityA
AND EntityC DERIVED_FROM EntityB
THEN EntityA MAY_BE AncestorSourceOf EntityC;

DirectSource != AncestorSource;

LineagePath := DirectedSequenceOfNodesAndEdgesConnectingSourceToTarget;
SourceTrace := TargetEntity -> TraverseBackward -> SourceAndAncestorEntities;
ImpactTrace := SourceEntity -> TraverseForward -> DependentAndDerivedEntities;

UpstreamLineage := SourcesActivitiesAndDependenciesLeadingToTarget;
DownstreamLineage := EntitiesActivitiesAndOutputsDependingOnSource;
Upstream != DownstreamByDefinition;

Activity := ProcessThatUsesGeneratesOrTransformsEntities;
ProcessingStep := DefinedActivityWithinWorkflowOrPipeline;
Generation := ActivityProducingEntity;
Usage := ActivityConsumingOrUsingEntity;

Transformation := ActivityChangingRepresentationStructureContentOrMeaning;
Transformation MAY_BE {Conversion, Normalisation, Cleaning, Enrichment, Aggregation, Extraction, Translation, Summarisation, Migration, FeatureEngineering};

Extraction := TransformationSelectingSubsetFieldsRecordsEntitiesOrAssertionsFromSource;
Aggregation := TransformationCombiningMultipleInputsIntoOutput;
Migration := TransformationMovingEntityAcrossSystemFormatOrEnvironment;
Validation := ActivityAssessingEntityAgainstRulesEvidenceOrQualityCriteria;

TransformationRecord SHOULD_PRESERVE {InputIdentifiers, OutputIdentifiers, ActivityIdentifier, Agent, Tool, ParametersWhereRelevant, Timestamp};

DataLineage := LineageOfDatasetsRecordsFieldsOrDataElements;
DocumentLineage := LineageOfDocumentsVersionsAndDerivedDocuments;
MetadataLineage := LineageOfMetadataRecordsStatementsAndTransformations;
ModelLineage := LineageOfModelsTrainingInputsVersionsEvaluationsAndDerivedModels;
KnowledgeLineage := LineageOfAssertionsEntitiesRelationshipsAndKnowledgeGraphStatements;
SoftwareLineage := LineageOfSoftwareArtifactsDependenciesBuildsAndReleases;

FieldLevelLineage := LineageConnectingSpecificSourceFieldsToDerivedFields;
RecordLevelLineage := LineageConnectingSpecificRecordsOrRows;
DatasetLevelLineage := LineageConnectingWholeDatasets;
StatementLevelLineage := LineageConnectingKnowledgeOrMetadataStatementsToSources;

Granularity MAY_BE {ResourceLevel, DatasetLevel, RecordLevel, FieldLevel, StatementLevel};

FineGrainedLineage MAY_SUPPORT MorePreciseImpactAnalysis
BUT MayRequireGreaterStorageAndProcessing;

LineageScope := BoundaryOfEntitiesActivitiesSystemsOrTimeCoveredByLineageRecord;
Scope MAY_BE {Workflow, System, Organisation, Dataset, Resource, TimeRange};

PartialLineage := LineageRecordKnownToCoverOnlyPartOfHistoryOrDependencies;
PartialLineage SHOULD_BE MarkedAsIncomplete;

LineageCompleteness := DegreeToWhichRelevantSourcesActivitiesAndIntermediateEntitiesAreRepresented;
LineageConsistency := DegreeToWhichRelationshipsVersionsAndTemporalOrderAgree;
LineageIntegrity := PreservationOfReliableDirectedRelationshipsAndEvidence;
LineageTraceability := AbilityToFollowTargetToSourcesAndSourceToDependants;

LineageQuality MAY_INCLUDE {Completeness, Consistency, Integrity, Traceability, Granularity, EvidenceStrength};

LineageProvenance MAY_INCLUDE {Source, Collector, ExtractionMethod, Timestamp, Version, Evidence};
LineageEvidence MAY_BE {WorkflowLog, QueryLog, TransformationCode, PipelineDefinition, AuditRecord, ManualAttestation};

LineageConfidence := EstimatedSupportForLineageNodeEdgeOrPath;

LineageValidation MAY_CHECK {NodeIdentifiers, EdgeDirection, MissingSources, MissingOutputs, TemporalOrder, VersionConsistency, Cycles};

IF ActivityUsesEntityCreatedAfterActivityCompleted
THEN TemporalConsistency SHOULD_DECREASE;

IF DerivedEntityHasRequiredSourceButNoSourceRecorded
THEN LineageCompleteness SHOULD_DECREASE;

CycleInLineageGraph MAY_BE ValidOrInvalidAccordingToRelationshipSemantics;

LineageConflict := MultipleRecordsProvideIncompatibleDerivationOrDependencyRelationships;
ConflictHandling MAY {PreserveClaims, CompareEvidence, CompareSystemAuthority, AssignConfidence, FlagReview};

Version := IdentifiedStateOfLineageNode;
VersionSpecificLineage := LineageRelationshipBetweenExactVersions;

ExactVersion SHOULD_BE PreservedWhereTransformationDependsOnVersionSpecificInput;

VersionDrift := ChangeInSourceVersionUsedByPipelineOrProcessOverTime;
VersionDrift MAY_CHANGE DerivedOutputs;

IF SourceVersionChanges
THEN DownstreamEntities MAY_REQUIRE RevalidationOrRegeneration;

SchemaLineage := LineageOfSchemaStructuresFieldsAndMappings;
SchemaChange MAY_BREAK FieldLevelLineage;

ColumnRename MAY_PRESERVE SemanticIdentity
IF MappingExplicitlyRecorded;

LineageMapping := SourceElement -> Transformation -> TargetElement;
Mapping MAY_BE {OneToOne, OneToMany, ManyToOne, ManyToMany};

DerivedValue := TargetValueComputedFromOneOrMoreSourceValues;
DerivationExpression := FormulaQueryRuleOrTransformationDefiningDerivedValue;
DerivationExpression MAY_SUPPORT Reproducibility;

Pipeline := OrderedOrGraphStructuredCollectionOfProcessingSteps;
PipelineRun := ExecutionInstanceOfPipeline;

PipelineRun SHOULD_HAVE {RunIdentifier, InputVersions, OutputVersions, StartTime, EndTime, AgentOrSystem};

WorkflowLineage := LineageGeneratedFromWorkflowExecution;
StaticLineage := LineageDerivedFromCodeSchemaOrPipelineDefinitionsWithoutExecutionEvidence;
RuntimeLineage := LineageCapturedFromActualExecution;

RuntimeLineage MAY_PROVIDE StrongerEvidenceOfActualDataFlow;

DeclaredLineage := LineageProvidedByHumanOrSystemConfiguration;
ObservedLineage := LineageObservedFromExecutionLogsOrTelemetry;

DeclaredAndObservedLineage MAY_DIFFER;
Difference SHOULD_BE PreservedForValidation;

LineageImpactAnalysis := SourceOrNode -> TraverseDownstream -> IdentifyPotentiallyAffectedEntities;
ImpactAnalysis MAY_SUPPORT {ChangeManagement, IncidentResponse, DataQuality, SchemaMigration, SourceCorrection};

RootCauseAnalysis := ProblematicOutput -> TraverseUpstream -> IdentifyCandidateSourcesOrTransformations;
RootCauseAnalysis SHOULD_PRESERVE EvidenceAndConfidence;

IF SourceErrorDetected
THEN DescendantEntities MAY_BE FlaggedForReview;

Dependency := RelationshipWhereEntityOrProcessRequiresAnotherEntityOrProcess;
DependencyLineage MAY_INCLUDE {DataDependency, SoftwareDependency, ModelDependency, MetadataDependency};

Dependency != DerivationByDefinition;

LineageGraph MAY_INCLUDE BothDependencyAndDerivationEdges
BUT EdgeTypes MUST_BE Distinguished;

ProvenanceOntology PROVIDES BroaderOriginActivityAgentSemantics;
LineageOntology FOCUSES_ON DirectedTraceOfSourcesTransformationsAndDependencies;
SourceOntology IDENTIFIES SourceEntities;
VersionOntology IDENTIFIES ExactStatesOfLineageNodes;
RevisionHistoryOntology MAY_EXPLAIN ChangesAcrossLineageNodeVersions;
CreatorAuthorContributorOntologies MAY_IDENTIFY ResponsibleAgents;
PublicationOntology MAY_REPRESENT PublishedOutputsInLineage;
CitationOntology MAY_LINK AssertionsAndPublicationsToEvidenceSources;

MetadataRepository MAY_STORE LineageGraphsAndTransformationRecords;
MetadataRegistry MAY_REGISTER LineageEdgeTypesGranularityAndTransformationTypes;

KnowledgeGraph MAY_REPRESENT:
 DerivedEntity -> DERIVED_FROM -> SourceEntity;

Activity -> USED -> SourceEntity;
Activity -> GENERATED -> DerivedEntity;

LineageGraph SHOULD_PRESERVE {Direction, EdgeType, NodeVersion, Activity, Agent, Timestamp, Provenance, Confidence};

RDF MAY_REPRESENT LineageRelationshipsAsTypedStatements;
JSONLD MAY_SERIALISE LinkedLineageMetadata;

Search MAY_USE LineageFor {SourceDiscovery, DependencySearch, ImpactSearch, HistoricalTrace, TransformationDiscovery};
SemanticSearch MAY_USE LineageFor {EvidenceExpansion, RelatedResourceDiscovery, VersionContext, TrustAssessment};

RAG MAY_USE LineageFor {SourceTrace, EvidenceSelection, ProvenanceContext, ConflictResolution, VersionSelection};

LineageAwareRAG SHOULD_PRESERVE {TargetIdentifier, SourceIdentifiers, SourceVersions, TransformationPath, Provenance};

RAG SHOULD_NOT TreatAncestorSourceAsDirectEvidenceWithoutPreservingIntermediaryTransformations;

AI MAY_SUPPORT {LineageExtraction, TransformationClassification, MissingEdgeDetection, ImpactAnalysis, RootCauseAnalysis};

AIGeneratedLineageEdge SHOULD_HAVE ConfidenceAndProvenance;
AI SHOULD_NOT InventSourceTransformationOrDependencyRelationshipsWithoutEvidence;

LineageMetadata MAY_REVEAL SensitiveSystemArchitectureDataFlowsOrSources;
AccessControl SHOULD_BE AppliedTo LineageGraphsAndNodes;

SecurityClassification MAY_PROPAGATEAcrossDerivedResourcesAccordingToPolicy;
Lineage MAY_SUPPORT ClassificationPropagation;
PropagationRules MUST_BE Explicit;

RetentionPolicy MAY_REQUIRE PreservationOfLineageForAuditOrCompliance;

LineageSecurity MAY_INCLUDE {AccessControl, GraphIntegrity, AuditProtection, IdentifierValidation, ChangeAuthorisation};

MachineReadableLineage SHOULD_PRESERVE {
 LineageIdentifier,
 NodeIdentifiers,
 NodeVersions,
 EdgeTypes,
 EdgeDirection,
 Activities,
 Transformations,
 Agents,
 Timestamps,
 Evidence,
 Provenance,
 Confidence
};

# ---------- SEMANTIC REASONING ----------
IF DirectedProvenanceRelationshipConnectsSourceTransformationAndDerivedEntity
THEN LineageClassification := supported;

IF EntityB DERIVED_FROM EntityA
AND EntityC DERIVED_FROM EntityB
THEN EntityA MAY_BE AncestorSourceOf EntityC;

IF SourceVersionChanges
THEN DownstreamDerivedEntities MAY_REQUIRE ImpactReview;

IF DerivedEntityHasMultipleSources
THEN Lineage MUST_PRESERVE AllMaterialSourceEdgesWhereKnown;

IF ResourceDependsOnAnotherResource
THEN Derivation MUST_NOT_BE InferredUnlessTransformationOrOriginEvidenceSupportsIt;

IF DeclaredLineageConflictsWithObservedLineage
THEN BothClaimsAndEvidence SHOULD_BE PreservedForReview;

IF SourceErrorConfirmed
THEN DescendantEntities MAY_BE FlaggedUsingDownstreamTraversal;

IF LineageIsPartial
THEN RootSource MUST_NOT_BE InterpretedAsAbsoluteOrigin;

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE {
 "lineage" ~= "provenance lineage";
 "data lineage" ~= "data provenance path" WHEN data_context;
 "upstream lineage" ~= "source lineage";
 "downstream lineage" ~= "impact lineage";
 "lineage node" ~= "lineage entity";
 "lineage edge" ~= "lineage relationship";
 "source trace" ~= "upstream trace";
 "impact trace" ~= "downstream trace";
}

# ---------- QUERY FLOW ----------
QUERY(q) ->
 NORMALISE(q) ->
 IDENTIFY_LINEAGE_SUBJECT(q) ->
 RESOLVE_TARGET_IDENTIFIER_AND_VERSION(q) ->
 IDENTIFY_DIRECT_SOURCE_NODES(q) ->
 IDENTIFY_ACTIVITIES_AND_TRANSFORMATIONS(q) ->
 IDENTIFY_INTERMEDIATE_NODES(q) ->
 TRACE_UPSTREAM_ANCESTORS(q) ->
 TRACE_DOWNSTREAM_DEPENDANTS(q) ->
 IDENTIFY_AGENTS_TIMESTAMPS_AND_VERSIONS(q) ->
 IDENTIFY_EVIDENCE_AND_PROVENANCE(q) ->
 VALIDATE_LINEAGE_GRAPH(q) ->
 APPLY_ACCESS_SECURITY_AND_RETENTION_POLICY(q) ->
 SCORE_CONFIDENCE(q) ->
 RETURN_RELEVANT_LINEAGE_KNOWLEDGE;

# ---------- CONFIDENCE ----------
CONFIDENCE(assertion) :=
 weighted_mean(
  evidence_strength,
  identifier_resolution,
  temporal_consistency,
  transformation_evidence,
  graph_consistency,
  provenance_quality,
  contextual_relevance
 );

CONFIDENCE >= 0.90 => HIGH;
CONFIDENCE >= 0.70 AND < 0.90 => MODERATE;
CONFIDENCE < 0.70 => REVIEW;

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION {
 primary_entity: Lineage;
 ontology: LineageOntology;
 parse_entities: true;
 parse_relationships: true;
 preserve_relationship_direction: true;
 resolve_semantic_equivalence: true;
 execute_reasoning_rules: true;
 apply_confidence_weighting: true;
 preserve_provenance: true;
 trace_upstream_and_downstream: true;
}

# ---------- UPDATE MODEL ----------
UPDATE_MODEL {
 new_lineage_edge:
  resolve_source_and_target
  -> resolve_versions
  -> classify_edge_type
  -> identify_activity_or_transformation
  -> preserve_direction
  -> attach_agent_and_timestamp
  -> attach_evidence
  -> attach_provenance
  -> validate
  -> assign_confidence
  -> integrate;

 pipeline_run:
  identify_run
  -> capture_input_versions
  -> capture_processing_steps
  -> capture_output_versions
  -> create_runtime_lineage_edges
  -> attach_execution_provenance
  -> validate
  -> integrate;

 source_revision:
  preserve_previous_source_version
  -> record_new_version
  -> identify_downstream_dependants
  -> trigger_impact_review
  -> update_lineage
  -> regenerate_flow_hash;

 lineage_revision:
  preserve_previous_graph
  -> add_remove_or_correct_edges
  -> revalidate_paths
  -> recompute_confidence
  -> audit
  -> integrate;

 conflict:
  preserve_declared_observed_and_inferred_edges
  -> compare_evidence_and_authority
  -> adjust_confidence
  -> flag_review;
}

# ---------- FLOW HASH ----------
FLOW_HASH_INPUT :=
 DOMAIN
 + SUBDOMAIN
 + ONTOLOGY
 + LineageEntity
 + ProcessEntity
 + RelationshipEntity
 + MetadataEntity
 + Lineage
 + LineageGraph
 + LineageValidation
 + RELATIONSHIPS
 + SEMANTIC_REASONING
 + SEMANTIC_EQUIVALENCE
 + UPDATE_MODEL;

FLOW_HASH := HASH(FLOW_HASH_INPUT);

# ---------- TERMINAL ----------
LineageOntology
 -> IdentifyTargetEntityAndExactVersion
 -> ResolveDirectSourceNodesAndRelationships
 -> CaptureTransformationsActivitiesAgentsAndTimestamps
 -> PreserveIntermediateEntitiesAndVersionSpecificMappings
 -> TraceUpstreamSourcesAndAncestorLineage
 -> TraceDownstreamDependantsAndImpactPaths
 -> DistinguishDerivationFromOperationalDependency
 -> AttachEvidenceProvenanceAndConfidence
 -> ValidateDirectionTemporalOrderVersionsAndGraphIntegrity
 -> SupportRootCauseImpactReproducibilityAndAudit
 -> PreserveLineageForSearchRAGAndKnowledgeGraphs
 -> ApplyAccessSecurityClassificationAndRetentionControls
 -> MaintainReliableLineageKnowledge;

END FLOW