Embedding Metadata Ontology

FLOW ALGEBRA

Ontology: Embedding Metadata Ontology
Domain: Metadata Ontologies > Semantic Search Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/semantic-search-metadata/embedding-metadata-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "EmbeddingMetadataOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: SemanticSearchMetadata
  subject: EmbeddingMetadata
  purpose:
    - represent metadata describing vector embeddings used in semantic search and machine reasoning
    - preserve embedding identity, source resource, model, model version, dimensions, vector space, normalisation, metric, and provenance
    - support semantic similarity, vector retrieval, clustering, classification, recommendation, and retrieval-augmented systems
    - distinguish vector similarity from semantic equivalence and factual correctness
    - enable machine-readable reasoning over embedding lifecycles

# ---------- CORE ENTITY ----------
entity EmbeddingMetadata:
  type: SemanticSearchMetadata
  describes: Embedding
  may_include:
    - embedding_identifier
    - source_resource
    - source_fragment
    - embedding_type
    - embedding_model
    - model_version
    - vector_dimension
    - vector_datatype
    - normalisation
    - distance_metric
    - pooling_method
    - tokenisation
    - language
    - creation_time
    - validity
    - confidence
    - provenance

entity Embedding:
  type: VectorRepresentation
  represents: SemanticOrFeatureRepresentation

entity EmbeddingIdentifier:
  type: Identifier
  identifies: Embedding

# ---------- RELATIONSHIPS ----------
relationships:
  EmbeddingMetadata -> DESCRIBES -> Embedding
  Embedding -> IDENTIFIED_BY -> EmbeddingIdentifier
  Embedding -> DERIVED_FROM -> SourceResource
  Embedding -> GENERATED_BY -> EmbeddingModel
  EmbeddingModel -> HAS_VERSION -> ModelVersion
  Embedding -> HAS_DIMENSION -> VectorDimension
  Embedding -> MAY_HAVE_NORMALISATION -> VectorNormalisation
  Embedding -> MAY_USE_METRIC -> DistanceMetric
  Embedding -> MAY_BE_STORED_IN -> VectorIndex
  Embedding -> MAY_REFERENCE -> SourceFragment
  EmbeddingMetadata -> MAY_REFERENCE -> EmbeddingProvenance

# ---------- EMBEDDING IDENTITY ----------
rule embedding_identity:
  Embedding SHOULD have stable EmbeddingIdentifier
  where reproducibility, indexing, or auditing is required

rule vector_not_identity:
  vector values alone SHOULD NOT be assumed universal persistent identity

rule regenerated_embedding:
  regenerated embedding MAY differ
  when model, version, preprocessing, or parameters change

# ---------- EMBEDDING TYPES ----------
embedding_types:
  - text
  - sentence
  - paragraph
  - document
  - query
  - entity
  - concept
  - image
  - audio
  - video
  - graph
  - multimodal
  - code
  - structured_data

rule embedding_type:
  EmbeddingType SHOULD preserve source and intended representation semantics

# ---------- SOURCE RESOURCE ----------
entity EmbeddingSource:
  type: InformationResource
  may_include:
    - resource_identifier
    - resource_version
    - content_hash
    - fragment_identifier
    - source_text_or_reference

rule source:
  Embedding SHOULD preserve source resource identity
  and version where available

rule source_change:
  material source change SHOULD trigger embedding validity review

# ---------- SOURCE FRAGMENT ----------
entity SourceFragment:
  type: ResourceFragment
  may_include:
    - fragment_identifier
    - parent_resource
    - start_offset
    - end_offset
    - section
    - chunk_index

rule fragment:
  fragment embedding SHOULD preserve parent-resource relationship

rule fragment_not_document:
  SourceFragment != complete source resource

# ---------- EMBEDDING MODEL ----------
entity EmbeddingModel:
  type: AIModel
  may_include:
    - model_identifier
    - model_name
    - provider
    - architecture
    - model_version
    - output_dimension
    - supported_modalities

rule model:
  Embedding SHOULD preserve generating model identity

rule model_version:
  model version SHOULD be explicit
  where embedding compatibility depends on it

# ---------- VECTOR DIMENSION ----------
entity VectorDimension:
  type: NumericMetadata
  range: positive_integer

rule dimension:
  vector dimension MUST be positive

rule dimension_match:
  stored vector length MUST match declared VectorDimension

# ---------- VECTOR DATATYPE ----------
vector_datatypes:
  - float64
  - float32
  - float16
  - bfloat16
  - int8
  - binary
  - implementation_specific

rule datatype:
  vector datatype SHOULD be preserved
  where precision or storage affects retrieval behaviour

# ---------- VECTOR SPACE ----------
entity EmbeddingSpace:
  type: RepresentationSpace
  may_include:
    - model
    - model_version
    - dimension
    - normalisation
    - training_objective
    - modality

rule space:
  embeddings MUST be treated as directly comparable
  only when representation spaces are compatible

rule same_dimension:
  same vector dimension != compatible embedding space

# ---------- NORMALISATION ----------
vector_normalisation:
  - none
  - L1
  - L2
  - unit_length
  - model_specific

rule normalisation:
  normalisation method SHOULD be preserved
  where similarity semantics depend on it

# ---------- DISTANCE / SIMILARITY ----------
distance_metrics:
  - cosine_similarity
  - cosine_distance
  - dot_product
  - euclidean
  - manhattan
  - hamming
  - model_specific

entity DistanceMetric:
  type: SimilarityMetadata

rule metric:
  metric MUST be explicit
  when interpreting vector scores

rule metric_direction:
  higher score MAY mean more similar for some metrics
  while lower distance MAY mean more similar for others

rule metric_comparison:
  scores from different metrics MUST NOT be assumed directly comparable

# ---------- COSINE ----------
rule cosine:
  cosine similarity measures vector orientation
  and != semantic equivalence

rule cosine_normalisation:
  dot product on unit-normalised vectors MAY correspond to cosine similarity
  according_to implementation semantics

# ---------- DOT PRODUCT ----------
rule dot_product:
  dot product magnitude MAY depend on vector norms
  unless normalisation constrains them

# ---------- EUCLIDEAN ----------
rule euclidean:
  lower Euclidean distance generally indicates closer vectors
  within same compatible embedding space

# ---------- TOKENISATION ----------
entity EmbeddingTokenisation:
  type: ProcessingMetadata
  may_include:
    - tokenizer
    - tokenizer_version
    - token_count
    - truncation
    - max_tokens

rule tokenisation:
  tokenisation SHOULD be preserved
  where truncation or model input changes representation

rule truncation:
  truncated source embedding SHOULD preserve truncation state

# ---------- POOLING ----------
pooling_methods:
  - CLS
  - mean
  - max
  - weighted_mean
  - last_token
  - model_native
  - none

rule pooling:
  pooling method SHOULD be preserved
  where model exposes or requires explicit pooling

# ---------- CHUNKING ----------
entity EmbeddingChunking:
  type: ProcessingMetadata
  may_include:
    - chunk_size
    - chunk_overlap
    - boundary_method
    - chunk_count

rule chunking:
  chunking configuration SHOULD be preserved
  for document-level retrieval reproducibility

rule overlap:
  overlapping chunks MAY produce semantically related
  but distinct Embeddings

# ---------- QUERY EMBEDDING ----------
entity QueryEmbedding:
  subtype_of: Embedding
  derived_from: SearchQuery

rule query_embedding:
  QueryEmbedding SHOULD be generated in compatible space
  with target VectorIndex

rule query_text:
  QueryEmbedding != original QueryText

# ---------- DOCUMENT EMBEDDING ----------
entity DocumentEmbedding:
  subtype_of: Embedding
  derived_from: SearchableResource

rule document_embedding:
  DocumentEmbedding SHOULD preserve resource version

# ---------- ENTITY EMBEDDING ----------
entity EntityEmbedding:
  subtype_of: Embedding
  derived_from: Entity

rule entity_embedding:
  entity identity SHOULD remain separately preserved
  from its vector representation

# ---------- MULTIMODAL EMBEDDING ----------
entity MultimodalEmbedding:
  subtype_of: Embedding
  may_represent:
    - text
    - image
    - audio
    - video
    - mixed_modalities

rule multimodal:
  cross-modal comparison SHOULD occur only
  where model defines shared embedding space

# ---------- VECTOR INDEX ----------
entity EmbeddingIndexMetadata:
  type: VectorIndexMetadata
  may_include:
    - index_identifier
    - embedding_model
    - model_version
    - dimension
    - metric
    - ANN_method
    - quantisation

rule index:
  VectorIndex configuration SHOULD be compatible
  with stored Embeddings

# ---------- QUANTISATION ----------
quantisation_methods:
  - none
  - scalar
  - product
  - binary
  - implementation_specific

rule quantisation:
  quantisation MAY alter retrieval precision
  and SHOULD be preserved in index metadata

# ---------- SIMILARITY RESULT ----------
entity EmbeddingSimilarity:
  type: SimilarityMetadata
  may_include:
    - embedding_a
    - embedding_b
    - metric
    - score
    - model_space

rule similarity:
  similarity score SHOULD preserve compared embedding identities
  and metric

rule similarity_not_equivalence:
  high similarity != semantic equivalence

rule similarity_not_truth:
  high similarity != factual correctness

# ---------- CLUSTERING ----------
entity EmbeddingCluster:
  type: DerivedSemanticStructure
  may_include:
    - cluster_identifier
    - members
    - clustering_method
    - centroid
    - parameters

rule cluster:
  cluster membership indicates algorithmic grouping
  NOT necessarily ontology class membership

# ---------- DRIFT ----------
entity EmbeddingDrift:
  type: ChangeMetadata
  may_include:
    - source_change
    - model_change
    - model_version_change
    - distribution_change
    - measured_difference

rule drift:
  embedding drift SHOULD preserve cause and comparison method

# ---------- VALIDITY ----------
entity EmbeddingValidity:
  type: ValidityMetadata
  may_include:
    - valid_from
    - valid_until
    - source_version
    - model_version
    - status

embedding_status:
  - current
  - stale
  - superseded
  - incompatible
  - unknown

rule validity:
  Embedding MAY become stale
  when source or model semantics change

# ---------- CONFIDENCE ----------
entity EmbeddingConfidence:
  type: EpistemicMetadata
  range: [0.0, 1.0]

rule confidence_bounds:
  REQUIRE 0.0 <= confidence <= 1.0

rule confidence:
  confidence SHOULD qualify inferred interpretation
  NOT raw vector coordinates

# ---------- PROVENANCE ----------
entity EmbeddingProvenance:
  type: ProvenanceMetadata
  may_include:
    - source_resource
    - source_version
    - source_fragment
    - model
    - model_version
    - tokenizer
    - preprocessing
    - chunking
    - pooling
    - normalisation
    - creation_time

rule provenance:
  EmbeddingMetadata SHOULD preserve generation context

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_embedding
    - source_or_semantic_target
    - embedding_model
    - vector_dimension

  validate_if_present:
    - model_version
    - vector_datatype
    - normalisation
    - metric
    - tokenisation
    - chunking
    - pooling
    - validity
    - confidence
    - provenance

  reject:
    - non_positive_vector_dimension
    - vector_length_mismatch
    - incompatible_embedding_spaces_compared_directly
    - metric_score_interpreted_without metric_semantics
    - similarity_presented_as semantic_equivalence
    - similarity_presented_as factual_correctness
    - fragment_embedding_presented_as complete_document_embedding
    - stale_embedding_presented_as guaranteed_current_without validation

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "embedding metadata" ~= "metadata about vector embeddings"
  "vector embedding" ~= "dense or encoded vector representation"
  "query embedding" ~= "vector representation of a search query"
  "document embedding" ~= "vector representation of a document"
  "embedding space" ~= "vector representation space"
  "embedding similarity" ~= "similarity measured between embeddings"

  PRESERVE distinctions:
    embedding != source_resource
    embedding != semantic_identity
    vector_values != persistent_identity
    same_dimension != compatible_space
    similarity != semantic_equivalence
    similarity != factual_correctness
    query_embedding != query_text
    entity_embedding != entity_identity
    fragment_embedding != full_document_embedding
    cluster_membership != ontology_class_membership
    stale_embedding != current_embedding

# ---------- CONFIDENCE MODEL ----------
CONFIDENCE:
  explicit_embedding_metadata:
    score: 1.00

  explicit_embedding_identifier:
    score: 1.00

  model_version_and_source_preserved:
    score: 0.99

  compatible_embedding_space:
    score: 0.99

  validated_vector_dimension:
    score: 0.99

  source_fragment_traceable:
    score: 0.98

  model_version_unknown:
    score: <= 0.75

  source_version_unknown:
    score: <= 0.75

  embedding_space_ambiguous:
    score: <= 0.50
    action: do_not_compare_directly

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_embedding_creation:
    - assign EmbeddingIdentifier
    - preserve source resource and version
    - record model and model version
    - record dimension and datatype
    - record preprocessing
    - initialise validity and provenance

  on_source_change:
    - preserve previous Embedding
    - mark validity for review
    - regenerate embedding where required
    - preserve source-version relationship
    - update provenance

  on_model_change:
    - preserve prior EmbeddingSpace
    - create new Embedding with new model version
    - prevent unsupported direct comparison
    - update provenance

  on_index_insertion:
    - validate dimension
    - validate embedding-space compatibility
    - record index identifier
    - record metric and quantisation
    - update provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - embedding_entities
    - relationships
    - source_rules
    - model_rules
    - vector_space_rules
    - metric_rules
    - processing_rules
    - validation
    - semantic_equivalence
    - confidence_model
    - update_model

  canonicalisation:
    - preserve_embedding_identifiers
    - preserve_source_identifiers_and_versions
    - preserve_model_and_version
    - preserve_vector_dimension
    - preserve_metric_semantics
    - preserve_processing_order
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - semantic search metadata ontology
    - vector embedding lifecycle model
    - embedding-space compatibility structure
    - semantic similarity and vector-index metadata model

  operations:
    - identify embeddings
    - resolve source resources and fragments
    - resolve embedding model and version
    - validate dimensions and datatypes
    - inspect tokenisation and chunking
    - interpret pooling and normalisation
    - validate embedding-space compatibility
    - calculate metric-aware similarity
    - manage query and document embeddings
    - inspect multimodal embeddings
    - detect stale or superseded embeddings
    - trace generation provenance
    - preserve uncertainty

  constraints:
    - do_not_equate_embedding_with source_resource
    - do_not_equate_embedding_with semantic_identity
    - do_not_compare_incompatible_embedding_spaces
    - do_not_equate_same_dimension_with compatible_space
    - do_not_equate_similarity_with semantic_equivalence
    - do_not_equate_similarity_with factual_correctness
    - preserve_source_model_version_dimension_metric_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - embedding_identifier
    - source_resource
    - source_fragment
    - embedding_type
    - model_identifier
    - model_version
    - dimension
    - metric
    - validity_status

  resolve:
    1. identify Embedding
    2. resolve EmbeddingIdentifier
    3. resolve source resource and version
    4. resolve model and model version
    5. validate vector dimension and datatype
    6. inspect preprocessing, chunking, and pooling
    7. resolve normalisation and metric
    8. evaluate EmbeddingSpace compatibility
    9. inspect validity and provenance
    10. return confidence

  output:
    - embedding
    - identifier
    - embedding_type
    - source_resource
    - source_fragment
    - model
    - model_version
    - dimension
    - datatype
    - normalisation
    - metric
    - validity
    - provenance
    - confidence

# ---------- INFERENCE ----------
inference:
  IF two embeddings share model, compatible version, dimension, and space semantics
  THEN direct metric comparison MAY be permitted

  IF query embedding and vector index are compatible
  THEN semantic nearest-neighbour retrieval MAY be performed

  IF source resource changes materially
  THEN mark previous Embedding potentially_stale

  IF model version changes
  THEN create distinct EmbeddingSpace candidate
  unless compatibility is explicitly established

  IF embedding compatibility remains ambiguous
  THEN do_not compare directly
  AND preserve uncertainty

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: EmbeddingMetadata
  embedding_identity: machine_interpretable
  source_relationship: traceable
  model_and_version: explicit
  vector_dimension_and_datatype: validated
  embedding_space: compatibility_aware
  normalisation_and_metric: explicit
  query_document_entity_embeddings: distinguishable
  multimodal_embeddings: supported
  similarity: metric_aware
  validity_and_drift: reportable
  vector_index_compatibility: enforceable
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow