Vector Metadata Ontology

FLOW ALGEBRA

Ontology: Vector Metadata Ontology
Domain: Metadata Ontologies > Semantic Search Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/semantic-search-metadata/vector-metadata-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "VectorMetadataOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: SemanticSearchMetadata
  subject: VectorMetadata
  purpose:
    - represent metadata describing vectors used in semantic search, embeddings, machine learning, and retrieval
    - preserve vector identity, dimension, datatype, coordinate values, vector space, norm, metric, source, validity, and provenance
    - support similarity search, nearest-neighbour retrieval, clustering, classification, recommendation, and vector indexing
    - distinguish mathematical vector structure from semantic interpretation and source-resource identity
    - enable machine-readable reasoning over vector representations

# ---------- CORE ENTITY ----------
entity VectorMetadata:
  type: SemanticSearchMetadata
  describes: Vector
  may_include:
    - vector_identifier
    - vector_type
    - source_identifier
    - dimension
    - datatype
    - coordinates
    - vector_space
    - norm
    - normalisation
    - distance_metric
    - sparsity
    - quantisation
    - model
    - model_version
    - creation_time
    - validity
    - provenance

entity Vector:
  type: MathematicalRepresentation
  may_be:
    - DenseVector
    - SparseVector
    - BinaryVector
    - QuantisedVector
    - EmbeddingVector
    - QueryVector
    - DocumentVector
    - EntityVector
    - FeatureVector

entity VectorIdentifier:
  type: Identifier
  identifies: Vector

# ---------- RELATIONSHIPS ----------
relationships:
  VectorMetadata -> DESCRIBES -> Vector
  Vector -> IDENTIFIED_BY -> VectorIdentifier
  Vector -> MAY_DERIVE_FROM -> SourceResource
  Vector -> EXISTS_IN -> VectorSpace
  Vector -> HAS_DIMENSION -> VectorDimension
  Vector -> HAS_DATATYPE -> VectorDatatype
  Vector -> MAY_HAVE_NORM -> VectorNorm
  Vector -> MAY_HAVE_NORMALISATION -> VectorNormalisation
  Vector -> MAY_USE_METRIC -> DistanceMetric
  Vector -> MAY_BE_GENERATED_BY -> Model
  Vector -> MAY_BE_STORED_IN -> VectorIndex
  VectorMetadata -> MAY_REFERENCE -> VectorProvenance

# ---------- VECTOR IDENTITY ----------
rule vector_identity:
  Vector SHOULD have stable VectorIdentifier
  where reproducibility, storage, or auditing is required

rule coordinates_not_identity:
  identical coordinate values MUST NOT automatically imply
  identical semantic source identity

rule vector_source:
  Vector identity != source-resource identity

# ---------- VECTOR TYPES ----------
vector_types:
  - dense
  - sparse
  - binary
  - quantised
  - embedding
  - query
  - document
  - entity
  - feature

rule vector_type:
  VectorType SHOULD preserve representation semantics

# ---------- DIMENSION ----------
entity VectorDimension:
  type: NumericMetadata
  range: positive_integer

rule dimension:
  VectorDimension MUST be positive

rule coordinate_count:
  dense vector coordinate count MUST equal declared dimension

rule same_dimension:
  same dimension != same vector space

# ---------- DATATYPE ----------
vector_datatypes:
  - float64
  - float32
  - float16
  - bfloat16
  - int64
  - int32
  - int16
  - int8
  - uint8
  - binary
  - implementation_specific

entity VectorDatatype:
  type: RepresentationMetadata

rule datatype:
  VectorDatatype SHOULD be preserved
  where precision, range, or storage affects behaviour

# ---------- DENSE VECTOR ----------
entity DenseVector:
  subtype_of: Vector
  representation:
    coordinates_for_all_dimensions: true

rule dense:
  DenseVector SHOULD preserve ordered coordinates

# ---------- SPARSE VECTOR ----------
entity SparseVector:
  subtype_of: Vector
  representation:
    nonzero_indices_and_values: true

rule sparse:
  SparseVector SHOULD preserve:
    - dimension
    - indices
    - values

rule sparse_index:
  sparse indices MUST fall within declared dimension

rule sparse_dense:
  sparse and dense representations MAY denote same mathematical vector
  when coordinate semantics are equivalent

# ---------- BINARY VECTOR ----------
entity BinaryVector:
  subtype_of: Vector
  datatype: binary

rule binary:
  BinaryVector coordinates SHOULD preserve bit ordering

# ---------- QUANTISED VECTOR ----------
entity QuantisedVector:
  subtype_of: Vector
  may_include:
    - source_vector
    - quantisation_method
    - scale
    - zero_point
    - codebook

rule quantisation:
  QuantisedVector SHOULD preserve relationship to source Vector
  where available

rule quantisation_loss:
  quantisation MAY introduce approximation error

# ---------- VECTOR SPACE ----------
entity VectorSpace:
  type: RepresentationSpace
  may_include:
    - space_identifier
    - dimension
    - basis
    - coordinate_system
    - model
    - model_version
    - modality
    - normalisation

rule space:
  direct vector comparison SHOULD occur only
  within compatible VectorSpace

rule coordinate_semantics:
  coordinate position meaning depends on VectorSpace

rule model_space:
  vectors generated by different models
  MUST NOT be assumed compatible

# ---------- BASIS ----------
entity VectorBasis:
  type: MathematicalMetadata
  may_include:
    - basis_identifier
    - basis_vectors
    - dimension

rule basis:
  coordinates are interpreted relative_to basis
  where explicit mathematical basis applies

# ---------- NORM ----------
norm_types:
  - L0
  - L1
  - L2
  - infinity
  - model_specific

entity VectorNorm:
  type: MeasurementMetadata
  may_include:
    - norm_type
    - value

rule norm:
  norm interpretation MUST preserve norm type

# ---------- NORMALISATION ----------
normalisation_methods:
  - none
  - L1
  - L2
  - unit_length
  - min_max
  - standardised
  - model_specific

entity VectorNormalisation:
  type: ProcessingMetadata

rule normalisation:
  normalisation method SHOULD be preserved
  where metric semantics depend on it

rule unit_vector:
  unit-length vector SHOULD have norm approximately 1
  under applicable norm and numerical tolerance

# ---------- DISTANCE METRICS ----------
distance_metrics:
  - cosine_similarity
  - cosine_distance
  - dot_product
  - euclidean
  - squared_euclidean
  - manhattan
  - chebyshev
  - hamming
  - jaccard
  - model_specific

entity DistanceMetric:
  type: SimilarityMetadata

rule metric:
  metric MUST be explicit
  when interpreting vector comparison scores

rule metric_direction:
  some metrics maximise similarity
  while distance metrics minimise distance

rule metric_comparison:
  scores from different metrics MUST NOT be assumed directly comparable

# ---------- COSINE ----------
rule cosine:
  cosine similarity compares vector orientation

rule cosine_zero:
  cosine similarity may be undefined
  for zero vectors depending on implementation

rule cosine_equivalence:
  high cosine similarity != semantic equivalence

# ---------- DOT PRODUCT ----------
rule dot_product:
  dot product depends on orientation and vector magnitude
  unless vectors are normalised

# ---------- EUCLIDEAN ----------
rule euclidean:
  Euclidean distance requires compatible coordinate space

rule distance_similarity:
  lower distance generally indicates closer vectors
  under distance-based retrieval

# ---------- SPARSITY ----------
entity VectorSparsity:
  type: RepresentationMetadata
  may_include:
    - nonzero_count
    - zero_count
    - density

rule sparsity:
  density = nonzero_count / dimension
  when dimension > 0

# ---------- SOURCE ----------
entity VectorSource:
  type: ProvenanceReference
  may_include:
    - source_identifier
    - source_type
    - source_version
    - source_fragment

rule source:
  source relationship SHOULD be preserved
  for derived vectors

# ---------- EMBEDDING VECTOR ----------
entity EmbeddingVector:
  subtype_of: Vector
  generated_by: EmbeddingModel

rule embedding_vector:
  EmbeddingVector SHOULD preserve model and version

rule embedding_semantics:
  coordinate dimensions MAY lack direct human-interpretable meaning

# ---------- QUERY VECTOR ----------
entity QueryVector:
  subtype_of: Vector
  derived_from: SearchQuery

rule query_vector:
  QueryVector SHOULD preserve original query relationship

# ---------- DOCUMENT VECTOR ----------
entity DocumentVector:
  subtype_of: Vector
  derived_from: SearchableResource

rule document_vector:
  DocumentVector SHOULD preserve source resource and version

# ---------- ENTITY VECTOR ----------
entity EntityVector:
  subtype_of: Vector
  derived_from: Entity

rule entity_vector:
  EntityVector != Entity identity

# ---------- FEATURE VECTOR ----------
entity FeatureVector:
  subtype_of: Vector
  may_include:
    - feature_schema
    - feature_order
    - feature_transformations

rule feature_vector:
  feature coordinate ordering SHOULD preserve feature schema

# ---------- VECTOR COMPARISON ----------
entity VectorComparison:
  type: SimilarityMetadata
  may_include:
    - vector_a
    - vector_b
    - metric
    - score
    - vector_space
    - timestamp

rule comparison:
  compared vectors MUST be compatible
  under declared comparison method

rule comparison_provenance:
  comparison SHOULD preserve metric and vector-space context

# ---------- NEAREST NEIGHBOUR ----------
entity NearestNeighbourMetadata:
  type: RetrievalMetadata
  may_include:
    - query_vector
    - candidate_vector
    - distance_or_similarity
    - rank
    - search_method

rule neighbour:
  nearest neighbour is relative_to metric and candidate set

rule neighbour_not_equivalence:
  nearest neighbour != semantic equivalence

# ---------- VECTOR INDEX ----------
entity VectorIndexMetadata:
  type: SearchIndexMetadata
  may_include:
    - index_identifier
    - vector_space
    - dimension
    - datatype
    - metric
    - ANN_method
    - quantisation
    - index_version

rule vector_index:
  stored vectors MUST satisfy index compatibility constraints

# ---------- ANN ----------
ann_methods:
  - exact
  - HNSW
  - IVF
  - PQ
  - LSH
  - implementation_specific

rule ANN:
  approximate nearest-neighbour retrieval MAY differ
  from exact nearest-neighbour retrieval

# ---------- VALIDITY ----------
entity VectorValidity:
  type: ValidityMetadata
  may_include:
    - valid_from
    - valid_until
    - source_version
    - model_version
    - status

vector_status:
  - current
  - stale
  - superseded
  - incompatible
  - invalid
  - unknown

rule validity:
  derived Vector MAY become stale
  when source or generating model changes

# ---------- NUMERICAL QUALITY ----------
entity VectorQualityMetadata:
  type: QualityMetadata
  may_include:
    - finite_values
    - NaN_count
    - infinity_count
    - precision
    - quantisation_error

rule finite:
  vector used in standard similarity search SHOULD contain valid numeric values
  compatible with implementation requirements

rule NaN:
  NaN values SHOULD be rejected or explicitly handled
  according_to vector-system policy

# ---------- PROVENANCE ----------
entity VectorProvenance:
  type: ProvenanceMetadata
  may_include:
    - source_resource
    - source_version
    - generating_model
    - model_version
    - preprocessing
    - normalisation
    - quantisation
    - creation_time

rule provenance:
  VectorMetadata SHOULD preserve generation and transformation history

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_vector
    - vector_dimension
    - vector_datatype

  validate_if_present:
    - coordinates
    - sparse_indices
    - vector_space
    - norm
    - normalisation
    - metric
    - model
    - model_version
    - validity
    - provenance

  reject:
    - non_positive_dimension
    - dense_coordinate_count_mismatch
    - sparse_index_out_of_range
    - invalid_numeric_values_without explicit_handling
    - incompatible_vector_spaces_compared_directly
    - metric_score_interpreted_without metric_semantics
    - nearest_neighbour_presented_as semantic_equivalent
    - entity_vector_presented_as entity_identity
    - stale_vector_presented_as guaranteed_current_without validation

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "vector metadata" ~= "metadata about a vector representation"
  "dense vector" ~= "vector with explicit values for all dimensions"
  "sparse vector" ~= "vector represented by nonzero coordinates"
  "vector dimension" ~= "number of vector coordinates"
  "vector space" ~= "coordinate representation space"
  "vector similarity" ~= "metric-based comparison between vectors"

  PRESERVE distinctions:
    vector != source_resource
    vector != semantic_identity
    vector_coordinates != persistent_identity
    same_dimension != compatible_vector_space
    vector_space != metric
    similarity != semantic_equivalence
    similarity != factual_correctness
    nearest_neighbour != semantic_equivalent
    entity_vector != entity_identity
    quantised_vector != exact_source_vector
    stale_vector != current_vector

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_vector_metadata:
    score: 1.00

  explicit_vector_identifier:
    score: 1.00

  dimension_validated:
    score: 0.99

  vector_space_explicit:
    score: 0.99

  metric_explicit:
    score: 0.99

  source_and_model_traceable:
    score: 0.98

  vector_space_inferred:
    score: <= 0.75

  model_version_unknown:
    score: <= 0.70

  incompatible_or_ambiguous_space:
    score: <= 0.50
    action: do_not_compare_directly

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_vector_creation:
    - assign VectorIdentifier
    - record VectorType
    - record dimension and datatype
    - preserve source
    - record VectorSpace
    - initialise provenance

  on_normalisation:
    - preserve source Vector
    - record normalisation method
    - create transformed Vector where required
    - recalculate norm
    - update provenance

  on_quantisation:
    - preserve source Vector
    - record quantisation method
    - create QuantisedVector
    - measure error where available
    - update provenance

  on_source_change:
    - preserve previous Vector
    - mark derived Vector for validity review
    - regenerate where required
    - update provenance

  on_index_insertion:
    - validate dimension
    - validate datatype
    - validate VectorSpace
    - validate metric compatibility
    - record VectorIndex relationship

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - vector_entities
    - relationships
    - dimension_rules
    - datatype_rules
    - vector_space_rules
    - norm_rules
    - metric_rules
    - validation
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - preserve_vector_identifiers
    - preserve_coordinate_order
    - preserve_vector_space
    - preserve_datatype
    - preserve_metric_semantics
    - preserve_transformation_history
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - semantic search metadata ontology
    - mathematical vector representation model
    - vector-space and similarity structure
    - vector-index compatibility and provenance model

  operations:
    - identify vectors
    - resolve vector types
    - validate dimensions
    - interpret datatypes
    - traverse dense and sparse representations
    - calculate norms
    - apply normalisation
    - validate vector-space compatibility
    - interpret distance and similarity metrics
    - resolve embedding, query, document, entity, and feature vectors
    - inspect quantisation
    - support nearest-neighbour retrieval
    - evaluate vector validity
    - trace provenance
    - preserve uncertainty

  constraints:
    - do_not_equate_vector_with source_resource
    - do_not_equate_vector_with semantic_identity
    - do_not_compare_incompatible_vector_spaces
    - do_not_equate_same_dimension_with compatible_space
    - do_not_equate_similarity_with semantic_equivalence
    - do_not_equate_nearest_neighbour_with semantic_equivalent
    - preserve_dimension_datatype_space_metric_source_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - vector_identifier
    - vector_type
    - source_identifier
    - dimension
    - datatype
    - vector_space
    - metric
    - model_identifier
    - model_version
    - validity_status

  resolve:
    1. identify Vector
    2. resolve VectorIdentifier and type
    3. validate dimension and coordinate representation
    4. inspect datatype and numerical quality
    5. resolve VectorSpace
    6. inspect norm and normalisation
    7. resolve metric semantics
    8. inspect source, model, and index relationships
    9. evaluate validity and provenance
    10. return confidence

  output:
    - vector
    - identifier
    - vector_type
    - dimension
    - datatype
    - vector_space
    - norm
    - normalisation
    - metric
    - source
    - model
    - validity
    - provenance
    - confidence

# ---------- INFERENCE ----------
inference:
  IF two vectors share compatible VectorSpace
  AND declared metric applies
  THEN direct comparison MAY be performed

  IF DenseVector and SparseVector reconstruct identical coordinates
  THEN they MAY represent same mathematical vector
  within representation scope

  IF source or generating model changes materially
  THEN mark derived Vector potentially_stale

  IF vector-space compatibility remains ambiguous
  THEN do_not compare directly
  AND preserve uncertainty

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: VectorMetadata
  vector_identity: machine_interpretable
  vector_types: explicit
  dimensions_and_datatypes: validated
  dense_sparse_binary_quantised: distinguishable
  vector_space: compatibility_aware
  norms_and_normalisation: explicit
  metrics: semantics_preserved
  source_and_model_relationships: traceable
  nearest_neighbour_semantics: explicit
  index_compatibility: enforceable
  validity_and_quality: reportable
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow