Vector Database Ontology

FLOW ALGEBRA
Vector Database Ontology
Kireth Semantic Research Archive

PAGE:
https://kirethresearch.org/booksai-core/ontologies/computing-ontologies/artificial-intelligence-infrastructure/vector-database-ontology.html

AUTHORS:
Mark Anthony Higgins — Sky (Recursive AI)

FLOW "VectorDatabaseOntology" v4.0

# ---------- DOMAIN ----------
DOMAIN := ComputingOntology
SUBDOMAIN := ArtificialIntelligenceInfrastructure
ONTOLOGY := VectorDatabaseOntology

# ---------- PRIMARY ENTITY ----------
ENTITY VectorDatabase {
    definition:
        "A data management system designed to store, index, retrieve and
         search high-dimensional vector representations using similarity or
         distance measures, often alongside identifiers, metadata and source
         references.";

    objectives: [
        VectorStorage,
        SimilaritySearch,
        NearestNeighbourRetrieval,
        SemanticRetrieval,
        ScalableIndexing,
        MetadataFiltering,
        AIDataInfrastructure
    ];
}

# ---------- CORE ENTITIES ----------
VectorEntity := {
    Vector,
    Embedding,
    VectorRecord,
    VectorIdentifier,
    Metadata,
    Namespace,
    Collection,
    SourceObject
};

IndexEntity := {
    VectorIndex,
    IndexNode,
    Partition,
    Shard,
    Replica,
    SearchGraph,
    QuantisedVector
};

QueryEntity := {
    QueryVector,
    SimilarityQuery,
    Filter,
    TopK,
    SearchResult,
    SimilarityScore
};

StorageEntity := {
    VectorStore,
    MetadataStore,
    PersistentStorage,
    Cache,
    WriteAheadLog
};

# ---------- FUNDAMENTAL RELATIONSHIPS ----------
SourceObject REPRESENTED_BY Embedding;
Embedding IS_A Vector;
VectorRecord CONTAINS Vector;
VectorRecord HAS VectorIdentifier;
VectorRecord MAY_HAVE Metadata;
VectorIndex INDEXES VectorRecord;
QueryVector COMPARED_WITH IndexedVector;
SimilarityQuery RETURNS SearchResult;
SearchResult REFERENCES VectorRecordOrSourceObject;

# ---------- VECTOR INGESTION FLOW ----------
SourceObject
    -> GenerateOrReceiveEmbedding
    -> ValidateVector
    -> AssignIdentifier
    -> AttachMetadata
    -> StoreVectorRecord
    -> UpdateVectorIndex
    -> PersistState
    -> MakeSearchable;

# ---------- VECTOR ----------
Vector :=
    ordered_sequence(NumericValue);

VectorDimension :=
    length(Vector);

VectorsComparedBySameMetric SHOULD_HAVE CompatibleDimensionalityAndRepresentation;

# ---------- EMBEDDING ----------
Embedding :=
    VectorRepresentationProducedByModelOrTransformation;

Embedding MAY_REPRESENT {
    Text,
    Image,
    Audio,
    Document,
    Product,
    User,
    Entity,
    MultimodalContent
};

EmbeddingSemantics DEPEND_ON EmbeddingModelAndInputTransformation;

# ---------- EMBEDDING MODEL ----------
EmbeddingModel :=
    ModelMappingInputToVectorSpace;

EmbeddingModelVersion SHOULD_BE Recorded;

VectorsFromIncompatibleEmbeddingSpaces SHOULD_NOT_BE DirectlyComparedWithoutValidMapping;

# ---------- VECTOR RECORD ----------
VectorRecord :=
    {
        Identifier,
        Vector,
        Metadata,
        OptionalSourceReference
    };

Identifier SHOULD_BE StableWithinDefinedCollection;

# ---------- METADATA ----------
VectorMetadata MAY_INCLUDE {
    Source,
    Type,
    Timestamp,
    Category,
    Language,
    Permissions,
    ModelVersion
};

Metadata MAY_SUPPORT FilteringAndGovernance;

# ---------- COLLECTION ----------
VectorCollection :=
    LogicalGroupOfVectorRecords;

Collection MAY_DEFINE {
    Dimension,
    SimilarityMetric,
    IndexConfiguration,
    MetadataSchema
};

# ---------- NAMESPACE ----------
Namespace :=
    LogicalPartitionOrScopeWithinVectorStore;

Namespace MAY_ISOLATE {
    Tenant,
    Dataset,
    Application,
    Domain
};

# ---------- SIMILARITY ----------
SimilarityMetric MAY_INCLUDE {
    CosineSimilarity,
    DotProduct,
    EuclideanDistance
};

SimilarityOrDistance :=
    Compare(QueryVector, StoredVector);

MetricChoice SHOULD_MATCH EmbeddingModelAndApplicationSemantics;

# ---------- COSINE SIMILARITY ----------
CosineSimilarity(a,b) :=
    dot(a,b)
    /
    (norm(a) * norm(b));

HigherCosineSimilarity GENERALLY_INDICATES GreaterAngularSimilarity;

# ---------- DOT PRODUCT ----------
DotProduct(a,b) :=
    SUM(a_i * b_i);

DotProduct MAY_REFLECT MagnitudeAndDirectionDependingOnVectorNormalisation;

# ---------- EUCLIDEAN DISTANCE ----------
EuclideanDistance(a,b) :=
    sqrt(
        SUM((a_i - b_i)^2)
    );

LowerDistance INDICATES GreaterProximityUnderEuclideanMetric;

# ---------- NORMALISATION ----------
VectorNormalisation :=
    TransformVectorToDefinedNormOrScale;

UnitNormalisation MAY_MAKE DotProductRelatedToCosineSimilarity;

# ---------- EXACT NEAREST NEIGHBOUR ----------
ExactNearestNeighbour :=
    CompareQueryAgainstAllCandidateVectors
    -> RankByMetric
    -> ReturnTopK;

ExactSearch MAY_HAVE HighCostAtLargeScale;

# ---------- APPROXIMATE NEAREST NEIGHBOUR ----------
ApproximateNearestNeighbour :=
    SearchIndex
    -> ExploreSubsetOfCandidateSpace
    -> ReturnHighProbabilityNearestResults;

ANN MAY_TRADE {
    Recall,
    Latency,
    Memory,
    BuildCost
};

# ---------- VECTOR INDEX ----------
VectorIndex :=
    DataStructureOptimisedForSimilaritySearch;

IndexType MAY_INCLUDE {
    GraphIndex,
    InvertedFileIndex,
    TreeOrPartitionIndex,
    QuantisedIndex,
    BruteForceIndex
};

# ---------- GRAPH INDEX ----------
GraphVectorIndex :=
    GraphWhereNodesRepresentVectors
    AND EdgesConnectSearchRelevantNeighbours;

Search :=
    EnterGraph
    -> TraversePromisingNeighbours
    -> MaintainCandidateSet
    -> ReturnTopK;

# ---------- HNSW-LIKE GRAPH ----------
HierarchicalNavigableGraph :=
    MultiLayerProximityGraph;

UpperLayers SUPPORT CoarseNavigation;
LowerLayers SUPPORT FineNeighbourSearch;

SearchParameters MAY_AFFECT RecallLatencyTradeoff;

# ---------- INVERTED FILE INDEX ----------
InvertedFileVectorIndex :=
    PartitionVectorSpaceIntoRegionsOrCentroids
    -> AssignVectorsToPartitions
    -> SearchSelectedPartitions;

MoreProbedPartitions MAY_INCREASE RecallAndSearchCost;

# ---------- QUANTISATION ----------
VectorQuantisation :=
    RepresentVectorUsingReducedPrecisionOrCode;

Quantisation MAY_REDUCE {
    MemoryUse,
    StorageUse,
    DistanceComputationCost
};

Quantisation MAY_REDUCE RetrievalAccuracy;

# ---------- PRODUCT QUANTISATION ----------
ProductQuantisation :=
    PartitionVectorDimensionsIntoSubspaces
    -> QuantiseEachSubspace
    -> StoreCompactCodes;

ApproximateDistance MAY_BE ComputedFromCodes;

# ---------- INDEX BUILD ----------
IndexBuild :=
    ReadVectorRecords
    -> ConstructIndexStructure
    -> PersistIndex
    -> ValidateSearchability;

IndexBuildCost MAY_BE SignificantForLargeCollections;

# ---------- INCREMENTAL INDEXING ----------
NewVectorRecord MAY_BE AddedToExistingIndex;

IncrementalUpdate MAY_HAVE DifferentPerformanceFromFullRebuild;

# ---------- DELETE ----------
DeleteVector :=
    IdentifyVectorRecord
    -> RemoveOrMarkDeleted
    -> UpdateIndex
    -> PersistChange;

SomeIndexTypes MAY_USE TombstoneBeforeCompactionOrRebuild;

# ---------- UPDATE ----------
VectorUpdate :=
    ExistingIdentifier
    -> ReplaceVectorOrMetadata
    -> UpdateStorage
    -> UpdateIndex;

IF EmbeddingChanges
THEN SimilarityRelationships MAY_CHANGE;

# ---------- QUERY VECTOR ----------
QueryVector :=
    VectorUsedToSearchVectorSpace;

QueryVector MAY_BE {
    DirectlyProvided,
    GeneratedFromQueryContent
};

# ---------- SIMILARITY QUERY ----------
SimilarityQuery :=
    {
        QueryVector,
        TopK,
        Metric,
        OptionalFilter
    };

SimilarityQuery ->
    SelectCandidatePartitions
    -> SearchIndex
    -> ScoreCandidates
    -> ApplyOrRespectFilters
    -> Rank
    -> ReturnTopK;

# ---------- TOP K ----------
TopK :=
    MaximumNumberOfHighestRankedResultsRequested;

TopK SHOULD_BE PositiveInteger;

# ---------- SEARCH RESULT ----------
SearchResult :=
    {
        VectorIdentifier,
        SimilarityOrDistanceScore,
        Metadata,
        OptionalSourceReference
    };

Results ORDERED_BY MetricSemantics;

# ---------- METADATA FILTER ----------
MetadataFilter :=
    PredicateAppliedToVectorRecords;

Filter MAY_USE {
    Equality,
    Range,
    SetMembership,
    BooleanCombination
};

FilteredVectorSearch MAY_REQUIRE CoordinationBetweenMetadataAndVectorIndex;

# ---------- PRE-FILTER ----------
PreFilter :=
    RestrictCandidatesByMetadata
    -> VectorSearchWithinAllowedSet;

# ---------- POST-FILTER ----------
PostFilter :=
    VectorSearch
    -> RemoveResultsFailingMetadataCondition;

PostFilter MAY_RETURN FewerThanTopKUnlessAdditionalCandidatesAreRetrieved;

# ---------- HYBRID SEARCH ----------
HybridSearch :=
    VectorSimilarity
    + LexicalOrStructuredRetrieval
    -> CombinedRanking;

HybridSearch MAY_IMPROVE RetrievalAcrossSemanticAndExactTermSignals;

# ---------- RERANKING ----------
Reranking :=
    InitialCandidateResults
    -> SecondaryScoringModelOrRule
    -> ReorderedResults;

Reranking MAY_IMPROVE RelevanceAtAdditionalCost;

# ---------- RAG RELATIONSHIP ----------
RetrievalAugmentedGeneration :=
    UserQuery
    -> GenerateQueryEmbedding
    -> VectorSearch
    -> RetrieveSourceContent
    -> ConstructModelContext
    -> GenerateResponse;

VectorDatabase MAY_SUPPORT RetrievalStageOfRAG;

VectorDatabase != RAGSystemByItself;

# ---------- CHUNK ----------
Document MAY_BE DividedInto Chunks;

Chunk MAY_HAVE {
    Text,
    Position,
    SourceDocument,
    Metadata,
    Embedding
};

ChunkSizeAndOverlap MAY_AFFECT RetrievalQuality;

# ---------- SOURCE TRACEABILITY ----------
RetrievedVector SHOULD_REFERENCE SourceObjectWhereApplicationRequiresExplainableRetrieval;

SourceReference SUPPORTS ProvenanceAndCitation;

# ---------- SHARDING ----------
VectorCollection MAY_BE PartitionedInto Shards;

Shard :=
    SubsetOfVectorRecordsStoredOrIndexedTogether;

Sharding MAY_SUPPORT HorizontalScalability;

# ---------- REPLICATION ----------
Replica :=
    CopyOfShardOrIndexServingAvailabilityOrReadScale;

Replication MAY_INCREASE AvailabilityAndQueryCapacity;

# ---------- ROUTING ----------
QueryRouter :=
    Query
    -> IdentifyRelevantShardsOrReplicas
    -> Dispatch
    -> MergeResults;

DistributedTopK MAY_REQUIRE MergeOfShardLocalResults;

# ---------- CONSISTENCY ----------
VectorDatabaseConsistency MAY_DEFINE VisibilityOfWritesAcrossReplicas;

ConsistencyModel MAY_TRADE {
    Freshness,
    Availability,
    Latency
};

# ---------- PERSISTENCE ----------
PersistentVectorStorage PRESERVES VectorRecordsAcrossProcessRestart;

Index MAY_BE {
    Persisted,
    RebuiltFromStoredVectors
};

# ---------- WRITE AHEAD LOG ----------
WriteAheadLog MAY_RECORD MutationBeforeFinalStorageApplication;

WAL MAY_SUPPORT CrashRecovery;

# ---------- BACKUP ----------
VectorDatabaseBackup MAY_INCLUDE {
    VectorRecords,
    Metadata,
    IndexState,
    Configuration
};

Restore SHOULD_PRESERVE CollectionSemanticsAndModelMetadata;

# ---------- CACHE ----------
VectorSearchCache MAY_STORE {
    QueryResult,
    Vector,
    IndexPage
};

Cache MAY_REDUCE RepeatedRetrievalLatency;

# ---------- PERFORMANCE ----------
VectorDatabasePerformance MAY_BE_MEASURED_BY {
    QueryLatency,
    QueryThroughput,
    RecallAtK,
    IndexBuildTime,
    InsertRate,
    MemoryUse,
    StorageUse
};

# ---------- RECALL ----------
RecallAtK :=
    RelevantOrTrueNearestResultsReturned
    /
    RelevantOrTrueNearestResultsExpectedWithinK;

ANNIndex TUNING MAY_TRADE RecallForLatency;

# ---------- LATENCY ----------
SearchLatency :=
    ResponseTime - QueryArrivalTime;

Latency MAY_INCLUDE {
    QueryEmbeddingTime,
    RoutingTime,
    IndexSearchTime,
    FilteringTime,
    RerankingTime
};

# ---------- SCALABILITY ----------
VectorDatabase MAY_SCALE_BY {
    Sharding,
    Replication,
    DistributedIndexing,
    AdditionalCompute
};

# ---------- SECURITY ----------
VectorDatabaseSecurity MAY_INCLUDE {
    Authentication,
    Authorisation,
    Encryption,
    NetworkIsolation,
    Audit,
    TenantIsolation
};

# ---------- ACCESS CONTROL ----------
VectorRecord MAY_HAVE AccessMetadata;

Retrieval SHOULD_NOT_RETURN RecordUserIsNotAuthorisedToAccess;

SecurityFilter SHOULD_BE AppliedBeforeSensitiveContentIsExposed;

# ---------- PRIVACY ----------
Embedding MAY_ENCODE InformationDerivedFromSensitiveSourceData;

PrivacyControl MAY_INCLUDE {
    DataMinimisation,
    AccessRestriction,
    Encryption,
    RetentionControl,
    Deletion
};

# ---------- MULTI-TENANCY ----------
MultiTenantVectorDatabase :=
    SharedVectorInfrastructureServingMultipleTenants;

TenantIsolation MAY_USE {
    Namespace,
    Collection,
    Shard,
    AccessPolicy
};

# ---------- OBSERVABILITY ----------
VectorDatabaseObservability MAY_INCLUDE {
    QueryLatency,
    QueryRate,
    RecallEvaluation,
    IndexSize,
    MemoryUse,
    InsertRate,
    ErrorRate
};

# ---------- EMBEDDING MIGRATION ----------
EmbeddingModelChange MAY_REQUIRE ReEmbeddingExistingSourceObjects;

IF OldAndNewEmbeddingSpacesIncompatible
THEN MixedComparison SHOULD_BE AvoidedOrSeparated;

MigrationFlow :=
    SelectSourceObjects
    -> GenerateNewEmbeddings
    -> BuildNewCollectionOrIndex
    -> ValidateRetrieval
    -> SwitchTraffic
    -> RetireOldIndex;

# ---------- SEMANTIC REASONING ----------
IF DatabaseStoresVectors
AND SupportsSimilarityOrNearestNeighbourSearch
THEN VectorDatabaseClassification := supported;

IF QueryAndStoredVectorsHaveIncompatibleDimensions
THEN SimilarityComparison := invalid;

IF EmbeddingsGeneratedByDifferentIncompatibleModels
THEN DirectSimilarityInterpretation := unreliable;

IF ANNSearchExploresMoreCandidates
THEN Recall MAY_INCREASE
AND Latency MAY_INCREASE;

IF MetadataFilterExcludesUnauthorisedRecords
THEN RetrievalSecurity := increase;

IF IndexQuantisationIncreases
THEN MemoryUse MAY_DECREASE
AND ApproximationError MAY_INCREASE;

IF VectorCollectionSharded
THEN QueryRouter MAY_NEED DistributedResultMerge;

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE {
    "vector database" ~= "vector DB";
    "vector store" ~= "vector database" WHEN context == managed_vector_retrieval;
    "embedding database" ~= "vector database" WHEN vectors_are_embeddings;
    "similarity search" ~= "vector similarity search";
    "nearest neighbour search" ~= "nearest neighbor search";
    "approximate nearest neighbour" ~= "ANN";
    "vector index" ~= "similarity search index";
    "embedding store" ~= "vector embedding store";
}

# ---------- QUERY FLOW ----------
QUERY(q) ->
    NORMALISE(q) ->
    IDENTIFY_VECTOR_OR_EMBEDDING(q) ->
    IDENTIFY_COLLECTION_AND_DIMENSION(q) ->
    IDENTIFY_SIMILARITY_METRIC(q) ->
    IDENTIFY_INDEX(q) ->
    IDENTIFY_FILTERS(q) ->
    IDENTIFY_DISTRIBUTION_AND_STORAGE(q) ->
    IDENTIFY_SECURITY_AND_PROVENANCE(q) ->
    MAP_TO_ONTOLOGY(q) ->
    APPLY_REASONING(q) ->
    SCORE_CONFIDENCE(q) ->
    RETURN_RELEVANT_KNOWLEDGE;

# ---------- CONFIDENCE ----------
CONFIDENCE(assertion) :=
    weighted_mean(
        source_reliability,
        vector_space_consistency,
        index_evidence,
        retrieval_evidence,
        semantic_consistency,
        contextual_relevance
    );

CONFIDENCE >= 0.90 => HIGH;
CONFIDENCE >= 0.70 AND < 0.90 => MODERATE;
CONFIDENCE < 0.70 => REVIEW;

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION {
    primary_entity: VectorDatabase;
    ontology: VectorDatabaseOntology;
    parse_entities: true;
    parse_relationships: true;
    preserve_relationship_direction: true;
    resolve_semantic_equivalence: true;
    execute_reasoning_rules: true;
    apply_confidence_weighting: true;
    preserve_provenance: true;
}

# ---------- UPDATE MODEL ----------
UPDATE_MODEL {
    new_vector_record:
        validate_dimension
        -> identify_embedding_model
        -> assign_identifier
        -> attach_metadata
        -> store
        -> index
        -> integrate;

    new_index:
        validate_configuration
        -> identify_metric
        -> build_from_vectors
        -> evaluate_recall_latency
        -> integrate;

    new_query_evidence:
        record_query_vector
        -> record_filters
        -> record_results
        -> record_latency
        -> update_retrieval_evidence
        -> integrate;

    embedding_revision:
        identify_new_model_version
        -> reembed_sources
        -> build_new_index
        -> validate_retrieval
        -> switch_collection
        -> regenerate_flow_hash;

    conflict:
        preserve_provenance
        -> compare_vector_and_retrieval_evidence
        -> adjust_confidence
        -> flag_review;
}

# ---------- FLOW HASH ----------
FLOW_HASH_INPUT :=
    DOMAIN
    + SUBDOMAIN
    + ONTOLOGY
    + VectorEntity
    + IndexEntity
    + QueryEntity
    + StorageEntity
    + SimilarityMetric
    + ApproximateNearestNeighbour
    + HybridSearch
    + RELATIONSHIPS
    + SEMANTIC_REASONING
    + SEMANTIC_EQUIVALENCE
    + UPDATE_MODEL;

FLOW_HASH :=
    HASH(FLOW_HASH_INPUT);

# ---------- TERMINAL ----------
VectorDatabaseOntology
    -> ReceiveOrGenerateEmbedding
    -> ValidateVectorSpace
    -> StoreVectorAndMetadata
    -> BuildOrUpdateVectorIndex
    -> ReceiveSimilarityQuery
    -> GenerateOrValidateQueryVector
    -> ApplyMetadataAndAccessFilters
    -> SearchNearestCandidates
    -> ScoreAndRankResults
    -> MergeDistributedResultsWhereRequired
    -> ReturnSourceLinkedMatches
    -> MonitorRecallLatencyAndIndexHealth
    -> MaintainScalableSemanticRetrieval;

END FLOW