Search Index Ontology

FLOW ALGEBRA

Ontology: Search Index Ontology
Domain: Metadata Ontologies > Semantic Search Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/semantic-search-metadata/search-index-ontology-2.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "SearchIndexOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: SemanticSearchMetadata
  subject: SearchIndexMetadata
  purpose:
    - represent metadata describing search indexes and indexed resources
    - preserve index identity, type, schema, fields, documents, terms, vectors, versions, update state, and provenance
    - support lexical, semantic, vector, graph, structured, and hybrid retrieval
    - distinguish indexed representations from source resources and ranking results
    - enable machine-readable reasoning over search-index lifecycles

# ---------- CORE ENTITY ----------
entity SearchIndexMetadata:
  type: SearchMetadata
  describes: SearchIndex
  may_include:
    - index_identifier
    - index_type
    - index_name
    - schema
    - indexed_fields
    - indexed_resources
    - term_dictionary
    - vector_configuration
    - semantic_annotations
    - ranking_configuration
    - shard_metadata
    - version
    - creation_time
    - update_time
    - freshness
    - status
    - provenance

entity SearchIndex:
  type: RetrievalIndex
  may_be:
    - LexicalIndex
    - SemanticIndex
    - VectorIndex
    - GraphIndex
    - StructuredIndex
    - HybridIndex

entity SearchIndexIdentifier:
  type: Identifier
  identifies: SearchIndex

# ---------- RELATIONSHIPS ----------
relationships:
  SearchIndexMetadata -> DESCRIBES -> SearchIndex
  SearchIndex -> IDENTIFIED_BY -> SearchIndexIdentifier
  SearchIndex -> HAS_TYPE -> SearchIndexType
  SearchIndex -> HAS_SCHEMA -> IndexSchema
  SearchIndex -> CONTAINS -> IndexedResource
  SearchIndex -> MAY_HAVE_FIELD -> IndexField
  SearchIndex -> MAY_HAVE_TERM_DICTIONARY -> TermDictionary
  SearchIndex -> MAY_HAVE_VECTOR_CONFIG -> VectorConfiguration
  SearchIndex -> MAY_HAVE_SHARD -> IndexShard
  SearchIndex -> MAY_HAVE_VERSION -> SearchIndexVersion
  SearchIndex -> USED_BY -> SearchProcess
  SearchIndexMetadata -> MAY_REFERENCE -> SearchIndexProvenance

# ---------- INDEX IDENTITY ----------
rule index_identity:
  SearchIndex SHOULD have stable SearchIndexIdentifier
  where reproducibility, updates, or auditing are required

rule index_name:
  IndexName != globally unique identity

rule index_version_identity:
  SearchIndex != SearchIndexVersion

# ---------- INDEX TYPES ----------
search_index_types:
  - lexical
  - semantic
  - vector
  - graph
  - structured
  - hybrid

rule index_type:
  SearchIndexType SHOULD preserve retrieval semantics

# ---------- INDEX SCHEMA ----------
entity IndexSchema:
  type: StructuralMetadata
  may_include:
    - fields
    - datatypes
    - analyzers
    - tokenisers
    - normalisers
    - vector_fields
    - semantic_fields
    - stored_fields
    - sortable_fields
    - filterable_fields

rule schema:
  IndexSchema SHOULD preserve field definitions
  required to interpret indexed values

rule schema_version:
  material schema change SHOULD create distinguishable schema or index version

# ---------- INDEX FIELDS ----------
entity IndexField:
  type: StructuredField
  may_include:
    - field_name
    - datatype
    - indexed
    - stored
    - searchable
    - filterable
    - sortable
    - analyzer
    - boost
    - vector_dimension

rule field:
  field semantics SHOULD preserve datatype and indexing configuration

rule field_name:
  field name != semantic concept identifier
  unless explicitly mapped

# ---------- INDEXED RESOURCE ----------
entity IndexedResource:
  type: IndexedRepresentation
  derived_from: SearchableResource
  may_include:
    - resource_identifier
    - index_document_identifier
    - indexed_fields
    - source_version
    - index_time
    - checksum
    - provenance

rule indexed_resource:
  IndexedResource != source SearchableResource

rule source_link:
  indexed representation SHOULD preserve source identifier
  and source version where available

# ---------- LEXICAL INDEX ----------
entity LexicalIndex:
  subtype_of: SearchIndex
  may_include:
    - token_dictionary
    - postings
    - document_frequency
    - term_frequency
    - field_norms

rule lexical:
  LexicalIndex supports token or term-based retrieval

# ---------- TOKENISATION ----------
entity TokenisationMetadata:
  type: IndexProcessingMetadata
  may_include:
    - tokenizer
    - language
    - token_filters
    - case_normalisation
    - stemming
    - lemmatisation
    - stop_words

rule tokenisation:
  tokenisation configuration SHOULD be preserved
  where lexical matching depends on it

rule token_not_concept:
  token != semantic concept

# ---------- TERM DICTIONARY ----------
entity TermDictionary:
  type: LexicalIndexStructure
  contains: IndexedTerm

entity IndexedTerm:
  type: LexicalResource
  may_include:
    - term
    - document_frequency
    - collection_frequency

rule term:
  IndexedTerm SHOULD preserve analyzer context

# ---------- INVERTED INDEX ----------
entity InvertedIndex:
  type: LexicalIndexStructure
  maps:
    - term
    - documents
    - positions_optional
    - frequencies_optional

rule inverted:
  postings SHOULD preserve document relationships
  and positional metadata where configured

# ---------- SEMANTIC INDEX ----------
entity SemanticIndex:
  subtype_of: SearchIndex
  may_include:
    - entities
    - concepts
    - relationships
    - ontology_mappings
    - semantic_annotations

rule semantic:
  SemanticIndex SHOULD preserve semantic identifiers
  and mapping provenance

rule annotation:
  inferred semantic annotation SHOULD preserve confidence

# ---------- VECTOR INDEX ----------
entity VectorIndex:
  subtype_of: SearchIndex
  may_include:
    - embedding_model
    - model_version
    - vector_dimension
    - distance_metric
    - nearest_neighbour_method
    - vector_normalisation

entity VectorConfiguration:
  type: IndexConfiguration
  may_include:
    - embedding_model
    - model_version
    - dimension
    - metric
    - normalisation
    - ANN_algorithm

rule vector:
  vector index SHOULD preserve embedding-space definition

rule dimension:
  indexed vector dimension MUST match configured dimension

rule embedding_compatibility:
  query and indexed embeddings MUST be compatible
  for direct similarity comparison

# ---------- ANN ----------
ann_methods:
  - exact
  - HNSW
  - IVF
  - PQ
  - LSH
  - implementation_specific

rule ANN:
  approximate nearest-neighbour retrieval MAY trade exactness
  for speed or storage efficiency

rule approximate:
  approximate result != guaranteed exact nearest neighbour

# ---------- GRAPH INDEX ----------
entity GraphIndex:
  subtype_of: SearchIndex
  may_include:
    - nodes
    - edges
    - entity_identifiers
    - relationship_types
    - graph_partitions

rule graph:
  GraphIndex SHOULD preserve edge direction and relationship semantics

# ---------- STRUCTURED INDEX ----------
entity StructuredIndex:
  subtype_of: SearchIndex
  may_include:
    - fields
    - datatypes
    - ranges
    - facets
    - filter_structures

rule structured:
  StructuredIndex supports field-aware filtering and retrieval

# ---------- HYBRID INDEX ----------
entity HybridIndex:
  subtype_of: SearchIndex
  combines:
    - lexical_index
    - semantic_index_optional
    - vector_index_optional
    - structured_index_optional

rule hybrid:
  HybridIndex SHOULD preserve component-index identities
  and score-combination semantics

# ---------- FACETS ----------
entity FacetMetadata:
  type: IndexMetadata
  may_include:
    - field
    - value
    - count

rule facet:
  facet count applies to indexed corpus and query context
  where calculated

# ---------- SHARDING ----------
entity IndexShard:
  type: PartitionMetadata
  may_include:
    - shard_identifier
    - shard_key
    - document_range
    - replica_count
    - status

rule shard:
  sharding SHOULD preserve logical SearchIndex identity

rule shard_not_index:
  IndexShard != complete SearchIndex

# ---------- REPLICAS ----------
entity IndexReplica:
  type: AvailabilityMetadata
  may_include:
    - replica_identifier
    - shard_identifier
    - synchronisation_state

rule replica:
  replica SHOULD preserve source shard relationship

# ---------- VERSION ----------
entity SearchIndexVersion:
  type: VersionMetadata
  may_include:
    - version_identifier
    - predecessor
    - successor
    - schema_version
    - creation_time
    - source_snapshot
    - status

rule version:
  material index rebuild or schema change SHOULD create distinguishable version
  where reproducibility matters

# ---------- FRESHNESS ----------
entity IndexFreshness:
  type: TemporalMetadata
  may_include:
    - source_update_time
    - index_update_time
    - lag
    - freshness_state

freshness_states:
  - current
  - delayed
  - stale
  - rebuilding
  - unknown

rule freshness:
  indexed state MAY lag source resource state

rule stale_index:
  stale index != stale source resource

# ---------- INDEXING PROCESS ----------
entity IndexingProcess:
  type: DataProcessingActivity
  may_include:
    - source_resource
    - extraction_method
    - transformation
    - analyzer
    - semantic_annotation
    - embedding_generation
    - index_target
    - timestamp

rule indexing:
  IndexingProcess SHOULD preserve transformations
  applied between source and index representation

# ---------- DELETION / TOMBSTONES ----------
entity IndexDeletionMetadata:
  type: LifecycleMetadata
  may_include:
    - resource_identifier
    - deletion_time
    - tombstone
    - reason

rule deletion:
  source deletion SHOULD propagate according_to index update policy

rule tombstone:
  tombstone != active indexed resource

# ---------- STATUS ----------
index_status:
  - creating
  - active
  - updating
  - rebuilding
  - degraded
  - stale
  - archived
  - retired

rule status:
  SearchIndex status SHOULD remain explicit

# ---------- PROVENANCE ----------
entity SearchIndexProvenance:
  type: ProvenanceMetadata
  may_include:
    - source_corpus
    - source_versions
    - indexing_process
    - schema
    - analyzers
    - embedding_model
    - build_time
    - update_history

rule provenance:
  SearchIndexMetadata SHOULD preserve source and build history

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_search_index

  validate_if_present:
    - index_identifier
    - index_type
    - schema
    - fields
    - indexed_resources
    - vector_configuration
    - shards
    - version
    - freshness
    - status
    - provenance

  reject:
    - incompatible_vector_dimension
    - incompatible_embedding_space
    - field_value_incompatible_with_schema
    - shard_presented_as complete_index
    - indexed_representation_presented_as source_resource
    - stale_index_presented_as guaranteed_current_source
    - approximate_result_semantics_presented_as exact_without evidence

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "search index metadata" ~= "metadata about a search index"
  "index schema" ~= "search index field schema"
  "indexed document" ~= "indexed resource representation"
  "inverted index" ~= "term-to-document postings index"
  "vector index" ~= "embedding search index"
  "semantic index" ~= "index of entities, concepts, and semantic annotations"

  PRESERVE distinctions:
    search_index != source_corpus
    indexed_resource != source_resource
    index_name != index_identity
    index_version != index
    field_name != semantic_identifier
    token != concept
    vector_similarity != factual_correctness
    approximate_neighbour != guaranteed_exact_neighbour
    shard != complete_index
    stale_index != stale_source
    tombstone != active_resource

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_index_metadata:
    score: 1.00

  explicit_index_identifier:
    score: 1.00

  schema_validated_index:
    score: 0.99

  compatible_vector_configuration:
    score: 0.99

  source_linked_indexed_resource:
    score: 0.98

  inferred_semantic_annotation:
    score: <= 0.90

  unknown_index_freshness:
    score: <= 0.70

  ambiguous_index_configuration:
    score: <= 0.60
    action: preserve uncertainty

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_index_creation:
    - assign SearchIndexIdentifier
    - classify SearchIndexType
    - record schema
    - record source corpus
    - initialise version and provenance

  on_resource_index:
    - resolve source identifier
    - extract configured fields
    - apply analyzers
    - generate semantic annotations or embeddings where configured
    - create IndexedResource
    - update freshness and provenance

  on_index_update:
    - preserve previous version where required
    - apply additions, updates, and deletions
    - update shard state
    - update freshness
    - preserve provenance

  on_index_rebuild:
    - preserve prior SearchIndexVersion
    - create new version
    - apply current schema and configuration
    - validate compatibility
    - update provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - index_entities
    - relationships
    - schema_rules
    - lexical_rules
    - semantic_rules
    - vector_rules
    - lifecycle_rules
    - validation
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - preserve_index_identifiers
    - preserve_schema_field_semantics
    - preserve_embedding_model_version
    - preserve_shard_relationships
    - sort_unordered_configuration_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - semantic search metadata ontology
    - search index structure model
    - lexical, semantic, vector, graph, and hybrid index model
    - index lifecycle and provenance model

  operations:
    - identify search indexes
    - resolve index type and schema
    - inspect indexed fields
    - trace indexed resources to sources
    - inspect tokenisation and term dictionaries
    - traverse inverted indexes
    - inspect semantic annotations
    - validate vector configuration
    - inspect graph and structured indexes
    - resolve shards and replicas
    - evaluate freshness and status
    - traverse index versions
    - trace indexing provenance
    - preserve uncertainty

  constraints:
    - do_not_equate_index_with source_corpus
    - do_not_equate_indexed_representation_with source_resource
    - do_not_equate_token_with semantic_concept
    - do_not_compare_incompatible_embedding_spaces
    - do_not_present_approximate_neighbour_as exact_without evidence
    - do_not_present_stale_index_as guaranteed_current_source
    - preserve_schema_versions_models_freshness_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - index_identifier
    - index_type
    - field
    - resource_identifier
    - term
    - entity
    - concept
    - vector_configuration
    - version
    - freshness_state

  resolve:
    1. identify SearchIndex
    2. resolve SearchIndexIdentifier and type
    3. inspect IndexSchema and fields
    4. resolve indexed resource representations
    5. inspect lexical, semantic, vector, graph, or structured components
    6. inspect shards and replicas
    7. resolve SearchIndexVersion
    8. evaluate freshness and status
    9. inspect validation and provenance
    10. return confidence

  output:
    - search_index
    - identifier
    - index_type
    - schema
    - fields
    - indexed_resources
    - lexical_structures
    - semantic_structures
    - vector_configuration
    - shards
    - version
    - freshness
    - status
    - provenance
    - confidence

# ---------- INFERENCE ----------
inference:
  IF index contains term postings
  THEN classify LexicalIndex candidate
  WITH confidence dependent_on structure

  IF index stores compatible embedding vectors
  THEN classify VectorIndex candidate
  WITH confidence dependent_on configuration

  IF index stores entity and concept mappings
  THEN classify SemanticIndex candidate
  WITH confidence dependent_on semantic metadata

  IF index combines multiple retrieval structures
  THEN classify HybridIndex candidate

  IF index freshness is unknown
  THEN preserve source-index temporal uncertainty
  AND do_not assume current source equivalence

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: SearchIndexMetadata
  index_identity: machine_interpretable
  index_types: explicit
  schema_and_fields: preserved
  indexed_resources: source_linked
  lexical_structures: inspectable
  semantic_annotations: traceable
  vector_configuration: compatibility_aware
  graph_and_structured_indexes: supported
  shards_and_replicas: explicit
  versions: traversable
  freshness_and_status: reportable
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow