AI Confidence Metadata Ontology

FLOW ALGEBRA

Ontology: AI Confidence Metadata Ontology
Domain: Metadata Ontologies > AI Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/ai-metadata/ai-confidence-metadata-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "AIConfidenceMetadataOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: AIMetadata
  subject: AIConfidenceMetadata
  purpose:
    - represent metadata describing confidence associated with AI outputs, predictions, inferences, and decisions
    - preserve confidence value, meaning, scope, source, calibration, uncertainty, and provenance
    - support confidence interpretation, comparison, validation, thresholding, and governance
    - distinguish generic confidence from probability, certainty, evidence strength, and model accuracy
    - enable machine-readable reasoning over confidence-bearing AI results

# ---------- CORE ENTITY ----------
entity AIConfidenceMetadata:
  type: AIMetadata
  describes: ConfidenceAssertion
  may_include:
    - confidence_identifier
    - confidence_value
    - confidence_type
    - confidence_scope
    - target_resource
    - model_identifier
    - model_version
    - calibration_metadata
    - uncertainty_metadata
    - threshold
    - evidence
    - method
    - timestamp
    - provenance
    - validity

entity ConfidenceAssertion:
  type: EpistemicMetadata
  qualifies: AIResource

entity ConfidenceIdentifier:
  type: Identifier
  identifies: ConfidenceAssertion

entity ConfidenceValue:
  type: QuantitativeValue
  range: [0.0, 1.0]

# ---------- CONFIDENCE TYPES ----------
confidence_types:
  - classification_confidence
  - prediction_confidence
  - inference_confidence
  - entity_resolution_confidence
  - retrieval_confidence
  - mapping_confidence
  - reasoning_confidence
  - generation_confidence
  - decision_confidence
  - source_confidence

# ---------- RELATIONSHIPS ----------
relationships:
  AIConfidenceMetadata -> DESCRIBES -> ConfidenceAssertion
  ConfidenceAssertion -> IDENTIFIED_BY -> ConfidenceIdentifier
  ConfidenceAssertion -> HAS_VALUE -> ConfidenceValue
  ConfidenceAssertion -> HAS_TYPE -> ConfidenceType
  ConfidenceAssertion -> HAS_SCOPE -> ConfidenceScope
  ConfidenceAssertion -> QUALIFIES -> AIResource
  ConfidenceAssertion -> MAY_REFERENCE_MODEL -> AIModel
  ConfidenceAssertion -> MAY_REFERENCE_MODEL_VERSION -> ModelVersion
  ConfidenceAssertion -> MAY_HAVE_CALIBRATION -> CalibrationMetadata
  ConfidenceAssertion -> MAY_HAVE_UNCERTAINTY -> UncertaintyMetadata
  ConfidenceAssertion -> MAY_HAVE_THRESHOLD -> ConfidenceThreshold
  ConfidenceAssertion -> MAY_REFERENCE -> Evidence
  AIConfidenceMetadata -> MAY_REFERENCE -> ConfidenceProvenance

# ---------- CONFIDENCE IDENTITY ----------
rule confidence_identity:
  ConfidenceAssertion SHOULD have stable identifier
  where auditability or versioned confidence history is required

rule target:
  ConfidenceAssertion MUST identify the resource, claim, prediction, or inference it qualifies

rule confidence_not_target:
  ConfidenceAssertion != TargetResource

# ---------- VALUE ----------
rule confidence_bounds:
  REQUIRE 0.0 <= ConfidenceValue <= 1.0

rule scale:
  alternative confidence scales MUST declare scale and conversion semantics

rule value_context:
  numeric confidence without semantic definition SHOULD NOT be assumed comparable
  across models, tasks, or confidence methods

# ---------- SCOPE ----------
entity ConfidenceScope:
  type: MetadataScope
  may_be:
    - output
    - token
    - classification
    - prediction
    - entity_link
    - relationship
    - retrieval_result
    - inference
    - decision
    - source

rule scope:
  ConfidenceAssertion SHOULD identify ConfidenceScope

rule scope_inheritance:
  output-level confidence MUST NOT automatically be assigned
  to every component of the output

# ---------- CONFIDENCE VS PROBABILITY ----------
rule probability_distinction:
  ConfidenceValue != Probability
  unless confidence method explicitly defines calibrated probabilistic semantics

rule probability:
  IF confidence represents probability
  THEN probability model and calibration SHOULD be explicit

rule certainty:
  ConfidenceValue != Certainty

rule accuracy:
  ConfidenceValue != ModelAccuracy

# ---------- CALIBRATION ----------
entity ConfidenceCalibration:
  type: CalibrationMetadata
  may_include:
    - calibration_method
    - calibration_dataset
    - calibration_curve
    - expected_calibration_error
    - reliability_measure
    - calibration_time
    - model_version

calibration_methods:
  - Platt_scaling
  - isotonic_regression
  - temperature_scaling
  - histogram_binning
  - Bayesian_calibration
  - empirical_calibration

rule calibration:
  calibrated confidence SHOULD preserve calibration method and dataset

rule model_version:
  calibration MUST be associated with applicable ModelVersion

rule calibration_transfer:
  calibration from one model or task MUST NOT automatically be applied to another

# ---------- UNCERTAINTY ----------
entity UncertaintyMetadata:
  type: EpistemicMetadata
  may_include:
    - uncertainty_type
    - uncertainty_value
    - interval
    - distribution
    - method

uncertainty_types:
  - aleatoric
  - epistemic
  - predictive
  - measurement
  - model
  - data

rule confidence_uncertainty:
  confidence and uncertainty MAY be related
  but MUST NOT be assumed exact inverses

rule uncertainty_not_error:
  uncertainty != error

# ---------- THRESHOLDS ----------
entity ConfidenceThreshold:
  type: DecisionConstraint
  may_include:
    - minimum_confidence
    - warning_threshold
    - abstention_threshold
    - escalation_threshold
    - action_threshold

rule threshold:
  threshold SHOULD preserve task and decision context

rule threshold_not_truth:
  confidence above threshold != guaranteed correctness

rule abstention:
  IF confidence below abstention threshold
  THEN system MAY abstain, defer, or request review
  according_to policy

# ---------- EVIDENCE ----------
entity ConfidenceEvidence:
  type: Evidence
  may_include:
    - model_score
    - ensemble_agreement
    - retrieval_support
    - source_quality
    - validation_result
    - human_review

rule evidence:
  confidence SHOULD preserve evidence basis
  where available

rule evidence_strength:
  EvidenceStrength != ConfidenceValue
  unless confidence method explicitly derives one from the other

# ---------- METHODS ----------
entity ConfidenceMethod:
  type: AIMethod
  may_be:
    - model_probability
    - margin_score
    - ensemble_agreement
    - Bayesian_posterior
    - similarity_score
    - retrieval_score
    - heuristic_score
    - rule_based_score
    - human_assessment

rule method:
  ConfidenceAssertion SHOULD identify method
  where interpretation depends on how confidence was produced

rule score_normalisation:
  normalised score in [0,1] MUST NOT automatically be called probability

# ---------- CLASSIFICATION ----------
rule classification:
  classification confidence SHOULD identify:
    - predicted_class
    - score
    - scoring_method
    - competing_classes_optional
    - calibration_optional

rule multiclass:
  class scores MAY sum to 1
  only when scoring method defines probability distribution semantics

# ---------- ENTITY RESOLUTION ----------
rule entity_resolution:
  entity-link confidence SHOULD preserve:
    - mention_or_candidate
    - candidate_entity
    - evidence
    - context
    - resolution_method

rule merge_threshold:
  high entity-resolution confidence MAY support merge
  but identity policy MUST govern final merge decision

# ---------- RETRIEVAL ----------
rule retrieval:
  retrieval confidence or relevance score SHOULD preserve ranking method

rule retrieval_score:
  retrieval score != factual confidence
  unless explicitly transformed and validated for that purpose

# ---------- REASONING ----------
rule reasoning_confidence:
  confidence in Conclusion SHOULD preserve:
    - input confidence
    - evidence
    - assumptions
    - reasoning_method
    - validation

rule propagation:
  confidence propagation SHOULD use declared model
  and MUST NOT assume simple multiplication or averaging without rule

# ---------- GENERATION ----------
rule generation_confidence:
  free-form generated content MAY lack a single meaningful scalar confidence

rule token_confidence:
  token-level likelihood or score MUST NOT automatically be interpreted
  as factual confidence of complete generated statement

# ---------- HUMAN REVIEW ----------
entity HumanConfidenceAssessment:
  type: ConfidenceAssertion
  produced_by: HumanAgent

rule human_confidence:
  human-assigned confidence SHOULD preserve assessor and method

rule model_human_distinction:
  model confidence != human confidence

# ---------- TEMPORAL VALIDITY ----------
entity ConfidenceValidity:
  type: ValidityMetadata
  may_include:
    - valid_from
    - valid_until
    - conditions

rule validity:
  confidence MAY become stale
  when model, data, context, or calibration changes

# ---------- PROVENANCE ----------
entity ConfidenceProvenance:
  type: ProvenanceMetadata
  may_include:
    - model
    - model_version
    - method
    - calibration
    - input
    - context
    - evidence
    - timestamp

rule provenance:
  AIConfidenceMetadata SHOULD preserve confidence derivation context
  at granularity available

# ---------- VALIDATION ----------
validation:
  require:
    - target_resource
    - confidence_value_or_interpretable_confidence_state

  validate_if_present:
    - confidence_identifier
    - confidence_type
    - confidence_scope
    - method
    - model_version
    - calibration
    - uncertainty
    - threshold
    - evidence
    - provenance

  reject:
    - confidence_value_outside_declared_scale
    - probability_claim_without probabilistic_semantics
    - calibration_applied_to_wrong_model_version
    - confidence_without_target
    - incompatible_confidence_comparison
    - confidence_above_threshold_presented_as guaranteed_truth

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "AI confidence metadata" ~= "metadata about AI confidence"
  "confidence score" ~= "AI confidence value" WHEN score semantics are confidence
  "prediction confidence" ~= "confidence in prediction"
  "classification confidence" ~= "confidence in predicted class"
  "confidence calibration" ~= "calibration of confidence scores"
  "abstention threshold" ~= "minimum confidence for automatic response" WHEN policy semantics match

  PRESERVE distinctions:
    confidence != probability
    confidence != certainty
    confidence != accuracy
    confidence != evidence_strength
    confidence != relevance
    confidence != uncertainty
    uncertainty != error
    retrieval_score != factual_confidence
    token_probability != statement_truth
    high_confidence != guaranteed_correctness

# ---------- CONFIDENCE MODEL ----------
CONFIDENCE:
  explicit_confidence_metadata:
    score: 1.00

  explicit_calibrated_probability:
    score: 1.00

  validated_calibration_context:
    score: 0.98

  explicit_model_score_with_method:
    score: 0.95

  inferred_confidence_semantics:
    score: <= 0.80

  normalised_unknown_score:
    score: <= 0.60
    action: do_not_assume probability

  ambiguous_confidence_context:
    score: <= 0.50
    action: preserve uncertainty

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_confidence_creation:
    - identify target resource
    - record confidence type and scope
    - record value and method
    - attach model version
    - attach calibration where applicable
    - record provenance

  on_model_update:
    - preserve previous ConfidenceAssertions
    - reassess applicability to new ModelVersion
    - invalidate transferred calibration unless validated
    - update provenance

  on_calibration_update:
    - preserve previous calibration
    - apply new calibration to compatible scores
    - record calibration version
    - update confidence semantics

  on_evidence_update:
    - preserve previous confidence state
    - reassess confidence according_to declared method
    - create revised ConfidenceAssertion
    - update provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - confidence_entities
    - confidence_types
    - relationships
    - scope_rules
    - calibration_rules
    - uncertainty_rules
    - threshold_rules
    - validation
    - semantic_equivalence
    - confidence_model
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - AI metadata ontology
    - confidence interpretation model
    - calibration and uncertainty structure
    - confidence-aware decision support model

  operations:
    - identify ConfidenceAssertions
    - resolve confidence types and scope
    - interpret confidence values
    - distinguish confidence from probability
    - inspect calibration
    - evaluate uncertainty
    - apply thresholds
    - inspect evidence
    - compare compatible confidence assertions
    - trace confidence provenance
    - preserve uncertainty

  constraints:
    - do_not_equate_confidence_with_probability_without explicit semantics
    - do_not_equate_confidence_with_accuracy
    - do_not_equate_relevance_score_with factual_confidence
    - do_not_present_high_confidence_as guaranteed_truth
    - do_not_transfer_calibration_across_models_without validation
    - preserve_scope_method_calibration_evidence_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - confidence_identifier
    - target_resource
    - confidence_type
    - confidence_scope
    - model_identifier
    - model_version
    - threshold
    - calibration_method
    - time_constraint

  resolve:
    1. identify ConfidenceAssertion
    2. resolve target resource
    3. resolve confidence type and scope
    4. inspect confidence method
    5. resolve model and ModelVersion
    6. inspect calibration
    7. inspect uncertainty and evidence
    8. evaluate thresholds
    9. apply validity
    10. return provenance and interpretation

  output:
    - confidence_assertion
    - target_resource
    - confidence_value
    - confidence_type
    - scope
    - method
    - calibration
    - uncertainty
    - threshold_state
    - provenance

# ---------- INFERENCE ----------
inference:
  IF confidence score is explicitly calibrated as probability
  THEN probability interpretation MAY be applied
  within calibration scope

  IF confidence score is only normalised to [0,1]
  THEN do_not infer probability semantics

  IF model version changes
  THEN previous calibration applicability becomes unknown
  unless compatibility is validated

  IF confidence falls below declared abstention threshold
  THEN infer abstention_candidate
  according_to policy

  IF confidence semantics remain ambiguous
  THEN preserve raw score and method
  AND avoid unsupported interpretation

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: AIConfidenceMetadata
  confidence_structure: machine_interpretable
  target_and_scope: explicit
  values: bounded_and_contextualised
  methods: preserved
  calibration: traceable
  uncertainty: distinguishable
  thresholds: machine_evaluable
  evidence: preserved
  provenance: preserved
  probability_semantics: explicit_only
  uncertainty_state: explicit
  semantic_retrieval: enabled

end flow