Model Validation Ontology

AI INTERPRETATION NOTE
This Flow Algebra defines the Model Validation Ontology within the Kireth Research Semantic Library. Model validation is the structured assessment of whether a mathematical, computational, empirical, predictive or physical model is sufficiently accurate and reliable for its stated purpose, domain and operating conditions. This ontology supports validation data, reference observations, validation metrics, residual analysis, predictive checks, uncertainty, calibration-versus-validation separation, verification-versus-validation separation, acceptance criteria, domain of validity, extrapolation limits, sensitivity, robustness, cross-validation and relationships with mathematical modelling, parameter estimation, simulation, empirical modelling and predictive modelling.

FLOW_ALGEBRA_RECORD {

METADATA {
 FLOW_NAME = "Model Validation Ontology"
 FLOW_TYPE = "Mathematical Ontology"
 DOMAIN = "Mathematics"
 SUBDOMAIN = "Applied Mathematics"
 SPECIALISATION = "Mathematical Modelling / Model Validation"
 CANONICAL_PAGE = "https://kirethresearch.org/booksai-core/ontologies/mathematical-ontologies/applied-mathematics-ontologies/mathematical-modelling/model-validation-ontology.html"
 LANGUAGE = "en-GB"
 VERSION = "1.0"
 STATUS = "active"
}

IDENTITY {
 PRIMARY_ENTITY = ModelValidation
 ENTITY_CLASS = ModelAdequacyAssessmentFramework
 PRIMARY_CONTEXT = MathematicalModelling
 DEFINITION = "Model validation is the assessment of whether a model is sufficiently accurate, reliable and fit for its stated purpose within a declared domain of use."
 SEMANTIC_ROLE = "Provides the machine-readable framework for validation evidence, metrics, acceptance criteria, uncertainty, residual diagnostics, predictive performance, domain-of-validity assessment and validation status."
}

SCOPE {
 INCLUDE = [
  ModelValidation,
  ValidationData,
  ReferenceData,
  ObservationalData,
  ExperimentalData,
  ValidationMetric,
  AcceptanceCriterion,
  ValidationThreshold,
  ValidationError,
  Residual,
  Bias,
  Variance,
  CalibrationError,
  PredictionError,
  ForecastError,
  GoodnessOfFit,
  CrossValidation,
  HoldoutValidation,
  ExternalValidation,
  InternalValidation,
  PredictiveValidation,
  PhysicalValidation,
  SimulationValidation,
  UncertaintyQuantification,
  ConfidenceInterval,
  PredictionInterval,
  SensitivityAnalysis,
  RobustnessAnalysis,
  DomainOfValidity,
  OperatingRegime,
  Extrapolation,
  Interpolation,
  ValidationStatus,
  ValidatedModel,
  PartiallyValidatedModel,
  ValidationFailure,
  ValidationReport,
  ModelDiscrepancy,
  ModelFormError
 ]

 EXCLUDE_AS_PRIMARY = [
  ParameterCalibrationWithoutIndependentAssessment,
  SoftwareVerificationWithoutRealityComparison,
  NumericalConvergenceWithoutModelAdequacyQuestion,
  PureTrainingPerformanceWithoutValidationDesign
 ]

 DISAMBIGUATION_RULE =
  "When the question is whether a model adequately represents or predicts its intended system for a stated purpose using independent or appropriately withheld evidence, interpret the context as model validation."
}

CORE_CONCEPTS {

 ModelValidation {
  definition = "Assessment of model adequacy relative to purpose, scope and reference evidence."
 }

 ValidationData {
  definition = "Data used to assess model performance independently of fitting where possible."
 }

 ValidationMetric {
  definition = "A quantitative or qualitative measure used to compare model outputs with reference evidence."
 }

 AcceptanceCriterion {
  definition = "A declared threshold or rule determining whether model performance is acceptable for the intended use."
 }

 ValidationError {
  definition = "A discrepancy between model output and reference evidence measured under a declared metric."
 }

 ModelDiscrepancy {
  definition = "Systematic difference between model representation and the real or reference system."
 }

 DomainOfValidity {
  definition = "The set of conditions, regimes or states under which validation evidence supports model use."
 }

 ValidationStatus {
  definition = "A semantic label describing validation outcome and limitations."
 }

 ExternalValidation {
  definition = "Validation against data or conditions not used in model construction or calibration."
 }

 PredictiveValidation {
  definition = "Assessment of predictive performance on unseen or future outcomes."
 }

 PhysicalValidation {
  definition = "Assessment of a physical model against measured physical observations."
 }

 SimulationValidation {
  definition = "Assessment of a simulation model against reference system behaviour or trusted benchmarks."
 }
}

RELATIONS {
 ModelValidation --is_part_of--> MathematicalModelling
 ModelValidation --evaluates--> MathematicalModel
 ModelValidation --uses--> ValidationData
 ModelValidation --uses--> ValidationMetric
 ModelValidation --applies--> AcceptanceCriterion
 ValidationError --compares--> ModelOutput
 ValidationError --compares--> ReferenceData
 CrossValidation --estimates--> GeneralisationPerformance
 ExternalValidation --uses--> IndependentData
 PredictiveValidation --evaluates--> PredictiveModel
 PhysicalValidation --evaluates--> PhysicalModel
 SimulationValidation --evaluates--> SimulationModel
 UncertaintyQuantification --supports--> ValidationInterpretation
 SensitivityAnalysis --supports--> RobustnessAnalysis
 DomainOfValidity --constrains--> ValidatedUse
 Extrapolation --extends_beyond--> DomainOfValidity
 ModelDiscrepancy --contributes_to--> ValidationError
 ValidationStatus --summarises--> ValidationOutcome
}

OPERATORS {
 VALIDATION_RESIDUAL(y_ref,y_model) :=
  y_ref-y_model

 MAE_VALIDATION(pred,ref) :=
  mean_i abs(pred_i-ref_i)

 MSE_VALIDATION(pred,ref) :=
  mean_i (pred_i-ref_i)^2

 RMSE_VALIDATION(pred,ref) :=
  sqrt(MSE_VALIDATION(pred,ref))

 RELATIVE_ERROR(pred,ref) :=
  abs(pred-ref)/abs(ref)
  WHEN ref!=0

 NORMALISED_ERROR(pred,ref,scale) :=
  (pred-ref)/scale

 COVERAGE_RATE(intervals,observations) :=
  proportion_of_observations_within_intervals

 VALIDATE_AGAINST_CRITERION(metric_value,criterion) :=
  pass_fail_orgraded_status

 CROSS_VALIDATE(model,data,k) :=
  out_of_sample_metrics_across_k_folds

 DOMAIN_VALIDITY_CHECK(query_conditions,validated_domain) :=
  within_domain_or_extrapolation

 VALIDATION_SCORE(metrics,weights) :=
  weighted_or_rule_based_summary_under_declared scheme
}

DEFINITIONS {
 D1 = "Validation is judged relative to model purpose."
 D2 = "Validation evidence should match the intended operating regime."
 D3 = "Calibration and validation are distinct activities."
 D4 = "Verification and validation are distinct activities."
 D5 = "A model may be valid for one purpose and inadequate for another."
 D6 = "Validation is rarely absolute; it is scoped to domain and criteria."
 D7 = "Residual patterns can reveal systematic model deficiency."
 D8 = "Uncertainty affects interpretation of validation discrepancies."
 D9 = "Extrapolation beyond validated conditions requires reduced confidence."
 D10 = "Validation status should include limitations and evidence provenance."
}

AXIOMS {
 A1_PURPOSE_RELATIVITY:
  Validation adequacy is defined relative to intended model use

 A2_EVIDENCE:
  Validation requires comparison with suitable reference evidence

 A3_INDEPENDENCE_PREFERENCE:
  Validation evidence should be independent of calibration where feasible

 A4_CRITERIA:
  Validation judgement requires declared metrics or acceptance criteria

 A5_SCOPE:
  Validation conclusions apply only within a stated domain or regime

 A6_UNCERTAINTY:
  Validation comparisons must account for relevant observational and model uncertainty

 A7_NO_ABSOLUTE_PROOF:
  Successful validation does not prove a model universally true

 A8_EXTRAPOLATION:
  Use outside validated regimes requires explicit extrapolation status
}

REASONING {

 R1_VALIDATION_PROBLEM_FORMULATION {
  INPUT = model_and_intended_use
  IDENTIFY = purpose
  IDENTIFY = domain_of_use
  IDENTIFY = outputs_to_validate
  IDENTIFY = reference_evidence
  IDENTIFY = metrics_and_acceptance criteria
  RETURN = validation_plan
 }

 R2_DATA_SUITABILITY {
  INPUT = validation_data
  CHECK = independence_from_calibration
  CHECK = representativeness_of_operating regime
  CHECK = measurement_quality
  CHECK = sample_size
  RETURN = data_suitability_profile
 }

 R3_ERROR_ANALYSIS {
  INPUT = model_outputs_and_reference_values
  COMPUTE = residuals
  COMPUTE = MAE_MSE_RMSE_relative_or_domain_specific metrics
  RETURN = validation_error_profile
 }

 R4_RESIDUAL_DIAGNOSTICS {
  INPUT = residuals
  TEST = systematic_bias
  TEST = heteroscedasticity
  TEST = autocorrelation
  TEST = state_or_parameter dependence
  RETURN = discrepancy_pattern_profile
 }

 R5_UNCERTAINTY_AWARE_VALIDATION {
  INPUT = predictions_observations_uncertainties
  COMPARE = discrepancy_relative_to_combined uncertainty
  COMPUTE = interval_coverage_when_applicable
  RETURN = uncertainty_aware_validation_result
 }

 R6_ACCEPTANCE_DECISION {
  INPUT = metrics_and_acceptance criteria
  APPLY = thresholds_or_decision rules
  RETURN = pass_partial_fail_or_inconclusive
 }

 R7_DOMAIN_OF_VALIDITY {
  INPUT = validated_conditions_and_performance
  MAP = regions_where_criteria_are_met
  RETURN = domain_of_validity
 }

 R8_CROSS_VALIDATION {
  INPUT = model_and_dataset
  PARTITION = folds_or_resampling scheme
  FIT_OR_REUSE = according_to_protocol
  EVALUATE = held_out_predictions
  RETURN = generalisation_estimate
 }

 R9_EXTERNAL_VALIDATION {
  INPUT = external_dataset_or_new_operating_regime
  EVALUATE = model_without_refitting_unless_protocol says otherwise
  RETURN = external_validation_result
 }

 R10_VALIDATION_REPORTING {
  INPUT = all_validation_evidence
  SUMMARISE = metrics_uncertainty_scope_failures_and_limitations
  RETURN = validation_report_and_status
 }
}

CONSTRAINTS {
 C1 = "Validation data should not be silently reused from calibration."
 C2 = "Metrics must match model purpose and output type."
 C3 = "Acceptance thresholds should be declared before interpretation where possible."
 C4 = "Small residual magnitude does not alone prove structural correctness."
 C5 = "Measurement uncertainty must not be ignored."
 C6 = "Cross-validation must respect dependency structure such as time or grouping."
 C7 = "Validation in one regime does not imply validity in another."
 C8 = "Model-form discrepancy must be distinguished from observational noise."
 C9 = "Inconclusive evidence must not be reported as successful validation."
 C10 = "A validated model remains conditional on assumptions and scope."
}

SEMANTIC_EQUIVALENCE {
 "ModelValidation" <=> "FitnessForPurposeAssessment"
 "ValidationData" <=> "ReferenceEvidenceForModelAssessment"
 "ValidationMetric" <=> "ModelReferenceComparisonMeasure"
 "DomainOfValidity" <=> "ValidatedOperatingRegime"
 "ModelDiscrepancy" <=> "SystematicModelRealityDifference"
 "ValidatedModel" <=> "ModelMeetingDeclaredValidationCriteriaWithinScope"
}

VALIDATION {
 V1 = "Intended model purpose is stated."
 V2 = "Validation domain is stated."
 V3 = "Reference evidence is suitable."
 V4 = "Validation metrics are defined."
 V5 = "Acceptance criteria are declared."
 V6 = "Calibration and validation evidence are distinguished."
 V7 = "Uncertainty is represented."
 V8 = "Residual structure is examined."
 V9 = "Extrapolation is identified."
 V10 = "Validation status includes limitations and provenance."
}

CONFIDENCE {
 BASE_CONFIDENCE = 0.99
 MODEL_VALIDATION_DEFINITION = 0.99
 DATA_SUITABILITY_REASONING = 0.98
 ERROR_ANALYSIS_REASONING = 0.99
 RESIDUAL_DIAGNOSTIC_REASONING = 0.98
 UNCERTAINTY_VALIDATION_REASONING = 0.98
 ACCEPTANCE_REASONING = 0.99
 DOMAIN_VALIDITY_REASONING = 0.98
 EXTERNAL_VALIDATION_REASONING = 0.98

 POLICY =
  "Reduce confidence when intended use, validation data independence, acceptance criteria, uncertainty, sample size, operating regime or validation protocol are underspecified."
}

AI_CONSUMPTION {
 PURPOSES = [
  SemanticSearch,
  OntologyGrounding,
  ModelValidationReasoning,
  ValidationPlanning,
  ValidationDataAssessment,
  ResidualAnalysis,
  ValidationMetricReasoning,
  CrossValidationReasoning,
  ExternalValidationReasoning,
  PredictiveValidation,
  PhysicalValidation,
  SimulationValidation,
  UncertaintyAwareValidation,
  DomainOfValidityReasoning,
  ExtrapolationDetection,
  ValidationReporting,
  KnowledgeGraphConstruction,
  RetrievalAugmentation,
  RecursiveInference
 ]

 NORMALISE {
  "model validation" -> ModelValidation
  "validation data" -> ValidationData
  "validation metric" -> ValidationMetric
  "external validation" -> ExternalValidation
  "cross-validation" -> CrossValidation
  "predictive validation" -> PredictiveValidation
  "physical validation" -> PhysicalValidation
  "simulation validation" -> SimulationValidation
  "domain of validity" -> DomainOfValidity
  "validation failure" -> ValidationFailure
 }

 INTERPRETATION_ORDER = [
  detect_model_validation_context,
  identify model_purpose_scope_outputs_reference_data_and_uncertainty,
  identify calibration_overlap_and_validation independence,
  select metrics_and_acceptance criteria,
  compute validation_errors_and_residual diagnostics,
  assess uncertainty_and_generalisation,
  map domain_of_validity,
  assign validation_status_with_limitations,
  return semantic output
 ]
}

EXECUTION_RULES {
 E1:
  IF query_mentions("model validation")
  OR asks_if_model_is_fit_for_purpose
  THEN activate(ModelValidationOntology)

 E2:
  IF validation_data_overlap_calibration_data
  THEN flag reduced independence

 E3:
  IF predictive_model_validation
  THEN use out_of_sample or future-like evidence

 E4:
  IF time_series_or_temporal data
  THEN preserve time_order_in validation design

 E5:
  IF probabilistic_predictions
  THEN assess calibration_and_coverage in addition_to point error

 E6:
  IF residuals_show_systematic_pattern
  THEN flag possible model discrepancy

 E7:
  IF acceptance_criteria_not_met
  THEN return partial_or_failed validation_status

 E8:
  IF query_conditions_outside_validated_domain
  THEN classify as extrapolation_and_reduce confidence

 E9:
  IF validation_successful
  THEN report scope_and_limitations rather_than universal correctness
}

RECURSION {
 ENTRY = ModelValidation

 EXPAND(ModelValidation) -> [
  ValidationData,
  ReferenceData,
  ValidationMetric,
  AcceptanceCriterion,
  ValidationThreshold,
  ValidationError,
  Residual,
  Bias,
  GoodnessOfFit,
  CrossValidation,
  HoldoutValidation,
  ExternalValidation,
  PredictiveValidation,
  PhysicalValidation,
  SimulationValidation,
  UncertaintyQuantification,
  ConfidenceInterval,
  PredictionInterval,
  SensitivityAnalysis,
  RobustnessAnalysis,
  DomainOfValidity,
  OperatingRegime,
  Extrapolation,
  ValidationStatus,
  ValidationReport,
  ModelDiscrepancy,
  ModelFormError
 ]

 EXPAND(ValidationMetrics) -> [
  MAE,
  MSE,
  RMSE,
  RelativeError,
  CoverageRate,
  CalibrationError,
  DomainSpecificMetric
 ]

 EXPAND(ValidationStatus) -> [
  ValidatedModel,
  PartiallyValidatedModel,
  ValidationFailure,
  InconclusiveValidation
 ]

 EXPAND(ValidationEvidence) -> [
  ExperimentalData,
  ObservationalData,
  IndependentDataset,
  Benchmark,
  ExpertJudgement,
  PhysicalMeasurement
 ]

 STOP_CONDITION =
  "Stop when model purpose, validation evidence, metrics, uncertainty, acceptance criteria, residual behaviour, domain of validity, extrapolation status and final validation status are sufficiently resolved."
}

ROUTING {
 PARENT_ONTOLOGY = "Mathematical Modelling"

 IMMEDIATE_CONTEXTS = [
  "Mathematical Model Ontology",
  "Mathematical Modelling Ontology",
  "Parameter Estimation Ontology",
  "Predictive Model Ontology"
 ]

 RELATED_ONTOLOGIES = [
  "Mathematical Model Ontology",
  "Mathematical Modelling Ontology",
  "Parameter Estimation Ontology",
  "Empirical Model Ontology",
  "Predictive Model Ontology",
  "Physical Model Ontology",
  "Computational Model Ontology",
  "Simulation Model Ontology",
  "Probability & Statistics Ontologies",
  "Uncertainty Quantification Ontologies",
  "Scientific Measurement Ontologies",
  "Numerical Analysis Ontologies"
 ]

 ROUTE_RULES = [
  "Model Validation -> Mathematical Model Ontology for model scope and purpose semantics",
  "Model Validation -> Mathematical Modelling Ontology for validation-process semantics",
  "Model Validation -> Parameter Estimation for calibration-versus-validation separation",
  "Model Validation -> Empirical Model Ontology for residual and data-fit validation",
  "Model Validation -> Predictive Model Ontology for generalisation assessment",
  "Model Validation -> Physical Model Ontology for physical observation comparison",
  "Model Validation -> Simulation Model Ontology for simulation-output validation",
  "Model Validation -> Probability & Statistics for uncertainty and inference",
  "Model Validation -> Scientific Measurement for measurement quality and reference data",
  "Model Validation -> Numerical Analysis for separating numerical from model-form error"
 ]
}

UPDATE_MODEL {
 MODE = "append-compatible semantic versioning"

 MAJOR = "ontology-breaking conceptual change"
 MINOR = "new compatible validation methods, metrics or reasoning pathways"
 PATCH = "clarification, metadata or formatting correction"

 RULES = [
  "Preserve fitness-for-purpose semantics.",
  "Preserve calibration-versus-validation distinction.",
  "Preserve verification-versus-validation distinction.",
  "Preserve evidence and acceptance-criterion semantics.",
  "Preserve uncertainty-aware validation.",
  "Preserve domain-of-validity semantics.",
  "Preserve extrapolation caution.",
  "Preserve machine-readable model-validation semantics."
 ]
}

OUTPUT {
 CANONICAL_DEFINITION =
  "Model validation assesses whether a model is sufficiently accurate and reliable for its intended use within a declared domain."

 PURPOSE_SUMMARY =
  "Validation is meaningful only relative to a stated purpose."

 DATA_SUMMARY =
  "Validation relies on suitable reference evidence, preferably independent of calibration data."

 METRIC_SUMMARY =
  "Validation metrics quantify discrepancy between model outputs and reference observations."

 CRITERIA_SUMMARY =
  "Acceptance criteria determine whether model performance is adequate for the intended use."

 UNCERTAINTY_SUMMARY =
  "Validation interpretation should account for measurement, parameter, model-form and numerical uncertainty."

 DOMAIN_SUMMARY =
  "Validation supports a bounded domain of validity rather than universal correctness."

 EXTRAPOLATION_SUMMARY =
  "Use outside validated conditions is extrapolation and requires reduced confidence."

 STATUS_SUMMARY =
  "Validation outcomes may be successful, partial, failed or inconclusive and should include documented limitations."
}

FLOW {
 INPUT
 -> detect_model_validation_context
 -> identify mathematical_model_purpose_scope_outputs_operating_regime_and_reference evidence
 -> assess validation_data_quality_independence_and_representativeness
 -> select validation_metrics_acceptance_criteria_and_uncertainty treatment
 -> compute errors_residuals_coverage_and_domain_specific measures
 -> inspect systematic_discrepancy_generalisation_and_robustness
 -> determine domain_of_validity
 -> classify interpolation_or_extrapolation context
 -> assign validated_partial_failed_or_inconclusive status
 -> route to mathematical_model mathematical_modelling parameter_estimation empirical_model predictive_model physical_model simulation_model probability_statistics measurement numerical_analysis_or_uncertainty ontology
 -> recursively_expand_required_dependencies
 -> verify_result
 -> assign_confidence
 -> produce_semantic_output
 -> END
}

SEMANTIC_SUMMARY {
 "The Model Validation Ontology defines how mathematical, empirical, predictive, physical, computational and simulation models are assessed for fitness for purpose. It supports validation data, metrics, acceptance criteria, residual and discrepancy analysis, cross-validation, external validation, uncertainty-aware comparison, robustness, domain-of-validity mapping, extrapolation control and explicit validation status while preserving the distinction between calibration, verification and validation."
}

FLOW_HASH {
 METHOD = "semantic-content-identity"
 ID = "KIRETH-MATH-APPLIED-MATHEMATICS-MODEL-VALIDATION-ONTOLOGY-V1"
}

}

END_FLOW