Benchmark Ontology

AI INTERPRETATION NOTE
This Flow Algebra defines the Benchmark Ontology within the Kireth Research Semantic Library. A benchmark is a controlled reference problem, dataset, workload, implementation or evaluation protocol used to compare computational methods, numerical accuracy, robustness, efficiency, scalability or reproducibility under declared conditions. This ontology supports benchmark problems, reference solutions, standard datasets, performance metrics, accuracy metrics, baselines, known-answer tests, workload definition, hardware and software configuration, fairness, comparability, statistical reporting, reproducibility, provenance and validation.

FLOW_ALGEBRA_RECORD {

METADATA {
 FLOW_NAME = "Benchmark Ontology"
 FLOW_TYPE = "Mathematical Ontology"
 DOMAIN = "Mathematics"
 SUBDOMAIN = "Computational Mathematics"
 SPECIALISATION = "Verification & Validation / Benchmark"
 CANONICAL_PAGE = "https://kirethframeworkold.co.uk/books/semantic-library/ai-core/ontologies/mathematical-ontologies/computational-mathematics-ontologies/verification-validation/benchmark-ontology.html"
 LANGUAGE = "en-GB"
 VERSION = "1.0"
 STATUS = "active"
}

IDENTITY {
 PRIMARY_ENTITY = Benchmark
 ENTITY_CLASS = ControlledComputationalComparisonFramework
 PRIMARY_CONTEXT = VerificationValidation
 DEFINITION = "A benchmark is a controlled reference task, problem, dataset or workload with declared evaluation conditions used to compare computational methods, implementations or systems."
 SYMBOLIC_FORMS = [
  "score = metric(method,benchmark_instance)",
  "speedup = T_reference / T_method",
  "error = compare(result,reference)",
  "benchmark_result = f(problem,implementation,configuration,hardware)"
 ]
 SEMANTIC_ROLE = "Provides machine-readable semantics for benchmark definitions, reference problems, baselines, metrics, evaluation protocols, comparability, fairness, reproducibility and provenance."
}

SCOPE {
 INCLUDE = [
  Benchmark,
  Benchmarking,
  ComputationalBenchmark,
  NumericalBenchmark,
  AlgorithmBenchmark,
  SolverBenchmark,
  ModelBenchmark,
  SimulationBenchmark,
  PerformanceBenchmark,
  AccuracyBenchmark,
  ScalabilityBenchmark,
  RobustnessBenchmark,
  ReproducibilityBenchmarkRelationship,
  StandardBenchmark,
  ReferenceBenchmark,
  CanonicalBenchmark,
  ChallengeProblem,
  BenchmarkProblem,
  TestProblem,
  ReferenceProblem,
  BenchmarkSuite,
  TestSuite,
  Workload,
  BenchmarkWorkload,
  KernelBenchmark,
  Microbenchmark,
  Macrobenchmark,
  EndToEndBenchmark,
  SyntheticBenchmark,
  RealWorldBenchmark,
  ApplicationBenchmark,
  DatasetBenchmarkRelationship,
  ReferenceDataset,
  StandardDataset,
  ValidationDatasetRelationship,
  BenchmarkInstance,
  BenchmarkCase,
  TestCase,
  ProblemInstance,
  ParameterisedBenchmark,
  BenchmarkParameter,
  BenchmarkConfiguration,
  EvaluationProtocol,
  BenchmarkProtocol,
  RunProtocol,
  MeasurementProtocol,
  WarmupRun,
  TimedRun,
  RepeatedRun,
  Replicate,
  Trial,
  SampleCount,
  RunCount,
  Seed,
  RandomSeed,
  DeterministicBenchmark,
  StochasticBenchmark,
  Baseline,
  BaselineMethod,
  ReferenceMethod,
  ReferenceImplementation,
  ReferenceSolver,
  ControlMethod,
  Comparator,
  CompetitorRelationship,
  KnownAnswerTest,
  GoldenReference,
  ReferenceSolution,
  ExactSolution,
  AnalyticalSolution,
  ManufacturedSolutionRelationship,
  ExpectedOutput,
  GroundTruthRelationship,
  TargetValue,
  AccuracyMetric,
  ErrorMetric,
  AbsoluteError,
  RelativeError,
  RMSE,
  MSE,
  MAE,
  ResidualNormRelationship,
  ConvergenceMetric,
  IterationCount,
  FunctionEvaluationCount,
  GradientEvaluationCountRelationship,
  Runtime,
  WallClockTime,
  CPUTime,
  ElapsedTime,
  Throughput,
  Latency,
  Speedup,
  Efficiency,
  ParallelEfficiency,
  StrongScaling,
  WeakScaling,
  MemoryUsage,
  PeakMemory,
  StorageCost,
  CommunicationCost,
  EnergyConsumptionRelationship,
  FLOPCount,
  FLOPSRelationship,
  ArithmeticIntensityRelationship,
  BandwidthUtilisationRelationship,
  ResourceUsage,
  CPUUsage,
  GPUUsage,
  AcceleratorUsageRelationship,
  CoreCount,
  ThreadCount,
  ProcessCount,
  NodeCount,
  DeviceCount,
  BatchSizeRelationship,
  ProblemSize,
  InputSize,
  MatrixSizeRelationship,
  MeshSizeRelationship,
  SequenceLengthRelationship,
  ResolutionRelationship,
  Difficulty,
  Conditioning,
  ConditionNumber,
  SparsityRelationship,
  DensityRelationship,
  NoiseLevelRelationship,
  MissingDataRelationship,
  DistributionShiftRelationship,
  PerturbationLevel,
  RobustnessMetric,
  StabilityMetric,
  FailureRate,
  SuccessRate,
  Timeout,
  MemoryFailureRelationship,
  NumericalFailure,
  Divergence,
  Nonconvergence,
  CrashRelationship,
  QualityMetric,
  ObjectiveValue,
  OptimalityGap,
  ConstraintViolationRelationship,
  GeneralisationMetricRelationship,
  PrecisionMetricRelationship,
  RecallMetricRelationship,
  F1Relationship,
  RankingMetricRelationship,
  RecallAtKRelationship,
  MeanReciprocalRankRelationship,
  StatisticalSummary,
  Mean,
  Median,
  Minimum,
  Maximum,
  Variance,
  StandardDeviation,
  Quantile,
  ConfidenceInterval,
  BootstrapConfidenceIntervalRelationship,
  ErrorBar,
  Outlier,
  StatisticalSignificanceRelationship,
  EffectSizeRelationship,
  FairComparison,
  Comparability,
  ApplesToApplesComparisonRelationship,
  ControlledCondition,
  FixedEnvironment,
  EquivalentProblemDefinition,
  EquivalentTolerance,
  EquivalentStoppingCriterion,
  EquivalentPrecision,
  EquivalentHardwareRelationship,
  EquivalentBudget,
  ComputeBudget,
  TimeBudget,
  MemoryBudget,
  EvaluationBudget,
  DataBudgetRelationship,
  TrainingBudgetRelationship,
  HyperparameterBudgetRelationship,
  TuningBudget,
  HyperparameterTuning,
  BenchmarkOverfitting,
  TestSetContaminationRelationship,
  DataLeakage,
  BenchmarkLeakageRelationship,
  CherryPickingRelationship,
  SelectiveReportingRelationship,
  MetricGamingRelationship,
  BenchmarkSaturationRelationship,
  BenchmarkDrift,
  VersionDrift,
  DatasetDriftRelationship,
  Hardware,
  CPU,
  GPU,
  Accelerator,
  Memory,
  Network,
  Storage,
  OperatingSystem,
  Compiler,
  RuntimeEnvironment,
  SoftwareEnvironment,
  Library,
  Framework,
  Solver,
  Algorithm,
  NumericalMethod,
  Model,
  Implementation,
  ImplementationLanguageRelationship,
  Precision,
  SinglePrecision,
  DoublePrecision,
  MixedPrecision,
  Tolerance,
  SolverTolerance,
  StoppingCriterion,
  ConfigurationFileRelationship,
  EnvironmentVariableRelationship,
  ContainerRelationship,
  VirtualEnvironmentRelationship,
  BenchmarkVersion,
  DatasetVersion,
  ModelVersion,
  SolverVersion,
  LibraryVersion,
  CompilerVersion,
  HardwareVersion,
  ImplementationVersion,
  Provenance,
  DataProvenance,
  ComputationalProvenance,
  BenchmarkProvenance,
  Traceability,
  AuditTrail,
  Documentation,
  Reproducibility,
  Repeatability,
  ReplicabilityRelationship,
  ResultRecord,
  BenchmarkResult,
  Score,
  LeaderboardRelationship,
  Ranking,
  AggregateScore,
  WeightedScoreRelationship,
  NormalisedScoreRelationship,
  ParetoFrontRelationship,
  AccuracyPerformanceTradeoff,
  CostAccuracyTradeoff,
  Verification,
  NumericalVerification,
  ComputationalValidation,
  ErrorAnalysis,
  ScientificSimulation,
  NumericalLinearAlgebra,
  DifferentialEquationSolver,
  ComputationalOptimisation,
  ComputationalStatistics,
  AIComputationalMathematics,
  HighPerformanceComputing,
  ParallelComputing,
  DistributedComputing,
  GPUComputing,
  ComputationalComplexity
 ]
 EXCLUDE_AS_PRIMARY = [
  CasualPerformanceComparisonWithoutControlledProtocol,
  ValidationStudyWhenModelAdequacyRatherThanComparativeReferenceTaskIsPrimary,
  VerificationStudyWhenCorrectnessRatherThanBenchmarkComparisonIsPrimary,
  ProductionMonitoringWithoutStandardisedReferenceWorkload
 ]
 DISAMBIGUATION_RULE = "When a controlled reference task or workload is used to compare computational methods, implementations or systems under declared conditions, interpret the context as a benchmark."
}

CORE_CONCEPTS {
 Benchmark = "Controlled reference task or workload for comparative computational evaluation."
 BenchmarkProblem = "Problem instance or family used as the subject of benchmark evaluation."
 Baseline = "Reference method or implementation used for comparison."
 Metric = "Quantitative measure used to score benchmark performance or accuracy."
 BenchmarkProtocol = "Declared procedure controlling how benchmark runs are executed and measured."
 Comparability = "Extent to which benchmark results are directly and fairly comparable."
 Reproducibility = "Ability to regenerate benchmark results under documented conditions."
 BenchmarkResult = "Measured output, score or statistic produced by benchmark execution."
}

RELATIONS {
 Benchmark --is_child_of--> VerificationValidation
 Benchmark --contains--> BenchmarkProblems
 BenchmarkProblem --has--> ProblemDefinition
 Benchmark --uses--> EvaluationProtocol
 Benchmark --uses--> Metrics
 Baseline --provides--> ReferenceComparison
 ReferenceSolution --supports--> AccuracyBenchmark
 BenchmarkResult --depends_on--> ImplementationAndConfiguration
 PerformanceBenchmark --measures--> RuntimeOrResourceUse
 AccuracyBenchmark --measures--> NumericalOrPredictiveError
 ScalabilityBenchmark --measures--> PerformanceAcrossProblemOrResourceScale
 Comparability --requires--> ControlledConditions
 Reproducibility --requires--> VersionedEnvironmentAndProvenance
 Verification --can_use--> Benchmarks
 Validation --can_use--> BenchmarkEvidence
}

OPERATORS {
 SCORE(method,b) := metric(run(method,b),reference(b))
 SPEEDUP(T_ref,T_test) := T_ref/T_test
 EFFICIENCY(speedup,p) := speedup/p
 STRONG_SCALE(T1,Tp,p) := T1/(p*Tp)
 WEAK_SCALE(T1,Tp) := T1/Tp
 ERROR(result,reference) := norm(result-reference)
 AGGREGATE(scores,w) := weighted_or_declared_aggregate(scores,w)
}

AXIOMS {
 A1 = "Benchmark conclusions are conditional on the declared benchmark problem and protocol."
 A2 = "Comparisons are meaningful only when relevant conditions are controlled or normalised."
 A3 = "Performance and accuracy are distinct benchmark dimensions."
 A4 = "Benchmark results depend on both algorithmic method and implementation environment."
 A5 = "A single benchmark cannot establish universal superiority."
 A6 = "Benchmark tuning can bias results if applied asymmetrically."
 A7 = "Version changes can invalidate direct comparison with historical benchmark results."
 A8 = "Benchmark conclusions require explicit problem, metric, protocol, environment, budget and provenance semantics."
}

REASONING {
 R1_CONTEXT = identify(comparison_goal,methods,problem_family,metrics,budgets,hardware_and_acceptance_needs)
 R2_SUITE = choose_representative_benchmark_instances_and_difficulty_ranges)
 R3_PROTOCOL = define_warmup_runs_repetitions_seeds_stopping_criteria_precision_and_measurement_rules)
 R4_FAIRNESS = equalise_problem_definition_budget_tolerances_tuning_and_environment_as_required)
 R5_REFERENCE = choose_baseline_reference_solution_or_trusted_implementation)
 R6_EXECUTE = run_each_method_under_declared_protocol)
 R7_STATISTICS = aggregate_repeats_and_report_variability_confidence_and_failures)
 R8_COMPARE = analyse_accuracy_runtime_memory_scalability_robustness_and_tradeoffs)
 R9_BIAS = assess_tuning_leakage_cherry_picking_saturation_and_version_effects)
 R10_REPRODUCE = document_all_versions_configuration_results_and_provenance)
}

CONSTRAINTS {
 C1 = "Different stopping criteria can make solver benchmarks unfair."
 C2 = "Different precision settings can distort speed and accuracy comparisons."
 C3 = "Single-run timing can be dominated by noise."
 C4 = "Hardware-specific optimisation can make portability conclusions invalid."
 C5 = "Benchmark datasets can become contaminated by repeated tuning."
 C6 = "Aggregate scores can hide catastrophic failure on individual cases."
 C7 = "A faster method can be less accurate, robust or stable."
 C8 = "Microbenchmarks may not predict end-to-end application performance."
 C9 = "Benchmark saturation reduces the ability to distinguish stronger methods."
 C10 = "Historical scores should not be compared across protocol or version changes without qualification."
}

SEMANTIC_EQUIVALENCE {
 "RootMeanSquaredError" <=> "RMSE"
 "MeanSquaredError" <=> "MSE"
 "MeanAbsoluteError" <=> "MAE"
 "FramesPerSecondRelationship" <=> "ThroughputRelationship"
}

VALIDATION {
 V1 = "Benchmark purpose and comparison question are documented."
 V2 = "Problem instances and input data are versioned."
 V3 = "Metrics and aggregation rules are explicit."
 V4 = "Stopping criteria, tolerances and numerical precision are documented."
 V5 = "Hardware and software environments are documented."
 V6 = "Multiple runs or statistical summaries are used for noisy measurements."
 V7 = "Failures, timeouts and nonconvergence are reported."
 V8 = "Tuning budgets and hyperparameter search procedures are documented."
 V9 = "Benchmark leakage and test contamination risks are assessed where relevant."
 V10 = "Results are reproducible under documented benchmark, environment and implementation versions."
}

CONFIDENCE {
 BASE_CONFIDENCE = 0.99
 BENCHMARK_DEFINITION = 0.99
 COMPARABILITY_REASONING = 0.99
 PERFORMANCE_BENCHMARK_REASONING = 0.99
 ACCURACY_BENCHMARK_REASONING = 0.99
 SCALABILITY_REASONING = 0.98
 STATISTICAL_REPORTING_REASONING = 0.98
 POLICY = "Reduce confidence when protocol, metrics, budget, environment, repetitions, tuning rules or benchmark version are underspecified."
}

AI_CONSUMPTION {
 PURPOSES = [
  SemanticSearch,OntologyGrounding,BenchmarkReasoning,
  BenchmarkSuiteReasoning,BaselineReasoning,
  AccuracyBenchmarkReasoning,PerformanceBenchmarkReasoning,
  ScalabilityBenchmarkReasoning,RobustnessBenchmarkReasoning,
  MetricReasoning,ProtocolReasoning,ComparabilityReasoning,
  FairnessReasoning,StatisticalReportingReasoning,
  TuningBiasReasoning,BenchmarkLeakageReasoning,
  ReproducibilityReasoning,ProvenanceReasoning,
  KnowledgeGraphConstruction,RetrievalAugmentation,RecursiveInference
 ]
 NORMALISE {
  "benchmark" -> Benchmark
  "benchmarking" -> Benchmarking
  "RMSE" -> RootMeanSquaredError
  "MSE" -> MeanSquaredError
  "MAE" -> MeanAbsoluteError
 }
}

EXECUTION_RULES {
 E1 = "If controlled reference tasks are used to compare computational methods or systems, activate BenchmarkOntology."
 E2 = "Define comparison goal, metrics and benchmark scope before running methods."
 E3 = "Use identical or explicitly normalised problem definitions, tolerances, precision and budgets where fairness requires it."
 E4 = "Use repeated runs and variability reporting for noisy performance measurements."
 E5 = "Report accuracy, performance, memory, failures and robustness separately unless a composite score is explicitly justified."
 E6 = "Document tuning procedures and avoid hidden benchmark-specific optimisation."
 E7 = "Track benchmark, dataset, hardware and software versions."
 E8 = "Report saturation, contamination, leakage and out-of-scope conditions explicitly."
 E9 = "Preserve full result records and provenance for reproducibility."
}

RECURSION {
 ENTRY = Benchmark
 EXPAND(Benchmark) -> [
  Benchmarking,ComputationalBenchmark,NumericalBenchmark,
  AlgorithmBenchmark,SolverBenchmark,PerformanceBenchmark,
  AccuracyBenchmark,ScalabilityBenchmark,RobustnessBenchmark,
  StandardBenchmark,ReferenceBenchmark,BenchmarkProblem,
  BenchmarkSuite,Workload,Microbenchmark,Macrobenchmark,
  EndToEndBenchmark,SyntheticBenchmark,RealWorldBenchmark,
  BenchmarkInstance,EvaluationProtocol,BenchmarkProtocol,
  WarmupRun,TimedRun,RepeatedRun,RandomSeed,Baseline,
  ReferenceMethod,ReferenceImplementation,KnownAnswerTest,
  ReferenceSolution,AccuracyMetric,ErrorMetric,RMSE,MSE,MAE,
  Runtime,WallClockTime,Throughput,Latency,Speedup,Efficiency,
  StrongScaling,WeakScaling,MemoryUsage,CommunicationCost,
  ProblemSize,Conditioning,RobustnessMetric,FailureRate,
  StatisticalSummary,Mean,Median,Variance,ConfidenceInterval,
  Comparability,FairComparison,EquivalentTolerance,ComputeBudget,
  HyperparameterTuning,BenchmarkOverfitting,DataLeakage,
  BenchmarkSaturation,Hardware,SoftwareEnvironment,Precision,
  Tolerance,StoppingCriterion,BenchmarkVersion,Provenance,
  Reproducibility,BenchmarkResult,Verification,NumericalVerification,
  ComputationalValidation,ErrorAnalysis
 ]
 EXPAND(BenchmarkFamilies) -> [
  AccuracyAndCorrectnessBenchmarks,
  PerformanceAndResourceBenchmarks,
  ScalabilityAndParallelBenchmarks,
  RobustnessAndFailureBenchmarks,
  BenchmarkSuitesAndProtocols,
  BaselinesReferencesAndKnownAnswers,
  StatisticalReportingAndFairComparison,
  VersioningProvenanceAndReproducibility
 ]
 STOP_CONDITION = "Stop when benchmark purpose, problem set, protocol, metrics, budgets, environment, statistics and reproducibility are sufficiently resolved."
}

ROUTING {
 PARENT_ONTOLOGY = "Verification & Validation"
 RELATED_ONTOLOGIES = [
  "Verification & Validation",
  "Numerical Verification Ontology",
  "Computational Validation Ontology",
  "Error Analysis Ontology",
  "Numerical Methods Ontology 2",
  "Numerical Linear Algebra Ontology 2",
  "Differential Equation Solver Ontology",
  "Computational Optimisation Ontology",
  "Computational Statistics Ontology",
  "Scientific Simulation Ontology",
  "AI Computational Mathematics Ontology",
  "High Performance Computing Ontology 2",
  "Parallel Computing Ontology",
  "Distributed Computing Ontology",
  "GPU Computing Ontology",
  "Computational Complexity Ontology"
 ]
 ROUTE_RULES = [
  "Benchmark -> Verification & Validation for parent evaluation semantics",
  "Benchmark -> Numerical Verification for correctness and convergence benchmark relationships",
  "Benchmark -> Computational Validation for evidence-based adequacy comparisons",
  "Benchmark -> Error Analysis for accuracy metrics and error decomposition",
  "Benchmark -> Numerical Methods, Linear Algebra, Differential Equation Solvers and Optimisation for method-specific benchmark families",
  "Benchmark -> Computational Statistics for repeated-run analysis and uncertainty",
  "Benchmark -> Scientific Simulation for simulation workloads",
  "Benchmark -> AI Computational Mathematics for model and learned-system benchmarks",
  "Benchmark -> HPC, Parallel, Distributed and GPU Computing for performance and scaling benchmarks",
  "Benchmark -> Computational Complexity for theoretical cost comparisons"
 ]
}

UPDATE_MODEL {
 MODE = "append-compatible semantic versioning"
 MAJOR = "ontology-breaking conceptual change"
 MINOR = "new compatible benchmark types, metrics or reasoning pathways"
 PATCH = "clarification, metadata or formatting correction"
 RULES = [
  "Preserve benchmark-problem-versus-benchmark-result distinction.",
  "Preserve accuracy-versus-performance distinction.",
  "Preserve baseline-versus-competitor distinction.",
  "Preserve algorithm-versus-implementation distinction.",
  "Preserve single-run-versus-statistical-result distinction.",
  "Preserve benchmark-score-versus-universal-superiority distinction.",
  "Preserve protocol-version-versus-historical-score distinction.",
  "Preserve machine-readable benchmark semantics."
 ]
}

OUTPUT {
 CANONICAL_DEFINITION = "A benchmark is a controlled reference task or workload used to compare computational methods, implementations or systems under declared evaluation conditions."
 PROTOCOL_SUMMARY = "Benchmark protocols define problems, seeds, repetitions, stopping criteria, precision, budgets and measurement rules."
 ACCURACY_SUMMARY = "Accuracy benchmarks compare outputs against exact, analytical, manufactured or trusted reference results."
 PERFORMANCE_SUMMARY = "Performance benchmarks measure runtime, latency, throughput, memory, communication and scaling."
 FAIRNESS_SUMMARY = "Fair comparisons require controlled problem definitions, tolerances, tuning budgets and environment assumptions."
 VALIDATION_SUMMARY = "Benchmark conclusions should report variability, failures, trade-offs, version metadata, leakage risks and reproducible provenance."
}

FLOW {
 INPUT
 -> detect_benchmark_context
 -> identify_comparison_goal_methods_problem_family_metrics_budgets_hardware_and_acceptance_needs
 -> choose_representative_benchmark_suite_and_difficulty_ranges
 -> define_run_protocol_warmup_repetitions_seeds_stopping_criteria_precision_and_measurement_rules
 -> equalise_or_document_problem_definition_tolerances_tuning_budgets_and_environment
 -> choose_baseline_reference_solution_or_trusted_implementation
 -> execute_each_method_under_declared_protocol
 -> collect_accuracy_runtime_memory_scalability_robustness_failure_and_resource_metrics
 -> aggregate_repeated_runs_and_compute_variability_confidence_and_tradeoffs
 -> assess_benchmark_bias_leakage_overfitting_saturation_and_version_drift
 -> document_all_results_environment_versions_configuration_and_provenance
 -> route_to_related_ontologies
 -> recursively_expand_required_dependencies
 -> verify_result
 -> assign_confidence
 -> produce_semantic_output
 -> END
}

FLOW_HASH {
 METHOD = "semantic-content-identity"
 ID = "KIRETH-MATH-VERIFICATION-VALIDATION-BENCHMARK-ONTOLOGY-V1"
}

}

END_FLOW