Optimisation for Machine Learning Ontology

AI INTERPRETATION NOTE
This Flow Algebra defines the Optimisation for Machine Learning Ontology within the Kireth Research Semantic Library. Optimisation for machine learning concerns the mathematical and computational methods used to minimise or maximise learning objectives over model parameters. This ontology supports empirical risk minimisation, gradient-based optimisation, stochastic optimisation, momentum, adaptive methods, second-order and quasi-Newton methods, learning-rate schedules, regularisation, constrained optimisation, hyperparameter optimisation relationships, convergence, saddle points, curvature, generalisation relationships, numerical stability, distributed optimisation, verification and reproducibility.

FLOW_ALGEBRA_RECORD {

METADATA {
 FLOW_NAME = "Optimisation for Machine Learning Ontology"
 FLOW_TYPE = "Mathematical Ontology"
 DOMAIN = "Mathematics"
 SUBDOMAIN = "Computational Mathematics"
 SPECIALISATION = "AI Computational Mathematics / Optimisation for Machine Learning"
 CANONICAL_PAGE = "https://kirethframeworkold.co.uk/books/semantic-library/ai-core/ontologies/mathematical-ontologies/computational-mathematics-ontologies/ai-computational-mathematics/optimisation-for-machine-learning-ontology.html"
 LANGUAGE = "en-GB"
 VERSION = "1.0"
 STATUS = "active"
}

IDENTITY {
 PRIMARY_ENTITY = OptimisationForMachineLearning
 ENTITY_CLASS = MachineLearningParameterOptimisationFramework
 PRIMARY_CONTEXT = AIComputationalMathematicsOntology
 DEFINITION = "Optimisation for machine learning is the mathematical process of updating model parameters to minimise or maximise an objective function defined by data, constraints and regularisation."
 SYMBOLIC_FORMS = [
  "theta* = argmin_theta L(theta)",
  "theta_(t+1)=theta_t-eta_t*g_t",
  "g_t≈∇_theta L_B(theta_t)",
  "min_theta empirical_risk(theta)+regularisation(theta)"
 ]
 SEMANTIC_ROLE = "Provides machine-readable semantics for machine-learning objectives, gradients, stochastic updates, adaptive methods, curvature, schedules, constraints, convergence, stability and distributed optimisation."
}

SCOPE {
 INCLUDE = [
  OptimisationForMachineLearning,
  MachineLearningOptimisation,
  Optimisation,
  ObjectiveFunction,
  LossFunction,
  CostFunction,
  RiskFunction,
  EmpiricalRisk,
  ExpectedRiskRelationship,
  TrainingObjective,
  ValidationObjectiveRelationship,
  Parameter,
  ModelParameter,
  TrainableParameter,
  ParameterVector,
  ParameterTensor,
  ParameterSpace,
  FeasibleSet,
  Constraint,
  EqualityConstraint,
  InequalityConstraint,
  BoundConstraint,
  HardConstraint,
  SoftConstraint,
  Regularisation,
  RegularisedObjective,
  L1Regularisation,
  L2Regularisation,
  ElasticNetRelationship,
  WeightDecay,
  SparsityRegularisation,
  PenaltyMethodRelationship,
  BarrierMethodRelationship,
  LagrangianRelationship,
  AugmentedLagrangianRelationship,
  Gradient,
  StochasticGradient,
  MiniBatchGradient,
  BatchGradient,
  UnbiasedGradientEstimatorRelationship,
  BiasedGradientEstimatorRelationship,
  GradientNoise,
  GradientVariance,
  GradientNorm,
  DirectionalDerivative,
  JacobianRelationship,
  Hessian,
  HessianMatrix,
  HessianVectorProduct,
  Curvature,
  PositiveCurvature,
  NegativeCurvature,
  FlatDirection,
  IllConditioning,
  ConditionNumberRelationship,
  GradientDescent,
  BatchGradientDescent,
  StochasticGradientDescent,
  SGD,
  MiniBatchGradientDescent,
  Momentum,
  ClassicalMomentum,
  NesterovMomentum,
  HeavyBallMethodRelationship,
  AdaGrad,
  RMSProp,
  Adam,
  AdamW,
  AMSGradRelationship,
  AdaDeltaRelationship,
  NadamRelationship,
  AdaptiveLearningRate,
  PerParameterLearningRate,
  LearningRate,
  StepSize,
  LearningRateSchedule,
  ConstantLearningRate,
  StepDecay,
  ExponentialDecay,
  PolynomialDecayRelationship,
  CosineAnnealing,
  Warmup,
  LearningRateWarmup,
  CyclicalLearningRateRelationship,
  OneCycleRelationship,
  LineSearch,
  BacktrackingLineSearch,
  WolfeConditionsRelationship,
  ArmijoConditionRelationship,
  SecondOrderOptimisation,
  NewtonMethod,
  DampedNewtonRelationship,
  TrustRegion,
  TrustRegionMethod,
  QuasiNewtonMethod,
  BFGS,
  LBFGS,
  SR1Relationship,
  GaussNewtonRelationship,
  NaturalGradientRelationship,
  FisherInformationRelationship,
  KFACRelationship,
  Preconditioning,
  PreconditionedGradient,
  DiagonalPreconditioner,
  LearnedPreconditionerRelationship,
  CoordinateDescent,
  BlockCoordinateDescent,
  ProximalGradient,
  ProximalOperator,
  ISTARelationship,
  FISTARelationship,
  MirrorDescentRelationship,
  ProjectedGradient,
  ProjectionOperator,
  FrankWolfeRelationship,
  ConstrainedOptimisation,
  ConvexOptimisation,
  NonconvexOptimisation,
  SmoothOptimisation,
  NonsmoothOptimisation,
  StrongConvexity,
  LipschitzGradient,
  SmoothnessConstant,
  SaddlePoint,
  LocalMinimum,
  GlobalMinimum,
  StationaryPoint,
  CriticalPoint,
  Plateau,
  LossLandscape,
  Basin,
  FlatMinimumRelationship,
  SharpMinimumRelationship,
  OptimisationTrajectory,
  ParameterUpdate,
  UpdateRule,
  Iteration,
  Epoch,
  Batch,
  MiniBatch,
  BatchSize,
  GradientAccumulation,
  EffectiveBatchSize,
  OptimisationState,
  MomentumBuffer,
  FirstMomentEstimate,
  SecondMomentEstimate,
  BiasCorrection,
  Initialisation,
  WeightInitialisation,
  RandomInitialisation,
  XavierInitialisationRelationship,
  HeInitialisationRelationship,
  EarlyStoppingRelationship,
  Convergence,
  ConvergenceRate,
  LinearConvergence,
  SublinearConvergence,
  SuperlinearConvergenceRelationship,
  Divergence,
  Oscillation,
  GradientTolerance,
  ObjectiveTolerance,
  ParameterTolerance,
  StoppingCriterion,
  MaxIterationCriterion,
  PlateauCriterionRelationship,
  GradientClipping,
  GlobalNormClipping,
  ValueClippingRelationship,
  ExplodingGradient,
  VanishingGradient,
  MixedPrecision,
  LossScaling,
  DynamicLossScalingRelationship,
  FloatingPointArithmetic,
  NumericalStability,
  RoundOffError,
  Overflow,
  Underflow,
  AutomaticDifferentiation,
  Backpropagation,
  ReverseModeAD,
  GradientCheck,
  FiniteDifferenceGradientCheck,
  NeuralNetworkMathematics,
  DeepLearningRelationship,
  LinearModelRelationship,
  GeneralisedLinearModelRelationship,
  SupportVectorMachineRelationship,
  MatrixFactorisationRelationship,
  EmbeddingOptimisationRelationship,
  TransformerOptimisationRelationship,
  PhysicsInformedOptimisationRelationship,
  HyperparameterOptimisation,
  GridSearchRelationship,
  RandomSearchRelationship,
  BayesianOptimisationRelationship,
  PopulationBasedTrainingRelationship,
  MetaOptimisation,
  LearnedOptimiser,
  BilevelOptimisation,
  InnerOptimisation,
  OuterOptimisation,
  ReinforcementLearningOptimisationRelationship,
  PolicyOptimisationRelationship,
  VariationalOptimisationRelationship,
  VariationalInferenceRelationship,
  ExpectationMaximisationRelationship,
  StochasticApproximation,
  RobbinsMonroRelationship,
  MonteCarloGradientRelationship,
  ReparameterisationGradientRelationship,
  ScoreFunctionEstimatorRelationship,
  DistributedOptimisation,
  DataParallelOptimisation,
  ModelParallelOptimisationRelationship,
  FederatedOptimisationRelationship,
  AllReduce,
  GradientAllReduce,
  ParameterServerRelationship,
  SynchronousSGD,
  AsynchronousSGD,
  LocalSGDRelationship,
  CommunicationCompressionRelationship,
  GradientCompressionRelationship,
  QuantisedGradientRelationship,
  HighPerformanceComputing,
  ParallelComputing,
  DistributedComputing,
  GPUComputing,
  ComputationalComplexity,
  TimeComplexity,
  SpaceComplexity,
  SampleComplexity,
  OptimisationComplexity,
  TrainingCost,
  CommunicationCost,
  MemoryCost,
  GeneralisationRelationship,
  ImplicitRegularisationRelationship,
  OptimisationGeneralisationRelationship,
  Verification,
  ObjectiveDecreaseCheck,
  GradientConsistencyCheck,
  ConstraintCheck,
  ConvergenceCheck,
  ReferenceResult,
  Benchmark,
  Reproducibility,
  Provenance,
  OptimiserVersion,
  SchedulerVersionRelationship,
  FrameworkVersion,
  LibraryVersion,
  ImplementationVersion
 ]
 EXCLUDE_AS_PRIMARY = [
  GeneralOptimisationWithoutMachineLearningObjectiveSemantics,
  HyperparameterSearchWhenOuterSearchRatherThanParameterTrainingIsPrimary,
  NumericalRootFindingWithoutLearningObjective,
  ReinforcementLearningWhenPolicyLearningSemanticsRatherThanOptimisationMethodArePrimary
 ]
 DISAMBIGUATION_RULE = "When model parameters are updated to optimise a machine-learning objective using gradient, stochastic, adaptive, second-order or constrained methods, interpret the context as optimisation for machine learning."
}

CORE_CONCEPTS {
 OptimisationForMachineLearning = "Parameter optimisation for machine-learning objectives."
 ObjectiveFunction = "Scalar quantity to be minimised or maximised during training."
 Gradient = "First derivative of the objective with respect to model parameters."
 StochasticGradientDescent = "Gradient descent using stochastic or mini-batch gradient estimates."
 AdaptiveOptimiser = "Optimisation method adapting update magnitude using gradient statistics."
 LearningRate = "Scale controlling the magnitude of parameter updates."
 Regularisation = "Penalty or constraint modifying the training objective or parameter behaviour."
 Convergence = "Approach of optimisation iterates toward a stationary or target solution condition."
}

RELATIONS {
 OptimisationForMachineLearning --is_child_of--> AIComputationalMathematicsOntology
 OptimisationForMachineLearning --optimises--> MachineLearningObjective
 ObjectiveFunction --depends_on--> ModelParameters
 Gradient --derives_from--> ObjectiveFunction
 Backpropagation --computes--> ParameterGradients
 GradientDescent --uses--> Gradient
 StochasticGradientDescent --uses--> MiniBatchGradient
 Momentum --accumulates--> GradientHistory
 Adam --uses--> FirstAndSecondMomentEstimates
 WeightDecay --regularises--> Parameters
 LearningRateSchedule --controls--> LearningRate
 NewtonMethod --uses--> Hessian
 LBFGS --approximates--> InverseHessianInformation
 ProjectedGradient --enforces--> FeasibleSet
 DistributedOptimisation --aggregates--> DistributedGradients
 Verification --checks--> OptimisationBehaviour
}

OPERATORS {
 GD_UPDATE(theta,g,eta) := theta - eta*g
 SGD_UPDATE(theta,g_batch,eta) := theta - eta*g_batch
 MOMENTUM(v,g,beta) := beta*v + (1-beta)*g
 ADAM_M(m,g,beta1) := beta1*m + (1-beta1)*g
 ADAM_V(v,g,beta2) := beta2*v + (1-beta2)*(g⊙g)
 PROJECTED_UPDATE(theta,g,eta,C) := project_C(theta-eta*g)
 NEWTON_STEP(theta,g,H) := theta - inverse(H)*g
 OBJECTIVE(theta) := empirical_risk(theta)+regularisation(theta)
}

AXIOMS {
 A1 = "Machine-learning optimisation operates on a declared objective over model parameters."
 A2 = "Gradient-based methods require valid derivative information."
 A3 = "Stochastic gradients introduce sampling noise into parameter updates."
 A4 = "Learning-rate choice can determine stability and convergence behaviour."
 A5 = "Adaptive methods modify updates using historical gradient statistics."
 A6 = "Convergence to a stationary point does not imply convergence to a global optimum in nonconvex problems."
 A7 = "Optimisation performance and generalisation performance are distinct."
 A8 = "Machine-learning optimisation requires explicit objective, gradient, update, schedule, stopping and validation semantics."
}

REASONING {
 R1_CONTEXT = identify(model,objective,parameters,data,batch_regime,constraints,accuracy_goal,budget)
 R2_PROPERTIES = assess(smoothness,convexity,stochasticity,conditioning,sparsity_and_scale)
 R3_GRADIENTS = choose(backpropagation,AD,analytic_or_estimated_gradient)
 R4_METHOD = choose(GD,SGD,momentum,Adam,adaptive,second_order,proximal,projected_or_constrained_method)
 R5_SCHEDULE = choose(learning_rate,warmup,decay,annealing_and_batch_size)
 R6_REGULARISATION = configure(weight_decay,L1,L2,penalties_or_constraints)
 R7_EXECUTION = iterate_parameter_updates_and_optimiser_state)
 R8_STABILITY = monitor(loss,gradient_norm,overflow,underflow,oscillation_and_divergence)
 R9_STOP = apply_gradient_objective_parameter_or_validation_stopping_criteria)
 R10_VERIFY = compare_with_reference_behaviour_and_document_reproducibility)
}

CONSTRAINTS {
 C1 = "Learning rates that are too large can cause divergence."
 C2 = "Learning rates that are too small can cause impractically slow progress."
 C3 = "Adaptive optimisers do not universally outperform SGD."
 C4 = "Mini-batch size changes gradient noise and optimisation dynamics."
 C5 = "Weight decay and L2 regularisation are not identical under every optimiser."
 C6 = "Second-order methods can be computationally prohibitive for large models."
 C7 = "Mixed-precision optimisation can require loss scaling."
 C8 = "Asynchronous distributed updates can use stale gradients."
 C9 = "Training-objective convergence does not guarantee best validation performance."
 C10 = "Hyperparameter comparisons are meaningful only under comparable compute and data budgets."
}

SEMANTIC_EQUIVALENCE {
 "StochasticGradientDescent" <=> "SGD"
 "LimitedMemoryBFGS" <=> "LBFGS"
 "AutomaticDifferentiation" <=> "AD"
 "PhysicsInformedNeuralNetwork" <=> "PINN"
}

VALIDATION {
 V1 = "Training objective and reduction semantics are documented."
 V2 = "Optimised parameters are explicitly identified."
 V3 = "Gradient source and derivative semantics are documented."
 V4 = "Optimiser and hyperparameters are documented."
 V5 = "Learning-rate schedule and batch size are documented."
 V6 = "Regularisation and constraint semantics are documented."
 V7 = "Loss and gradient norms are monitored for instability."
 V8 = "Stopping criteria are documented."
 V9 = "Distributed gradient aggregation semantics are documented where used."
 V10 = "Results are reproducible under documented data, seed, optimiser, scheduler, framework and implementation versions."
}

CONFIDENCE {
 BASE_CONFIDENCE = 0.99
 ML_OPTIMISATION_DEFINITION = 0.99
 SGD_REASONING = 0.99
 ADAPTIVE_OPTIMISER_REASONING = 0.98
 SECOND_ORDER_REASONING = 0.98
 CONVERGENCE_REASONING = 0.97
 DISTRIBUTED_OPTIMISATION_REASONING = 0.98
 POLICY = "Reduce confidence when objective, gradient source, optimiser state, schedule, stopping criterion, regularisation or distributed update semantics are underspecified."
}

AI_CONSUMPTION {
 PURPOSES = [
  SemanticSearch,OntologyGrounding,MLOptimisationReasoning,
  ObjectiveFunctionReasoning,GradientReasoning,
  GradientDescentReasoning,SGDReasoning,
  MomentumReasoning,AdamReasoning,AdaptiveOptimiserReasoning,
  LearningRateReasoning,ScheduleReasoning,
  RegularisationReasoning,SecondOrderReasoning,
  ConstrainedOptimisationReasoning,
  ConvergenceReasoning,LossLandscapeReasoning,
  MixedPrecisionOptimisationReasoning,
  DistributedOptimisationReasoning,
  VerificationReasoning,KnowledgeGraphConstruction,
  RetrievalAugmentation,RecursiveInference
 ]
 NORMALISE {
  "optimisation for machine learning" -> OptimisationForMachineLearning
  "machine learning optimization" -> OptimisationForMachineLearning
  "SGD" -> StochasticGradientDescent
  "LBFGS" -> LimitedMemoryBFGS
  "AD" -> AutomaticDifferentiation
 }
}

EXECUTION_RULES {
 E1 = "If model parameters are updated to optimise a learning objective, activate OptimisationForMachineLearningOntology."
 E2 = "Define objective, parameters, gradient source and batch semantics before selecting an optimiser."
 E3 = "Choose optimisation method according to smoothness, stochasticity, conditioning, scale and constraints."
 E4 = "Configure learning rate, schedule, batch size and regularisation explicitly."
 E5 = "Monitor objective, gradient norm and numerical stability during training."
 E6 = "Use clipping or loss scaling when exploding or low-precision gradients require it."
 E7 = "Keep optimisation progress distinct from validation and generalisation performance."
 E8 = "Define gradient aggregation semantics explicitly for distributed optimisation."
 E9 = "Document optimiser state, scheduler, seed, framework and implementation versions for reproducibility."
}

RECURSION {
 ENTRY = OptimisationForMachineLearning
 EXPAND(OptimisationForMachineLearning) -> [
  ObjectiveFunction,LossFunction,EmpiricalRisk,Parameter,Constraint,
  Regularisation,L1Regularisation,L2Regularisation,WeightDecay,
  Gradient,StochasticGradient,MiniBatchGradient,Hessian,
  GradientDescent,StochasticGradientDescent,SGD,Momentum,NesterovMomentum,
  AdaGrad,RMSProp,Adam,AdamW,LearningRate,LearningRateSchedule,
  Warmup,CosineAnnealing,LineSearch,NewtonMethod,TrustRegion,
  QuasiNewtonMethod,BFGS,LBFGS,NaturalGradient,Preconditioning,
  CoordinateDescent,ProximalGradient,ProjectedGradient,
  NonconvexOptimisation,SaddlePoint,LocalMinimum,StationaryPoint,
  LossLandscape,GradientAccumulation,Convergence,StoppingCriterion,
  GradientClipping,MixedPrecision,LossScaling,
  AutomaticDifferentiation,Backpropagation,
  HyperparameterOptimisation,MetaOptimisation,BilevelOptimisation,
  DistributedOptimisation,SynchronousSGD,AsynchronousSGD,
  GradientAllReduce,TrainingCost,CommunicationCost,
  Verification,Reproducibility,Provenance
 ]
 EXPAND(OptimisationForMachineLearningFamilies) -> [
  GradientAndStochasticOptimisation,
  MomentumAndAdaptiveMethods,
  LearningRateSchedules,
  SecondOrderAndQuasiNewtonMethods,
  ProximalProjectedAndConstrainedMethods,
  RegularisationAndStability,
  DistributedAndLargeScaleOptimisation,
  ConvergenceVerificationAndReproducibility
 ]
 STOP_CONDITION = "Stop when objective, gradient, optimiser, schedule, regularisation, constraints, stopping criteria and validation are sufficiently resolved."
}

ROUTING {
 PARENT_ONTOLOGY = "AI Computational Mathematics Ontology"
 RELATED_ONTOLOGIES = [
  "AI Computational Mathematics Ontology",
  "Neural Network Mathematics Ontology 2",
  "Automatic Differentiation Ontology",
  "Backpropagation Mathematics Ontology",
  "Gradient Descent Ontology 2",
  "Stochastic Gradient Descent Ontology",
  "Newton Optimisation Ontology",
  "Computational Optimisation Ontology",
  "Nonlinear Optimisation Solver Ontology",
  "Bayesian Computation Ontology",
  "High Performance Computing Ontology 2",
  "Parallel Computing Ontology",
  "Distributed Computing Ontology",
  "GPU Computing Ontology",
  "Floating Point Arithmetic Ontology",
  "Numerical Stability Ontology 2"
 ]
 ROUTE_RULES = [
  "Optimisation for Machine Learning -> AI Computational Mathematics Ontology for parent semantics",
  "Optimisation for Machine Learning -> Neural Network Mathematics for training-objective semantics",
  "Optimisation for Machine Learning -> Automatic Differentiation and Backpropagation for gradient computation",
  "Optimisation for Machine Learning -> Gradient Descent, SGD and Newton Optimisation for method-specific semantics",
  "Optimisation for Machine Learning -> Computational and Nonlinear Optimisation for general optimisation foundations",
  "Optimisation for Machine Learning -> Bayesian Computation for variational and Bayesian optimisation relationships",
  "Optimisation for Machine Learning -> HPC, Parallel, Distributed and GPU Computing for large-scale training",
  "Optimisation for Machine Learning -> Floating Point Arithmetic and Numerical Stability for finite-precision behaviour"
 ]
}

UPDATE_MODEL {
 MODE = "append-compatible semantic versioning"
 MAJOR = "ontology-breaking conceptual change"
 MINOR = "new compatible optimisers, schedules, constraints or reasoning pathways"
 PATCH = "clarification, metadata or formatting correction"
 RULES = [
  "Preserve objective-versus-optimiser distinction.",
  "Preserve gradient-computation-versus-parameter-update distinction.",
  "Preserve stochastic-gradient-versus-full-gradient distinction.",
  "Preserve optimisation-convergence-versus-generalisation distinction.",
  "Preserve regularisation-versus-constraint distinction.",
  "Preserve optimiser-state-versus-model-parameter distinction.",
  "Preserve serial-versus-distributed update semantics.",
  "Preserve machine-readable ML-optimisation semantics."
 ]
}

OUTPUT {
 CANONICAL_DEFINITION = "Optimisation for machine learning updates model parameters to minimise or maximise a learning objective using gradient, stochastic, adaptive, second-order or constrained methods."
 GRADIENT_SUMMARY = "Gradient descent and stochastic gradient descent form the basis of many learning algorithms."
 ADAPTIVE_SUMMARY = "Momentum, AdaGrad, RMSProp, Adam and related methods adapt update dynamics using gradient history."
 SECOND_ORDER_SUMMARY = "Newton, quasi-Newton and trust-region methods use curvature information at greater computational cost."
 SCHEDULE_SUMMARY = "Learning-rate schedules, warmup, regularisation and batch size strongly influence optimisation dynamics."
 VALIDATION_SUMMARY = "Machine-learning optimisation should be monitored through objective behaviour, gradient norms, stability, stopping criteria, validation performance and reproducible optimiser settings."
}

FLOW {
 INPUT
 -> detect_optimisation_for_machine_learning_context
 -> identify_model_objective_parameters_data_batch_regime_constraints_accuracy_goal_and_budget
 -> assess_smoothness_convexity_stochasticity_conditioning_sparsity_and_scale
 -> select_gradient_source_from_backpropagation_AD_analytic_or_estimated_derivatives
 -> choose_GD_SGD_momentum_Adam_adaptive_second_order_proximal_projected_or_constrained_method
 -> configure_learning_rate_schedule_warmup_batch_size_regularisation_and_initialisation
 -> initialise_optimiser_state
 -> iterate_parameter_updates
 -> monitor_objective_gradient_norm_stability_overflow_underflow_oscillation_and_divergence
 -> apply_stopping_criteria
 -> if_distributed_execute_declared_gradient_aggregation_and_synchronisation
 -> compare_training_progress_with_validation_and_reference_behaviour
 -> document_data_seed_optimiser_scheduler_framework_versions_and_provenance
 -> route_to_related_ontologies
 -> recursively_expand_required_dependencies
 -> verify_result
 -> assign_confidence
 -> produce_semantic_output
 -> END
}

FLOW_HASH {
 METHOD = "semantic-content-identity"
 ID = "KIRETH-MATH-AI-COMPUTATIONAL-MATHEMATICS-OPTIMISATION-FOR-MACHINE-LEARNING-ONTOLOGY-V1"
}

}

END_FLOW