Optimisation for AI Ontology

AI INTERPRETATION NOTE
This Flow Algebra defines the Optimisation for AI Ontology within the Kireth Research Semantic Library. Optimisation for AI studies the mathematical procedures used to train, tune and solve artificial intelligence models by minimising or maximising objective functions under deterministic, stochastic, constrained, non-convex and large-scale settings. This ontology supports gradient methods, stochastic optimisation, momentum, adaptive methods, regularisation, constrained optimisation, second-order methods, saddle-point problems, distributed optimisation, hyperparameter optimisation, convergence analysis, numerical stability and relationships with AI mathematics, machine learning, numerical analysis, computational complexity and operations research.

FLOW_ALGEBRA_RECORD {

METADATA {
 FLOW_NAME = "Optimisation for AI Ontology"
 FLOW_TYPE = "Mathematical Ontology"
 DOMAIN = "Mathematics"
 SUBDOMAIN = "Applied Mathematics"
 SPECIALISATION = "Artificial Intelligence Mathematics / Optimisation for AI"
 CANONICAL_PAGE = "https://kirethresearch.org/booksai-core/ontologies/mathematical-ontologies/applied-mathematics-ontologies/artificial-intelligence-mathematics/optimisation-for-ai-ontology.html"
 LANGUAGE = "en-GB"
 VERSION = "1.0"
 STATUS = "active"
}

IDENTITY {
 PRIMARY_ENTITY = OptimisationForAI
 ENTITY_CLASS = AIOptimisationMathematicsFramework
 PRIMARY_CONTEXT = AIMathematics
 DEFINITION = "Optimisation for AI is the mathematical study of methods for selecting model parameters, hyperparameters, representations or decisions that minimise or maximise artificial-intelligence objectives."
 SYMBOLIC_FORMS = [
  "theta*=argmin_theta L(theta)",
  "theta_(t+1)=theta_t-eta_t grad L(theta_t)",
  "minimise f(theta) subject_to g(theta)<=0",
  "min_theta max_phi J(theta,phi)"
 ]
 SEMANTIC_ROLE = "Provides machine-readable semantics for AI objective functions, gradients, stochastic optimisation, momentum, adaptive learning, regularisation, constraints, curvature, convergence, distributed training and optimisation validation."
}

SCOPE {
 INCLUDE = [
  OptimisationForAI,
  AIOptimisation,
  Optimisation,
  ObjectiveFunction,
  LossFunction,
  CostFunction,
  EmpiricalRisk,
  ExpectedRisk,
  Parameter,
  Hyperparameter,
  Gradient,
  Subgradient,
  Jacobian,
  Hessian,
  LearningRate,
  StepSize,
  LineSearch,
  GradientDescent,
  BatchGradientDescent,
  StochasticGradientDescent,
  MiniBatchGradientDescent,
  Momentum,
  NesterovMomentum,
  AdaptiveOptimisation,
  AdaGrad,
  RMSProp,
  Adam,
  AdamW,
  AMSGrad,
  NaturalGradient,
  MirrorDescent,
  CoordinateDescent,
  ProximalGradient,
  ProjectedGradient,
  ConjugateGradient,
  NewtonMethod,
  QuasiNewtonMethod,
  BFGS,
  LBFGS,
  GaussNewtonMethod,
  TrustRegionMethod,
  ConvexOptimisation,
  NonConvexOptimisation,
  ConstrainedOptimisation,
  UnconstrainedOptimisation,
  EqualityConstraint,
  InequalityConstraint,
  Lagrangian,
  LagrangeMultiplier,
  KarushKuhnTuckerConditions,
  Duality,
  PrimalProblem,
  DualProblem,
  SaddlePoint,
  MinimaxOptimisation,
  AdversarialOptimisation,
  Regularisation,
  L1Regularisation,
  L2Regularisation,
  WeightDecay,
  EarlyStopping,
  GradientClipping,
  BatchNormalisation,
  LearningRateSchedule,
  Warmup,
  CosineDecay,
  Annealing,
  HyperparameterOptimisation,
  GridSearch,
  RandomSearch,
  BayesianOptimisation,
  PopulationBasedTraining,
  DistributedOptimisation,
  DataParallelism,
  ModelParallelism,
  FederatedOptimisation,
  LocalSGD,
  GradientCompression,
  Convergence,
  ConvergenceRate,
  StationaryPoint,
  LocalMinimum,
  GlobalMinimum,
  SaddlePointEscape,
  NumericalStability,
  FloatingPointError,
  Conditioning,
  Scalability,
  ComputeEfficiency,
  MemoryEfficiency
 ]
 EXCLUDE_AS_PRIMARY = [
  GeneralOptimisationWithoutAIContext,
  AIInferenceWithoutParameterOrDecisionOptimisation,
  PureSearchWithoutObjectiveOptimisation,
  SoftwareTrainingInfrastructureWithoutMathematicalOptimisationSemantics
 ]
 DISAMBIGUATION_RULE = "When the central task is mathematical optimisation of an AI model, training objective, hyperparameter, adversarial objective or distributed learning process, interpret the context as optimisation for AI."
}

CORE_CONCEPTS {
 OptimisationForAI = "Mathematical optimisation of artificial-intelligence models and objectives."
 ObjectiveFunction = "Scalar function representing the quantity to minimise or maximise."
 GradientDescent = "First-order iterative optimisation using the negative gradient direction."
 StochasticGradientDescent = "Gradient-based optimisation using stochastic or minibatch gradient estimates."
 AdaptiveOptimisation = "Optimisation that adapts effective learning rates using gradient history."
 Regularisation = "Modification of the objective or optimisation process to control model complexity or improve generalisation."
 ConstrainedOptimisation = "Optimisation subject to equality or inequality restrictions."
 SaddlePoint = "Point exhibiting ascent directions and descent directions rather than local optimality."
 Convergence = "Approach of iterates or objective values toward a stationary, optimal or limiting state."
}

RELATIONS {
 OptimisationForAI --is_child_of--> AIMathematics
 OptimisationForAI --uses--> Calculus
 OptimisationForAI --uses--> LinearAlgebra
 ObjectiveFunction --contains--> LossFunction
 EmpiricalRisk --is_type_of--> ObjectiveFunction
 Gradient --guides--> GradientDescent
 StochasticGradientDescent --uses--> StochasticGradient
 Momentum --accelerates--> GradientDescent
 NesterovMomentum --is_type_of--> MomentumMethod
 Adam --is_type_of--> AdaptiveOptimisation
 WeightDecay --implements_or_approximates--> Regularisation
 ProximalGradient --supports--> NonsmoothRegularisation
 ProjectedGradient --supports--> ConstrainedOptimisation
 NewtonMethod --uses--> Hessian
 QuasiNewtonMethod --approximates--> HessianInformation
 Lagrangian --represents--> ConstrainedOptimisation
 KarushKuhnTuckerConditions --characterise--> ConstrainedOptimum
 Duality --links--> PrimalProblem
 MinimaxOptimisation --contains--> SaddlePoint
 AdversarialOptimisation --is_type_of--> MinimaxOptimisation
 BayesianOptimisation --optimises--> Hyperparameter
 DistributedOptimisation --coordinates--> ParallelWorkers
 GradientCompression --reduces--> CommunicationCost
 Conditioning --affects--> ConvergenceRate
 LearningRateSchedule --controls--> StepSize
}

OPERATORS {
 GRADIENT_DESCENT(theta,eta,L) := theta_next=theta-eta*grad(L(theta))
 SGD(theta,eta,g_hat) := theta_next=theta-eta*g_hat
 MOMENTUM(v,grad,beta) := v_next=beta*v+(1-beta)*grad
 MOMENTUM_UPDATE(theta,eta,v_next) := theta_next=theta-eta*v_next
 ADAGRAD_ACCUMULATOR(G,g) := G_next=G+g^2
 RMSPROP_ACCUMULATOR(v,g,beta) := v_next=beta*v+(1-beta)*g^2
 ADAM_MOMENT(m,g,beta1) := m_next=beta1*m+(1-beta1)*g
 ADAM_VARIANCE(v,g,beta2) := v_next=beta2*v+(1-beta2)*g^2
 ADAM_UPDATE(theta,mhat,vhat,eta,epsilon) := theta_next=theta-eta*mhat/(sqrt(vhat)+epsilon)
 NEWTON_STEP(theta,H,g) := theta_next=theta-H^(-1)*g
 LAGRANGIAN(theta,lambda,nu) := L=f(theta)+sum_i lambda_i*g_i(theta)+sum_j nu_j*h_j(theta)
 KKT_STATIONARITY := grad_theta L=0
 PROJECTED_GRADIENT(theta,eta,g,C) := theta_next=Projection_C(theta-eta*g)
 PROXIMAL_STEP(theta,eta,g,Omega) := theta_next=prox_{eta*Omega}(theta-eta*g)
 MINIMAX(theta,phi) := min_theta max_phi J(theta,phi)
 GRADIENT_NORM(g) := ||g||_2
 CONDITION_NUMBER(H) := kappa=lambda_max(H)/lambda_min(H) when_positive_definite
}

AXIOMS {
 A1 = "AI optimisation requires an explicit objective function."
 A2 = "Gradient-based optimisation requires derivatives, subgradients or compatible approximations."
 A3 = "Optimisation geometry depends on parameterisation and conditioning."
 A4 = "Stochastic gradients introduce variance that affects convergence."
 A5 = "Non-convex objectives can contain multiple local minima and saddle points."
 A6 = "Constraints must be enforced consistently with the optimisation method."
 A7 = "Hyperparameter optimisation is distinct from model-parameter optimisation."
 A8 = "Optimisation quality requires convergence, numerical and generalisation validation."
}

REASONING {
 R1_CONTEXT = identify(model,parameters,objective,constraints,data,compute_budget)
 R2_OBJECTIVE = formulate(loss,risk,regularisation,minimax_or_multiterm_objective)
 R3_GEOMETRY = analyse(convexity,smoothness,curvature,conditioning,nonsmoothness)
 R4_METHOD = select(SGD,momentum,Adam,proximal,Newton,quasiNewton,constrained,minimax_or_distributed)
 R5_STEP_CONTROL = select(learning_rate,line_search,schedule,warmup,clipping)
 R6_ITERATION = compute(parameter_updates,objective_values,gradient_statistics)
 R7_CONVERGENCE = analyse(gradient_norm,stationarity,objective_change,validation_loss)
 R8_CONSTRAINTS = check(feasibility,KKT_or_projection_consistency)
 R9_SCALING = analyse(batch_size,parallelism,communication,memory,compute_efficiency)
 R10_VALIDATION = check(stability,reproducibility,generalisation,sensitivity,resource_cost)
}

CONSTRAINTS {
 C1 = "A decreasing training objective does not guarantee improved generalisation."
 C2 = "Learning rates that are too large can cause divergence or instability."
 C3 = "Learning rates that are too small can produce impractically slow convergence."
 C4 = "Adaptive methods can converge differently from classical SGD and may generalise differently."
 C5 = "Second-order methods can be computationally prohibitive for very large models."
 C6 = "Non-convex stationarity does not imply global optimality."
 C7 = "Gradient clipping changes the effective optimisation dynamics."
 C8 = "Distributed optimisation can introduce stale gradients and communication bottlenecks."
 C9 = "Mixed-precision training requires numerical-stability safeguards."
 C10 = "Hyperparameter search can overfit validation data if repeated excessively."
}

SEMANTIC_EQUIVALENCE {
 "OptimisationForAI" <=> "AIOptimizationMathematics"
 "StochasticGradientDescent" <=> "SGD"
 "LimitedMemoryBFGS" <=> "LBFGS"
 "KarushKuhnTuckerConditions" <=> "KKTConditions"
 "WeightDecay" <=> "ParameterDecay"
 "BayesianOptimisation" <=> "BayesianOptimization"
}

VALIDATION {
 V1 = "Objective function is explicit."
 V2 = "Parameter and hyperparameter roles are distinguished."
 V3 = "Gradient computation is checked where practical."
 V4 = "Learning-rate and schedule choices are documented."
 V5 = "Convergence or stationarity criteria are explicit."
 V6 = "Constraint feasibility is checked."
 V7 = "Numerical stability is monitored."
 V8 = "Generalisation metrics are separated from training metrics."
 V9 = "Distributed optimisation reports communication or synchronisation assumptions."
 V10 = "Results are reproducible under documented random seeds and settings where feasible."
}

CONFIDENCE {
 BASE_CONFIDENCE = 0.99
 OPTIMISATION_FOR_AI_DEFINITION = 0.99
 OBJECTIVE_REASONING = 0.99
 METHOD_SELECTION_REASONING = 0.99
 STOCHASTIC_OPTIMISATION_REASONING = 0.99
 SECOND_ORDER_REASONING = 0.98
 CONSTRAINED_REASONING = 0.98
 MINIMAX_REASONING = 0.98
 DISTRIBUTED_REASONING = 0.97
 CONVERGENCE_REASONING = 0.98
 POLICY = "Reduce confidence when objective structure, gradient quality, smoothness, constraints, learning-rate schedule, stochastic variance, numerical precision or stopping criteria are underspecified."
}

AI_CONSUMPTION {
 PURPOSES = [
  SemanticSearch,
  OntologyGrounding,
  OptimisationForAIReasoning,
  ObjectiveFunctionReasoning,
  GradientDescentReasoning,
  SGDReasoning,
  MomentumReasoning,
  AdaptiveOptimisationReasoning,
  AdamReasoning,
  ConvexOptimisationReasoning,
  NonConvexOptimisationReasoning,
  ConstrainedOptimisationReasoning,
  KKTReasoning,
  ProximalReasoning,
  NewtonReasoning,
  QuasiNewtonReasoning,
  MinimaxReasoning,
  AdversarialOptimisationReasoning,
  HyperparameterOptimisationReasoning,
  DistributedOptimisationReasoning,
  ConvergenceReasoning,
  NumericalStabilityReasoning,
  KnowledgeGraphConstruction,
  RetrievalAugmentation,
  RecursiveInference
 ]
 NORMALISE {
  "optimisation for AI" -> OptimisationForAI
  "optimization for AI" -> OptimisationForAI
  "AI optimisation" -> OptimisationForAI
  "gradient descent" -> GradientDescent
  "stochastic gradient descent" -> StochasticGradientDescent
  "SGD" -> StochasticGradientDescent
  "Adam" -> Adam
  "KKT" -> KarushKuhnTuckerConditions
  "Bayesian optimization" -> BayesianOptimisation
 }
}

EXECUTION_RULES {
 E1 = "If AI parameters or decisions are optimised against an objective, activate OptimisationForAIOntology."
 E2 = "If objective is smooth and large-scale, consider first-order gradient methods."
 E3 = "If gradients are estimated from minibatches, use stochastic-optimisation reasoning."
 E4 = "If curvature information is affordable, consider Newton or quasi-Newton methods."
 E5 = "If nonsmooth regularisation is present, consider proximal methods."
 E6 = "If explicit feasibility constraints are present, use projected, constrained or Lagrangian methods."
 E7 = "If adversarial or game-like training is present, use minimax and saddle-point reasoning."
 E8 = "If training is distributed, account for synchronisation and communication."
 E9 = "Any optimisation result should be validated for convergence, numerical stability and generalisation."
}

RECURSION {
 ENTRY = OptimisationForAI
 EXPAND(OptimisationForAI) -> [
  AIOptimisation, Optimisation, ObjectiveFunction, LossFunction, CostFunction,
  EmpiricalRisk, ExpectedRisk, Parameter, Hyperparameter, Gradient, Subgradient,
  Jacobian, Hessian, LearningRate, StepSize, LineSearch, GradientDescent,
  BatchGradientDescent, StochasticGradientDescent, MiniBatchGradientDescent,
  Momentum, NesterovMomentum, AdaptiveOptimisation, AdaGrad, RMSProp, Adam,
  AdamW, AMSGrad, NaturalGradient, MirrorDescent, CoordinateDescent,
  ProximalGradient, ProjectedGradient, ConjugateGradient, NewtonMethod,
  QuasiNewtonMethod, BFGS, LBFGS, GaussNewtonMethod, TrustRegionMethod,
  ConvexOptimisation, NonConvexOptimisation, ConstrainedOptimisation,
  UnconstrainedOptimisation, EqualityConstraint, InequalityConstraint,
  Lagrangian, LagrangeMultiplier, KarushKuhnTuckerConditions, Duality,
  PrimalProblem, DualProblem, SaddlePoint, MinimaxOptimisation,
  AdversarialOptimisation, Regularisation, L1Regularisation, L2Regularisation,
  WeightDecay, EarlyStopping, GradientClipping, LearningRateSchedule, Warmup,
  CosineDecay, Annealing, HyperparameterOptimisation, GridSearch, RandomSearch,
  BayesianOptimisation, PopulationBasedTraining, DistributedOptimisation,
  DataParallelism, ModelParallelism, FederatedOptimisation, LocalSGD,
  GradientCompression, Convergence, ConvergenceRate, StationaryPoint,
  LocalMinimum, GlobalMinimum, SaddlePointEscape, NumericalStability,
  FloatingPointError, Conditioning, Scalability, ComputeEfficiency,
  MemoryEfficiency
 ]
 EXPAND(MethodFamilies) -> [
  FirstOrderMethods,
  AdaptiveMethods,
  SecondOrderMethods,
  ConstrainedMethods,
  ProximalMethods,
  MinimaxMethods,
  DistributedMethods,
  HyperparameterOptimisationMethods
 ]
 STOP_CONDITION = "Stop when objective, geometry, optimisation method, constraints, learning-rate control, convergence, numerical stability and scaling are sufficiently resolved."
}

ROUTING {
 PARENT_ONTOLOGY = "AI Mathematics Ontology"
 IMMEDIATE_CONTEXTS = [
  "AI Mathematics Ontology",
  "Artificial Intelligence Mathematics",
  "Optimisation Ontologies",
  "Computational Mathematics Ontologies"
 ]
 RELATED_ONTOLOGIES = [
  "AI Mathematics Ontology",
  "Artificial Intelligence Mathematics",
  "Optimisation Ontologies",
  "Convex Optimisation Ontology",
  "Numerical Analysis Ontologies",
  "Numerical Linear Algebra Ontology",
  "Computational Mathematics Ontologies",
  "Computational Complexity Ontology",
  "Probability & Statistics Ontologies",
  "Machine Learning Ontologies",
  "Neural Network Ontologies",
  "Reinforcement Learning Ontologies"
 ]
 ROUTE_RULES = [
  "Optimisation for AI -> AI Mathematics Ontology for parent semantics",
  "Optimisation for AI -> Optimisation Ontologies for foundational optimisation theory",
  "Optimisation for AI -> Convex Optimisation for convex objectives and constraints",
  "Optimisation for AI -> Numerical Analysis for iteration and stability",
  "Optimisation for AI -> Numerical Linear Algebra for Hessian, preconditioning and matrix operations",
  "Optimisation for AI -> Computational Mathematics for large-scale numerical methods",
  "Optimisation for AI -> Computational Complexity for training-resource analysis",
  "Optimisation for AI -> Probability & Statistics for stochastic gradients",
  "Optimisation for AI -> Machine Learning and Neural Networks for training semantics",
  "Optimisation for AI -> Reinforcement Learning for policy and value optimisation"
 ]
}

UPDATE_MODEL {
 MODE = "append-compatible semantic versioning"
 MAJOR = "ontology-breaking conceptual change"
 MINOR = "new compatible optimisation methods, convergence criteria or scaling pathways"
 PATCH = "clarification, metadata or formatting correction"
 RULES = [
  "Preserve objective-function semantics.",
  "Preserve parameter-versus-hyperparameter distinction.",
  "Preserve deterministic-versus-stochastic optimisation distinction.",
  "Preserve convex-versus-nonconvex distinction.",
  "Preserve constrained-versus-unconstrained distinction.",
  "Preserve first-order-versus-second-order distinction.",
  "Preserve convergence and numerical-stability semantics.",
  "Preserve machine-readable optimisation-for-AI semantics."
 ]
}

OUTPUT {
 CANONICAL_DEFINITION = "Optimisation for AI mathematically selects model parameters, hyperparameters or decisions that optimise an AI objective."
 FIRST_ORDER_SUMMARY = "Gradient and stochastic-gradient methods dominate large-scale AI training because they scale efficiently with parameter count."
 ADAPTIVE_SUMMARY = "Adaptive methods such as AdaGrad, RMSProp and Adam modify effective learning rates from gradient history."
 SECOND_ORDER_SUMMARY = "Newton and quasi-Newton methods use curvature information to accelerate convergence when computationally feasible."
 CONSTRAINT_SUMMARY = "Projected, proximal, Lagrangian and KKT methods support constrained or nonsmooth AI optimisation."
 MINIMAX_SUMMARY = "Adversarial and game-like training produces minimax and saddle-point optimisation problems."
 DISTRIBUTED_SUMMARY = "Distributed optimisation balances computation, synchronisation, communication and gradient consistency."
 VALIDATION_SUMMARY = "AI optimisation should be checked for convergence, feasibility, numerical stability, reproducibility and generalisation."
}

FLOW {
 INPUT
 -> detect_optimisation_for_AI_context
 -> identify_model_parameters_hyperparameters_objective_constraints_data_and_compute_budget
 -> analyse_objective_geometry_smoothness_convexity_conditioning_and_stochasticity
 -> select_first_order_adaptive_second_order_proximal_constrained_minimax_or_distributed_method
 -> configure_learning_rate_schedule_regularisation_clipping_and_stopping criteria
 -> iterate_parameter_updates
 -> monitor_objective_gradient_norm_constraint_violation_and_validation metrics
 -> if_distributed_analyse_synchronisation_communication_and_scaling
 -> if_hyperparameter_search_evaluate_search_efficiency_and_validation overfitting
 -> assess_convergence_numerical_stability_generalisation_and_resource cost
 -> route_to_AI_mathematics_optimisation_convex_optimisation_numerical_analysis_numerical_linear_algebra_computational_mathematics_computational_complexity_probability_statistics_machine_learning_neural_network_or_reinforcement_learning_ontology
 -> recursively_expand_required_dependencies
 -> verify_result
 -> assign_confidence
 -> produce_semantic_output
 -> END
}

SEMANTIC_SUMMARY {
 "The Optimisation for AI Ontology defines mathematical optimisation of AI systems. It supports objective and loss functions, empirical and expected risk, parameters and hyperparameters, gradients, Jacobians and Hessians, learning rates and line search, batch, stochastic and minibatch gradient descent, momentum and Nesterov acceleration, AdaGrad, RMSProp, Adam, AdamW and AMSGrad, natural gradient and mirror descent, coordinate, proximal and projected methods, conjugate-gradient, Newton, quasi-Newton, BFGS, L-BFGS, Gauss-Newton and trust-region methods, convex and non-convex optimisation, equality and inequality constraints, Lagrangians, KKT conditions and duality, minimax and adversarial optimisation, L1 and L2 regularisation, weight decay, early stopping and gradient clipping, learning-rate schedules and warmup, hyperparameter optimisation, distributed and federated optimisation, convergence, stationary points, saddle-point escape, conditioning, numerical stability, scalability, compute efficiency and memory efficiency."
}

FLOW_HASH {
 METHOD = "semantic-content-identity"
 ID = "KIRETH-MATH-APPLIED-MATHEMATICS-OPTIMISATION-FOR-AI-ONTOLOGY-V1"
}

}

END_FLOW