Gradient Optimisation Ontology

AI INTERPRETATION NOTE
This Flow Algebra defines the Gradient Optimisation Ontology within the Kireth Research Semantic Library. Gradient optimisation studies mathematical methods that use first-order derivative information to minimise or maximise objective functions. This ontology supports gradient descent, stochastic and minibatch gradients, momentum, adaptive learning rates, line search, step-size control, convergence, conditioning, convex and non-convex objectives, projected and proximal updates, gradient clipping, numerical stability and relationships with optimisation for AI, neural network mathematics, calculus, numerical analysis and computational complexity.

FLOW_ALGEBRA_RECORD {

METADATA {
 FLOW_NAME = "Gradient Optimisation Ontology"
 FLOW_TYPE = "Mathematical Ontology"
 DOMAIN = "Mathematics"
 SUBDOMAIN = "Applied Mathematics"
 SPECIALISATION = "Artificial Intelligence Mathematics / Gradient Optimisation"
 CANONICAL_PAGE = "https://kirethresearch.org/booksai-core/ontologies/mathematical-ontologies/applied-mathematics-ontologies/artificial-intelligence-mathematics/gradient-optimisation-ontology.html"
 LANGUAGE = "en-GB"
 VERSION = "1.0"
 STATUS = "active"
}

IDENTITY {
 PRIMARY_ENTITY = GradientOptimisation
 ENTITY_CLASS = FirstOrderOptimisationFramework
 PRIMARY_CONTEXT = OptimisationForAI
 DEFINITION = "Gradient optimisation is the mathematical study of optimisation methods that update variables using first-order derivative information from an objective function."
 SYMBOLIC_FORMS = [
  "theta_(t+1)=theta_t-eta_t grad f(theta_t)",
  "g_t=grad f(theta_t)",
  "theta*=argmin_theta f(theta)",
  "||grad f(theta)|| -> 0"
 ]
 SEMANTIC_ROLE = "Provides machine-readable semantics for gradient directions, step-size selection, stochastic gradients, momentum, adaptive updates, constrained first-order methods, convergence and numerical stability."
}

SCOPE {
 INCLUDE = [
  GradientOptimisation,
  GradientOptimization,
  Gradient,
  ObjectiveFunction,
  LossFunction,
  CostFunction,
  ParameterVector,
  DirectionalDerivative,
  SteepestDescent,
  GradientDescent,
  BatchGradientDescent,
  StochasticGradientDescent,
  MiniBatchGradientDescent,
  StochasticGradient,
  GradientEstimator,
  UnbiasedGradientEstimator,
  BiasedGradientEstimator,
  LearningRate,
  StepSize,
  FixedStepSize,
  DiminishingStepSize,
  AdaptiveStepSize,
  LineSearch,
  BacktrackingLineSearch,
  ArmijoCondition,
  WolfeConditions,
  Momentum,
  HeavyBallMethod,
  NesterovMomentum,
  AdaptiveGradientMethod,
  AdaGrad,
  RMSProp,
  Adam,
  AdamW,
  AMSGrad,
  GradientClipping,
  GradientNorm,
  VanishingGradient,
  ExplodingGradient,
  GradientNoise,
  GradientVariance,
  GradientAccumulation,
  ProjectedGradient,
  ProximalGradient,
  SubgradientMethod,
  MirrorDescent,
  NaturalGradient,
  Preconditioning,
  ConditionNumber,
  LipschitzGradient,
  SmoothFunction,
  StrongConvexity,
  ConvexFunction,
  NonConvexFunction,
  StationaryPoint,
  CriticalPoint,
  LocalMinimum,
  GlobalMinimum,
  SaddlePoint,
  DescentDirection,
  Convergence,
  ConvergenceRate,
  LinearConvergence,
  SublinearConvergence,
  NumericalStability,
  FloatingPointError,
  TerminationCriterion,
  GradientTolerance,
  ObjectiveTolerance,
  IterationLimit
 ]
 EXCLUDE_AS_PRIMARY = [
  ZerothOrderOptimisationWithoutGradientInformation,
  PureSecondOrderOptimisationWithoutFirstOrderCore,
  GeneralSearchWithoutDerivativeObjective,
  SoftwareTrainingInfrastructureWithoutMathematicalGradientSemantics
 ]
 DISAMBIGUATION_RULE = "When optimisation proceeds primarily through first-order derivatives or estimated gradients of an objective, interpret the context as gradient optimisation."
}

CORE_CONCEPTS {
 GradientOptimisation = "Optimisation driven by first-order derivative information."
 Gradient = "Vector of first partial derivatives of a scalar objective."
 DescentDirection = "Direction expected to reduce the objective locally."
 LearningRate = "Scalar controlling the magnitude of a parameter update."
 StochasticGradient = "Gradient estimate computed from a random sample or minibatch."
 Momentum = "Update mechanism accumulating directional information from previous gradients."
 AdaptiveGradientMethod = "Method adapting effective learning rates using gradient history."
 Convergence = "Approach of iterates, gradient norms or objective values toward a limiting optimum or stationary state."
 Conditioning = "Sensitivity of optimisation progress to curvature scaling across directions."
}

RELATIONS {
 GradientOptimisation --is_child_of--> OptimisationForAI
 GradientOptimisation --uses--> Calculus
 Gradient --is_derivative_of--> ObjectiveFunction
 GradientDescent --uses--> NegativeGradient
 StochasticGradientDescent --uses--> StochasticGradient
 MiniBatchGradientDescent --is_type_of--> StochasticGradientDescent
 LearningRate --controls--> UpdateMagnitude
 LineSearch --selects--> StepSize
 BacktrackingLineSearch --uses--> ArmijoCondition
 WolfeConditions --constrain--> LineSearch
 Momentum --accumulates--> GradientDirection
 NesterovMomentum --is_type_of--> MomentumMethod
 AdaGrad --is_type_of--> AdaptiveGradientMethod
 RMSProp --is_type_of--> AdaptiveGradientMethod
 Adam --is_type_of--> AdaptiveGradientMethod
 AdamW --extends--> Adam
 GradientClipping --bounds--> GradientNorm
 ProjectedGradient --enforces--> FeasibleSet
 ProximalGradient --supports--> NonsmoothRegularisation
 NaturalGradient --uses--> InformationGeometry
 Preconditioning --improves--> Conditioning
 LipschitzGradient --supports--> StepSizeBound
 StrongConvexity --supports--> FasterConvergenceGuarantees
 StationaryPoint --satisfies--> ZeroOrSmallGradient
 SaddlePoint --is_type_of--> StationaryPoint
}

OPERATORS {
 GRADIENT(f,theta) := grad_f(theta)
 GRADIENT_DESCENT(theta,eta,g) := theta_next=theta-eta*g
 SGD_UPDATE(theta,eta,g_hat) := theta_next=theta-eta*g_hat
 GRADIENT_NORM(g) := ||g||_2
 DIRECTIONAL_DERIVATIVE(f,theta,d) := D_d f=grad f(theta)^T*d
 STEEPEST_DESCENT_DIRECTION(g) := d=-g
 MOMENTUM(v,g,beta) := v_next=beta*v+(1-beta)*g
 MOMENTUM_UPDATE(theta,eta,v) := theta_next=theta-eta*v
 NESTEROV_LOOKAHEAD(theta,v,beta) := theta_look=theta-beta*v
 ADAGRAD_ACCUMULATOR(G,g) := G_next=G+g^2
 RMSPROP_ACCUMULATOR(v,g,beta) := v_next=beta*v+(1-beta)*g^2
 ADAM_FIRST_MOMENT(m,g,beta1) := m_next=beta1*m+(1-beta1)*g
 ADAM_SECOND_MOMENT(v,g,beta2) := v_next=beta2*v+(1-beta2)*g^2
 ADAM_UPDATE(theta,mhat,vhat,eta,epsilon) := theta_next=theta-eta*mhat/(sqrt(vhat)+epsilon)
 CLIP_BY_NORM(g,c) := g_clipped=g*min(1,c/||g||)
 PROJECTED_UPDATE(theta,eta,g,C) := theta_next=Projection_C(theta-eta*g)
 PROXIMAL_UPDATE(theta,eta,g,Omega) := theta_next=prox_{eta*Omega}(theta-eta*g)
}

AXIOMS {
 A1 = "Gradient optimisation requires a differentiable objective or suitable generalised derivative."
 A2 = "The negative gradient is the direction of steepest local descent under the Euclidean norm."
 A3 = "Step size materially affects stability and convergence."
 A4 = "Stochastic gradients introduce sampling variance."
 A5 = "Momentum changes the effective optimisation dynamics by accumulating past directions."
 A6 = "Adaptive methods rescale updates using gradient history."
 A7 = "Non-convex objectives may contain saddle points and multiple local minima."
 A8 = "Gradient-based results require numerical and convergence validation."
}

REASONING {
 R1_CONTEXT = identify(objective,parameters,constraints,data,gradient_source)
 R2_GRADIENT = compute_or_estimate(gradient,gradient_norm,variance)
 R3_GEOMETRY = analyse(smoothness,convexity,strong_convexity,conditioning)
 R4_STEP_SIZE = select(fixed,diminishing,line_search,adaptive_schedule)
 R5_METHOD = select(GD,SGD,momentum,Nesterov,AdaGrad,RMSProp,Adam,projected,proximal,natural_gradient)
 R6_UPDATE = apply(parameter_update)
 R7_MONITOR = track(objective,gradient_norm,step_norm,validation_metric)
 R8_STABILITY = detect(oscillation,divergence,vanishing_gradient,exploding_gradient)
 R9_CONVERGENCE = assess(stationarity,rate,tolerance,iteration_limit)
 R10_VALIDATION = check(gradient_correctness,numerical_precision,reproducibility,generalisation)
}

CONSTRAINTS {
 C1 = "A large gradient does not by itself imply a large safe step."
 C2 = "A small gradient can occur near saddle points as well as minima."
 C3 = "Fixed learning rates can diverge when they exceed stability limits."
 C4 = "Stochastic gradient noise can help or hinder optimisation depending on scale."
 C5 = "Adaptive methods need not behave like SGD in generalisation or asymptotic dynamics."
 C6 = "Gradient clipping changes the original optimisation trajectory."
 C7 = "Poor conditioning can produce slow zig-zagging convergence."
 C8 = "Finite-precision arithmetic can corrupt very small or very large gradients."
 C9 = "Non-convex convergence guarantees usually concern stationary points rather than global minima."
 C10 = "Training convergence does not guarantee model generalisation."
}

SEMANTIC_EQUIVALENCE {
 "GradientOptimisation" <=> "GradientOptimization"
 "GradientDescent" <=> "SteepestDescentUnderEuclideanMetric"
 "StochasticGradientDescent" <=> "SGD"
 "NesterovMomentum" <=> "NesterovAcceleratedGradient"
 "GradientClipping" <=> "GradientNormClipping"
 "StationaryPoint" <=> "CriticalPoint"
}

VALIDATION {
 V1 = "Objective function is explicit."
 V2 = "Gradient source is explicit: analytic, automatic differentiation, stochastic estimate or finite approximation."
 V3 = "Gradient dimensions match parameter dimensions."
 V4 = "Learning-rate rule is documented."
 V5 = "Gradient norms are monitored where relevant."
 V6 = "Stopping criteria are explicit."
 V7 = "Constraint feasibility is checked for projected methods."
 V8 = "Numerical precision and overflow or underflow risks are assessed."
 V9 = "Convergence diagnostics distinguish optimisation from generalisation."
 V10 = "Results are reproducible under documented initialisation and randomness where feasible."
}

CONFIDENCE {
 BASE_CONFIDENCE = 0.99
 GRADIENT_OPTIMISATION_DEFINITION = 0.99
 GRADIENT_REASONING = 0.99
 STEP_SIZE_REASONING = 0.99
 SGD_REASONING = 0.99
 MOMENTUM_REASONING = 0.99
 ADAPTIVE_METHOD_REASONING = 0.98
 CONVERGENCE_REASONING = 0.98
 STABILITY_REASONING = 0.98
 POLICY = "Reduce confidence when objective smoothness, gradient quality, learning rate, stochastic variance, conditioning, constraints, precision or stopping criteria are underspecified."
}

AI_CONSUMPTION {
 PURPOSES = [
  SemanticSearch,
  OntologyGrounding,
  GradientOptimisationReasoning,
  GradientDescentReasoning,
  SGDReasoning,
  MiniBatchReasoning,
  LearningRateReasoning,
  LineSearchReasoning,
  MomentumReasoning,
  NesterovReasoning,
  AdaGradReasoning,
  RMSPropReasoning,
  AdamReasoning,
  GradientClippingReasoning,
  ProjectedGradientReasoning,
  ProximalGradientReasoning,
  NaturalGradientReasoning,
  ConditioningReasoning,
  ConvergenceReasoning,
  NumericalStabilityReasoning,
  KnowledgeGraphConstruction,
  RetrievalAugmentation,
  RecursiveInference
 ]
 NORMALISE {
  "gradient optimisation" -> GradientOptimisation
  "gradient optimization" -> GradientOptimisation
  "gradient descent" -> GradientDescent
  "stochastic gradient descent" -> StochasticGradientDescent
  "SGD" -> StochasticGradientDescent
  "Nesterov accelerated gradient" -> NesterovMomentum
  "Adam optimizer" -> Adam
  "gradient clipping" -> GradientClipping
 }
}

EXECUTION_RULES {
 E1 = "If an objective is optimised primarily with gradient information, activate GradientOptimisationOntology."
 E2 = "If exact full-data gradients are used, route to batch gradient descent."
 E3 = "If gradients come from random examples or minibatches, route to stochastic gradient descent."
 E4 = "If oscillation or slow progress occurs, consider momentum, preconditioning or learning-rate changes."
 E5 = "If gradient scales differ strongly by parameter, consider adaptive methods or preconditioning."
 E6 = "If explicit constraints exist, consider projected gradient methods."
 E7 = "If nonsmooth regularisation is present, consider proximal or subgradient methods."
 E8 = "If gradient explosion occurs, consider clipping and stability analysis."
 E9 = "Any gradient method should be validated for convergence, numerical stability and generalisation."
}

RECURSION {
 ENTRY = GradientOptimisation
 EXPAND(GradientOptimisation) -> [
  Gradient, ObjectiveFunction, LossFunction, CostFunction, ParameterVector,
  DirectionalDerivative, SteepestDescent, GradientDescent,
  BatchGradientDescent, StochasticGradientDescent, MiniBatchGradientDescent,
  StochasticGradient, GradientEstimator, UnbiasedGradientEstimator,
  BiasedGradientEstimator, LearningRate, StepSize, FixedStepSize,
  DiminishingStepSize, AdaptiveStepSize, LineSearch, BacktrackingLineSearch,
  ArmijoCondition, WolfeConditions, Momentum, HeavyBallMethod,
  NesterovMomentum, AdaptiveGradientMethod, AdaGrad, RMSProp, Adam, AdamW,
  AMSGrad, GradientClipping, GradientNorm, VanishingGradient,
  ExplodingGradient, GradientNoise, GradientVariance, GradientAccumulation,
  ProjectedGradient, ProximalGradient, SubgradientMethod, MirrorDescent,
  NaturalGradient, Preconditioning, ConditionNumber, LipschitzGradient,
  SmoothFunction, StrongConvexity, ConvexFunction, NonConvexFunction,
  StationaryPoint, CriticalPoint, LocalMinimum, GlobalMinimum, SaddlePoint,
  DescentDirection, Convergence, ConvergenceRate, LinearConvergence,
  SublinearConvergence, NumericalStability, FloatingPointError,
  TerminationCriterion, GradientTolerance, ObjectiveTolerance, IterationLimit
 ]
 EXPAND(MethodFamilies) -> [
  DeterministicGradientMethods,
  StochasticGradientMethods,
  MomentumMethods,
  AdaptiveGradientMethods,
  ConstrainedGradientMethods,
  GeometryAwareGradientMethods
 ]
 STOP_CONDITION = "Stop when gradient source, objective geometry, step-size rule, update method, convergence, stability and stopping criteria are sufficiently resolved."
}

ROUTING {
 PARENT_ONTOLOGY = "Optimisation for AI Ontology"
 IMMEDIATE_CONTEXTS = [
  "Optimisation for AI Ontology",
  "Neural Network Mathematics Ontology",
  "Calculus & Analysis Ontologies",
  "Numerical Analysis Ontologies"
 ]
 RELATED_ONTOLOGIES = [
  "Optimisation for AI Ontology",
  "AI Mathematics Ontology",
  "Neural Network Mathematics Ontology",
  "Calculus & Analysis Ontologies",
  "Convex Optimisation Ontology",
  "Numerical Analysis Ontologies",
  "Numerical Linear Algebra Ontology",
  "Probability & Statistics Ontologies",
  "Computational Complexity Ontology",
  "Dynamical Systems Ontologies"
 ]
 ROUTE_RULES = [
  "Gradient Optimisation -> Optimisation for AI for parent semantics",
  "Gradient Optimisation -> Neural Network Mathematics for backpropagated gradients",
  "Gradient Optimisation -> Calculus & Analysis for derivative foundations",
  "Gradient Optimisation -> Convex Optimisation for convex convergence theory",
  "Gradient Optimisation -> Numerical Analysis for iterative stability",
  "Gradient Optimisation -> Numerical Linear Algebra for conditioning and preconditioning",
  "Gradient Optimisation -> Probability & Statistics for stochastic-gradient variance",
  "Gradient Optimisation -> Computational Complexity for iteration and resource cost",
  "Gradient Optimisation -> Dynamical Systems for continuous-time gradient-flow interpretations"
 ]
}

UPDATE_MODEL {
 MODE = "append-compatible semantic versioning"
 MAJOR = "ontology-breaking conceptual change"
 MINOR = "new compatible gradient methods, convergence models or reasoning pathways"
 PATCH = "clarification, metadata or formatting correction"
 RULES = [
  "Preserve first-order derivative semantics.",
  "Preserve deterministic-versus-stochastic gradient distinction.",
  "Preserve step-size semantics.",
  "Preserve momentum-versus-adaptive method distinction.",
  "Preserve constrained-versus-unconstrained gradient distinction.",
  "Preserve stationary-point semantics.",
  "Preserve convergence and numerical-stability semantics.",
  "Preserve machine-readable gradient-optimisation semantics."
 ]
}

OUTPUT {
 CANONICAL_DEFINITION = "Gradient optimisation uses first-order derivative information to iteratively improve an objective."
 GRADIENT_SUMMARY = "The gradient identifies local objective sensitivity and, under the Euclidean norm, its negative defines the steepest local descent direction."
 STEP_SUMMARY = "Learning rate or line search controls how far an optimisation step moves along a descent direction."
 STOCHASTIC_SUMMARY = "Stochastic and minibatch gradients trade exactness for scalable, noisy parameter updates."
 MOMENTUM_SUMMARY = "Momentum accumulates gradient direction across iterations to reduce oscillation and accelerate progress."
 ADAPTIVE_SUMMARY = "AdaGrad, RMSProp and Adam rescale updates using historical gradient statistics."
 CONVERGENCE_SUMMARY = "Gradient convergence is assessed through objective change, gradient norm, stationarity and method-specific guarantees."
 VALIDATION_SUMMARY = "Gradient methods should be checked for derivative correctness, numerical stability, convergence, reproducibility and generalisation."
}

FLOW {
 INPUT
 -> detect_gradient_optimisation_context
 -> identify_objective_parameters_constraints_data_and_gradient_source
 -> compute_or_estimate_gradient
 -> analyse_smoothness_convexity_conditioning_and_gradient_variance
 -> select_batch_SGD_minibatch_momentum_adaptive_projected_proximal_or_natural_gradient_method
 -> select_learning_rate_or_line_search_rule
 -> update_parameters
 -> monitor_objective_gradient_norm_step_norm_and_validation_metrics
 -> detect_divergence_vanishing_exploding_or_noisy_gradients
 -> assess_stationarity_convergence_rate_and_stopping_criteria
 -> validate_numerical_precision_gradient_correctness_reproducibility_and_generalisation
 -> route_to_optimisation_for_AI_neural_network_mathematics_calculus_convex_optimisation_numerical_analysis_numerical_linear_algebra_probability_statistics_computational_complexity_or_dynamical_systems_ontology
 -> recursively_expand_required_dependencies
 -> verify_result
 -> assign_confidence
 -> produce_semantic_output
 -> END
}

SEMANTIC_SUMMARY {
 "The Gradient Optimisation Ontology defines first-order optimisation mathematics. It supports gradients and directional derivatives, steepest descent, batch, stochastic and minibatch gradient descent, gradient estimators, learning rates and step sizes, line search, Armijo and Wolfe conditions, momentum and Nesterov acceleration, AdaGrad, RMSProp, Adam, AdamW and AMSGrad, gradient clipping, vanishing and exploding gradients, gradient noise and variance, projected and proximal gradient methods, subgradient, mirror descent and natural gradient, preconditioning and conditioning, smoothness and strong convexity, convex and non-convex objectives, stationary points, minima and saddle points, convergence rates, numerical stability, floating-point error and termination criteria."
}

FLOW_HASH {
 METHOD = "semantic-content-identity"
 ID = "KIRETH-MATH-APPLIED-MATHEMATICS-GRADIENT-OPTIMISATION-ONTOLOGY-V1"
}

}

END_FLOW