Gradient Descent Ontology

AI INTERPRETATION NOTE
This Flow Algebra defines the Gradient Descent Ontology 2 within the Kireth Research Semantic Library. Gradient descent is a first-order optimisation method that iteratively updates decision variables in a direction derived from the negative gradient of an objective function. This ontology supports fixed and adaptive step sizes, line search, steepest descent, momentum, Nesterov acceleration, stochastic and mini-batch variants, convex and nonconvex objectives, conditioning, scaling, convergence rates, stopping criteria, finite precision, verification and reproducibility.

FLOW_ALGEBRA_RECORD {

METADATA {
 FLOW_NAME = "Gradient Descent Ontology 2"
 FLOW_TYPE = "Mathematical Ontology"
 DOMAIN = "Mathematics"
 SUBDOMAIN = "Computational Mathematics"
 SPECIALISATION = "Optimisation Computation / Gradient Descent"
 CANONICAL_PAGE = "https://kirethresearch.org/booksai-core/ontologies/mathematical-ontologies/computational-mathematics-ontologies/optimisation-computation/gradient-descent-ontology-2.html"
 LANGUAGE = "en-GB"
 VERSION = "1.0"
 STATUS = "active"
}

IDENTITY {
 PRIMARY_ENTITY = GradientDescent
 ENTITY_CLASS = FirstOrderOptimisationFramework
 PRIMARY_CONTEXT = ComputationalOptimisation
 DEFINITION = "Gradient descent is a first-order iterative optimisation method that updates decision variables by moving in a direction opposite to the gradient of a differentiable objective."
 SYMBOLIC_FORMS = [
  "x_(k+1)=x_k-alpha_k*grad f(x_k)",
  "p_k=-grad f(x_k)",
  "f(x_(k+1))<=f(x_k) under suitable step conditions",
  "||grad f(x_k)|| -> 0"
 ]
 SEMANTIC_ROLE = "Provides machine-readable semantics for first-order descent directions, learning rates, line searches, acceleration, stochastic gradients, convergence, conditioning, stopping criteria and verification."
}

SCOPE {
 INCLUDE = [
  GradientDescent,SteepestDescent,FirstOrderMethod,
  UnconstrainedOptimisation,ConstrainedOptimisation,
  SmoothOptimisation,ConvexOptimisation,NonconvexOptimisation,
  ObjectiveFunction,DecisionVariable,Gradient,
  GradientVector,PartialDerivative,DirectionalDerivative,
  SearchDirection,DescentDirection,NegativeGradient,
  StepSize,LearningRate,FixedStepSize,VariableStepSize,
  AdaptiveStepSize,StepSizeSchedule,LineSearch,
  ExactLineSearch,InexactLineSearch,BacktrackingLineSearch,
  ArmijoCondition,WolfeCondition,StrongWolfeCondition,
  SufficientDecrease,CurvatureCondition,
  LipschitzGradient,LipschitzConstant,SmoothnessConstant,
  ConvexFunction,StronglyConvexFunction,StrongConvexityParameter,
  ConditionNumber,Conditioning,IllConditioning,
  Scaling,VariableScaling,ObjectiveScaling,
  Preconditioning,PreconditionedGradientDescent,
  ProjectedGradientDescent,Projection,FeasibleSet,
  BoundConstraint,ConvexConstraint,
  Momentum,HeavyBallMethod,Velocity,NesterovAcceleration,
  AcceleratedGradientMethod,NesterovGradientMethod,
  StochasticGradientDescent,SGD,StochasticGradient,
  MiniBatchGradientDescent,MiniBatch,Epoch,Batch,
  SampleGradient,GradientNoise,Variance,GradientVariance,
  LearningRateDecay,ConstantLearningRate,
  DiminishingStepSize,PolynomialDecay,ExponentialDecay,
  AdaGrad,RMSProp,Adam,AdaptiveGradientMethod,
  CoordinateGradientDescent,BlockGradientDescent,
  GradientClipping,GradientNormalisation,
  Regularisation,L1Regularisation,L2Regularisation,
  WeightDecay,PenaltyTerm,CompositeObjective,
  ProximalGradient,SubgradientMethod,
  AutomaticDifferentiation,NumericalDifferentiation,
  FiniteDifferenceGradient,AnalyticGradient,
  GradientChecking,DerivativeVerification,
  Iteration,IterationCount,MaximumIterations,
  FunctionEvaluation,GradientEvaluation,
  FunctionEvaluationCount,GradientEvaluationCount,
  GradientNorm,StepNorm,ObjectiveChange,
  StoppingCriterion,GradientTolerance,StepTolerance,
  FunctionTolerance,MaximumTime,
  Convergence,ConvergenceRate,SublinearConvergence,
  LinearConvergence,GeometricConvergence,
  StationaryPoint,CriticalPoint,LocalMinimum,
  GlobalMinimum,SaddlePoint,Plateau,
  Stagnation,Divergence,Oscillation,
  Overshooting,VanishingGradient,ExplodingGradient,
  BasinOfAttraction,InitialGuess,Initialisation,
  QuadraticObjective,LeastSquaresObjective,
  Hessian,HessianSpectrum,SmallestEigenvalue,
  LargestEigenvalue,SpectralConditionNumber,
  FloatingPointArithmetic,MachinePrecision,
  RoundOffError,Underflow,Overflow,
  Reproducibility,Verification,Benchmarking,
  ReferenceSolution,SolverStatus,Success,Failure,Warning
 ]
 EXCLUDE_AS_PRIMARY = [
  NewtonOptimisationWithPrimarySecondOrderSemantics,
  QuasiNewtonOptimisationWithHessianApproximationAsPrimaryMechanism,
  DerivativeFreeOptimisationWithoutGradientInformation,
  PureBackpropagationWithoutOptimisationUpdateSemantics
 ]
 DISAMBIGUATION_RULE = "When an objective is iteratively reduced by moving opposite to its gradient or a stochastic approximation to that gradient, interpret the context as gradient descent."
}

CORE_CONCEPTS {
 GradientDescent = "First-order optimisation using the negative gradient as the canonical descent direction."
 Gradient = "Vector of first partial derivatives of the objective."
 LearningRate = "Scalar controlling the size of a gradient-descent update."
 DescentDirection = "Direction with negative directional derivative at the current point."
 LineSearch = "Procedure selecting a step size along a search direction."
 StrongConvexity = "Convexity property supporting linear convergence under suitable gradient-descent step sizes."
 Momentum = "Update mechanism accumulating previous search motion."
 StochasticGradientDescent = "Gradient-descent variant using noisy sample or mini-batch gradient estimates."
}

RELATIONS {
 GradientDescent --is_child_of--> ComputationalOptimisation
 GradientDescent --is_type_of--> FirstOrderMethod
 GradientDescent --uses--> Gradient
 GradientDescent --uses--> LearningRate
 NegativeGradient --is--> DescentDirectionUnderNonzeroGradient
 LineSearch --selects--> StepSize
 BacktrackingLineSearch --tests--> ArmijoCondition
 WolfeCondition --constrains--> StepSize
 LipschitzGradient --supports--> StepSizeBounds
 StronglyConvexFunction --supports--> LinearConvergence
 ConvexFunction --supports_under_conditions--> SublinearConvergence
 Momentum --modifies--> GradientDescentUpdate
 NesterovAcceleration --accelerates--> GradientBasedOptimisation
 StochasticGradientDescent --uses--> StochasticGradient
 MiniBatchGradientDescent --reduces_relative_to_single_sample--> GradientVariance
 PreconditionedGradientDescent --modifies--> GradientGeometry
 ProjectedGradientDescent --enforces--> FeasibleSet
 Verification --checks--> GradientDescentResult
}

OPERATORS {
 GRADIENT_DIRECTION(f,x) := p=-grad f(x)
 GRADIENT_STEP(x,g,alpha) := x_next=x-alpha*g
 PROJECTED_GRADIENT_STEP(x,g,alpha,C) := x_next=Projection_C(x-alpha*g)
 MOMENTUM_UPDATE(v,g,beta,alpha) := v_next=beta*v-alpha*g
 MOMENTUM_STEP(x,v_next) := x_next=x+v_next
 NESTEROV_LOOKAHEAD(x,v,beta) := x_look=x+beta*v
 SGD_STEP(x,g_hat,alpha) := x_next=x-alpha*g_hat
 GRADIENT_NORM(g) := ||g||
 OBJECTIVE_DECREASE(f,x,x_next) := f(x)-f(x_next)
}

AXIOMS {
 A1 = "The negative gradient is the steepest local descent direction under the Euclidean norm."
 A2 = "Gradient descent requires first-order derivative information or a numerical approximation to it."
 A3 = "Step-size choice affects both convergence speed and stability."
 A4 = "For smooth convex objectives, suitable step sizes provide global convergence guarantees."
 A5 = "For smooth strongly convex objectives, suitable gradient descent can converge linearly."
 A6 = "For nonconvex objectives, gradient descent may converge to stationary points that are not global minima."
 A7 = "Stochastic gradients introduce variance into update directions."
 A8 = "Gradient-descent results require explicit learning-rate, convergence, stopping and verification semantics."
}

REASONING {
 R1_CONTEXT = identify(objective,variables,constraints,smoothness,convexity,scale,gradient_source,accuracy_goal)
 R2_GRADIENT = compute_or_obtain(analytic,automatic,numerical_or_stochastic_gradient)
 R3_DIRECTION = form(negative_gradient_or_preconditioned_gradient)
 R4_STEP_CONTROL = select(fixed_step,line_search,schedule,adaptive_or_stochastic_learning_rate)
 R5_UPDATE = compute(next_iterate)
 R6_CONSTRAINTS = if_required apply_projection_or_constraint_compatible_update
 R7_ACCELERATION = optionally_apply(momentum,Nesterov_or_related_first_order_acceleration)
 R8_MONITOR = track(objective,gradient_norm,step_norm,oscillation,stagnation_and_gradient_noise)
 R9_CONVERGENCE = assess(rate,conditioning,stationarity_and_resource_limits)
 R10_VALIDATE = check_gradient_accuracy,reference_solution,repeatability_and_reproducibility)
}

CONSTRAINTS {
 C1 = "A learning rate that is too large can cause oscillation or divergence."
 C2 = "A learning rate that is too small can cause slow convergence."
 C3 = "Ill-conditioning can produce slow progress along shallow directions."
 C4 = "A small gradient norm does not distinguish a minimum from a saddle point without additional information."
 C5 = "Finite-difference gradients can be corrupted by truncation and round-off error."
 C6 = "Stochastic gradients can prevent exact monotonic objective decrease at every iteration."
 C7 = "Momentum can accelerate convergence but can also overshoot or destabilise poorly tuned problems."
 C8 = "Projected gradient methods depend on accurate projection onto the feasible set."
 C9 = "Stopping on objective change alone can mistake flat regions for convergence."
 C10 = "Finite precision can create apparent stagnation near numerical accuracy limits."
}

SEMANTIC_EQUIVALENCE {
 "GradientDescent" <=> "SteepestDescentInEuclideanMetric"
 "LearningRate" <=> "GradientStepSize"
 "StochasticGradientDescent" <=> "SGD"
 "MiniBatchGradientDescent" <=> "MiniBatchSGD"
 "NesterovAcceleration" <=> "NesterovGradientMethod"
 "HeavyBallMethod" <=> "MomentumGradientMethod"
}

VALIDATION {
 V1 = "Objective function is explicit."
 V2 = "Initial point is documented."
 V3 = "Gradient source is documented."
 V4 = "Learning-rate or line-search strategy is documented."
 V5 = "Gradient norm, objective progress and step norm are monitored."
 V6 = "Convergence or stopping tolerances are documented."
 V7 = "Gradient checking is used where derivative correctness is uncertain."
 V8 = "Conditioning and scaling are assessed where convergence is slow."
 V9 = "Stochastic runs report seed or sampling configuration where reproducibility matters."
 V10 = "Results are reproducible under documented arithmetic and algorithm settings."
}

CONFIDENCE {
 BASE_CONFIDENCE = 0.99
 GRADIENT_DESCENT_DEFINITION = 0.99
 STEP_SIZE_REASONING = 0.99
 CONVEX_CONVERGENCE_REASONING = 0.99
 STRONGLY_CONVEX_REASONING = 0.99
 MOMENTUM_REASONING = 0.98
 STOCHASTIC_GRADIENT_REASONING = 0.98
 NONCONVEX_REASONING = 0.97
 POLICY = "Reduce confidence when gradient quality, learning rate, smoothness, convexity, conditioning, stochastic variance or stopping criteria are underspecified."
}

AI_CONSUMPTION {
 PURPOSES = [
  SemanticSearch,OntologyGrounding,GradientDescentReasoning,
  GradientReasoning,LearningRateReasoning,
  LineSearchReasoning,BacktrackingReasoning,
  ConvexOptimisationReasoning,StrongConvexityReasoning,
  ConditioningReasoning,PreconditioningReasoning,
  ProjectedGradientReasoning,MomentumReasoning,
  NesterovReasoning,StochasticGradientReasoning,
  MiniBatchReasoning,AdaptiveLearningRateReasoning,
  StoppingCriterionReasoning,StationarityReasoning,
  VerificationReasoning,KnowledgeGraphConstruction,
  RetrievalAugmentation,RecursiveInference
 ]
 NORMALISE {
  "gradient descent" -> GradientDescent
  "steepest descent" -> GradientDescent
  "learning rate" -> LearningRate
  "SGD" -> StochasticGradientDescent
  "mini-batch gradient descent" -> MiniBatchGradientDescent
  "Nesterov accelerated gradient" -> NesterovAcceleration
  "momentum method" -> HeavyBallMethod
 }
}

EXECUTION_RULES {
 E1 = "If an optimisation iteration updates variables using a gradient-based descent direction, activate GradientDescentOntology."
 E2 = "Identify the source and reliability of gradient information."
 E3 = "Choose learning rate according to smoothness, conditioning and convergence requirements."
 E4 = "If fixed step size is unreliable, use line search or another controlled step strategy."
 E5 = "For convex constraints, use projection when projected-gradient semantics are appropriate."
 E6 = "For poorly conditioned problems, consider scaling or preconditioning."
 E7 = "For stochastic objectives, interpret per-iteration objective increases as compatible with noisy gradient updates."
 E8 = "Distinguish convergence from plateaus, saddle points, stagnation and numerical limits."
 E9 = "Validate gradient correctness, convergence behaviour and reproducibility."
}

RECURSION {
 ENTRY = GradientDescent
 EXPAND(GradientDescent) -> [
  SteepestDescent,FirstOrderMethod,UnconstrainedOptimisation,
  ConstrainedOptimisation,SmoothOptimisation,ConvexOptimisation,
  NonconvexOptimisation,ObjectiveFunction,DecisionVariable,Gradient,
  DirectionalDerivative,SearchDirection,DescentDirection,NegativeGradient,
  StepSize,LearningRate,FixedStepSize,VariableStepSize,AdaptiveStepSize,
  LineSearch,ExactLineSearch,InexactLineSearch,BacktrackingLineSearch,
  ArmijoCondition,WolfeCondition,StrongWolfeCondition,
  LipschitzGradient,LipschitzConstant,ConvexFunction,
  StronglyConvexFunction,StrongConvexityParameter,
  ConditionNumber,Conditioning,Scaling,Preconditioning,
  PreconditionedGradientDescent,ProjectedGradientDescent,Projection,
  Momentum,HeavyBallMethod,NesterovAcceleration,AcceleratedGradientMethod,
  StochasticGradientDescent,SGD,StochasticGradient,
  MiniBatchGradientDescent,MiniBatch,GradientNoise,Variance,
  LearningRateDecay,DiminishingStepSize,AdaGrad,RMSProp,Adam,
  CoordinateGradientDescent,GradientClipping,Regularisation,
  ProximalGradient,AutomaticDifferentiation,NumericalDifferentiation,
  FiniteDifferenceGradient,GradientChecking,Iteration,
  GradientNorm,StepNorm,ObjectiveChange,StoppingCriterion,
  GradientTolerance,StepTolerance,FunctionTolerance,
  Convergence,SublinearConvergence,LinearConvergence,
  StationaryPoint,LocalMinimum,GlobalMinimum,SaddlePoint,
  Stagnation,Divergence,Oscillation,BasinOfAttraction,
  InitialGuess,QuadraticObjective,Hessian,HessianSpectrum,
  FloatingPointArithmetic,MachinePrecision,RoundOffError,
  Verification,Benchmarking,ReferenceSolution,Reproducibility
 ]
 EXPAND(GradientDescentFamilies) -> [
  FixedStepGradientDescent,
  LineSearchGradientDescent,
  PreconditionedGradientDescent,
  ProjectedGradientDescent,
  MomentumGradientDescent,
  NesterovAcceleratedGradient,
  StochasticGradientDescent,
  MiniBatchGradientDescent
 ]
 STOP_CONDITION = "Stop when objective, gradient, step control, conditioning, acceleration, stochasticity, convergence and verification are sufficiently resolved."
}

ROUTING {
 PARENT_ONTOLOGY = "Computational Optimisation Ontology"
 RELATED_ONTOLOGIES = [
  "Computational Optimisation Ontology",
  "Optimisation Computation",
  "Nonlinear Optimisation Solver Ontology",
  "Numerical Differentiation Ontology 2",
  "Linear Solver Ontology",
  "Numerical Linear Algebra Ontology 2",
  "Numerical Analysis Ontology 2",
  "Numerical Error Ontology",
  "Numerical Stability Ontology 2",
  "Numerical Convergence Ontology",
  "Floating Point Arithmetic Ontology",
  "Artificial Intelligence Mathematics"
 ]
 ROUTE_RULES = [
  "Gradient Descent -> Computational Optimisation for parent semantics",
  "Gradient Descent -> Nonlinear Optimisation Solver for nonlinear objective context",
  "Gradient Descent -> Numerical Differentiation for finite-difference gradients",
  "Gradient Descent -> Numerical Linear Algebra for conditioning and preconditioning",
  "Gradient Descent -> Numerical Analysis for descent and convergence theory",
  "Gradient Descent -> Numerical Error for derivative and stopping error",
  "Gradient Descent -> Numerical Stability for step-size and finite-precision behaviour",
  "Gradient Descent -> Numerical Convergence for sublinear and linear convergence",
  "Gradient Descent -> Artificial Intelligence Mathematics for model-training optimisation"
 ]
}

UPDATE_MODEL {
 MODE = "append-compatible semantic versioning"
 MAJOR = "ontology-breaking conceptual change"
 MINOR = "new compatible gradient variants, learning-rate policies or reasoning pathways"
 PATCH = "clarification, metadata or formatting correction"
 RULES = [
  "Preserve gradient-versus-step-size distinction.",
  "Preserve deterministic-versus-stochastic gradient distinction.",
  "Preserve fixed-step-versus-line-search distinction.",
  "Preserve unconstrained-versus-projected gradient distinction.",
  "Preserve plain-versus-momentum-versus-Nesterov distinction.",
  "Preserve stationarity-versus-minimality distinction.",
  "Preserve convergence-versus-stagnation distinction.",
  "Preserve machine-readable gradient-descent semantics."
 ]
}

OUTPUT {
 CANONICAL_DEFINITION = "Gradient descent iteratively updates decision variables in the direction opposite to the objective gradient."
 STEP_SUMMARY = "Each update combines a descent direction with a learning rate or line-search-selected step size."
 CONVEX_SUMMARY = "For smooth convex objectives, suitable step-size rules yield global objective convergence; strong convexity can yield linear convergence."
 CONDITIONING_SUMMARY = "Poor conditioning slows progress by creating strongly unequal curvature across directions."
 ACCELERATION_SUMMARY = "Momentum and Nesterov methods incorporate iteration history to accelerate suitable first-order optimisation problems."
 STOCHASTIC_SUMMARY = "Stochastic and mini-batch variants replace the exact gradient with noisy gradient estimates and require noise-aware convergence reasoning."
 VALIDATION_SUMMARY = "Gradient-descent results should be checked through gradient correctness, objective progress, gradient norms, convergence rates, sensitivity to learning rate and reproducibility."
}

FLOW {
 INPUT
 -> detect_gradient_descent_context
 -> identify_objective_variables_constraints_smoothness_convexity_scale_gradient_source_and_accuracy_goal
 -> compute_or_obtain_gradient
 -> form_negative_gradient_or_preconditioned_descent_direction
 -> choose_fixed_line_search_scheduled_adaptive_or_stochastic_step_control
 -> optionally_apply_projection_momentum_Nesterov_or_related_update
 -> compute_next_iterate
 -> evaluate_objective_gradient_norm_step_norm_and_constraint_state
 -> monitor_convergence_conditioning_oscillation_stagnation_divergence_and_gradient_noise
 -> adjust_learning_rate_scaling_preconditioning_or_acceleration_if_required
 -> assess_stationarity_and_resource_limits
 -> validate_gradient_correctness_reference_quality_and_reproducibility
 -> route_to_related_optimisation_nonlinear_solver_differentiation_linear_algebra_analysis_error_stability_convergence_and_AI_mathematics_ontologies
 -> recursively_expand_required_dependencies
 -> verify_result
 -> assign_confidence
 -> produce_semantic_output
 -> END
}

SEMANTIC_SUMMARY {
 "The Gradient Descent Ontology 2 defines first-order optimisation using negative-gradient search directions. It supports fixed and variable learning rates, exact and inexact line searches, Armijo and Wolfe conditions, smooth, convex and strongly convex objectives, conditioning and preconditioning, projected gradient methods, momentum and heavy-ball updates, Nesterov acceleration, stochastic and mini-batch gradient descent, learning-rate schedules, adaptive gradient methods, regularisation, analytic, automatic and finite-difference gradients, gradient checking, stopping criteria, sublinear and linear convergence, nonconvex stationary points and saddles, finite precision, verification, benchmarking and reproducibility."
}

FLOW_HASH {
 METHOD = "semantic-content-identity"
 ID = "KIRETH-MATH-COMPUTATIONAL-MATHEMATICS-GRADIENT-DESCENT-ONTOLOGY-2-V1"
}

}

END_FLOW