Backpropagation Mathematics Ontology

flow "BackpropagationMathematicsOntology" v1.0

# BACKPROPAGATION MATHEMATICS ONTOLOGY — FLOW ALGEBRA
SOURCE_URL := "https://kirethresearch.org/booksai-core/ontologies/calculus-analysis-ontologies/ai-computational-analysis/backpropagation-mathematics-ontology-2.html"
DOMAIN := "Mathematical Ontologies"
CATEGORY := "Calculus and Analysis Ontologies"
SUBCATEGORY := "AI Computational Analysis"
ENTITY := "Backpropagation Mathematics"

BACKPROPAGATION := {
 type: ReverseModeGradientComputation,
 domain: AIComputationalAnalysis,
 purpose: "Compute gradients of a scalar loss with respect to neural-network parameters by propagating derivatives backward through a computational graph using the chain rule."
}

CONCEPTS := {
 BACKPROPAGATION,
 COMPUTATIONAL_GRAPH,
 CHAIN_RULE,
 NEURAL_NETWORK,
 LAYER,
 WEIGHT,
 BIAS,
 ACTIVATION_FUNCTION,
 LOSS_FUNCTION,
 FORWARD_PASS,
 REVERSE_PASS,
 GRADIENT,
 JACOBIAN,
 ADJOINT,
 ERROR_SIGNAL,
 PARAMETER_UPDATE
}

# ---------- NETWORK ----------
layer l:

z_l
 =
W_l*a_(l-1)
+
b_l

a_l
 =
phi_l(z_l)

where

W_l
 -> WEIGHT_MATRIX

b_l
 -> BIAS_VECTOR

phi_l
 -> ACTIVATION_FUNCTION

a_l
 -> LAYER_ACTIVATION

# ---------- FORWARD PASS ----------
input:

a_0
 =
x

for
 l=1,...,L:

z_l
 =
W_l*a_(l-1)+b_l

a_l
 =
phi_l(z_l)

output:

y_hat
 =
a_L

FORWARD_PASS
 -> computes(PREDICTION_AND_INTERMEDIATE_VALUES)

# ---------- LOSS ----------
scalar objective:

J
 =
L(
 y_hat,
 y
)

LOSS_FUNCTION
 -> measures(PREDICTION_ERROR)

BACKPROPAGATION
 -> computes(
 partial_J/partial_PARAMETERS
)

# ---------- CHAIN RULE ----------
if

J
 =
J(a_L)

and

a_L
 =
phi_L(z_L)

then

partial_J/partial_z_L
 =
partial_J/partial_a_L
 elementwise_times
phi_L_prime(z_L)

CHAIN_RULE
 -> local_derivatives_compose_backward

# ---------- ERROR SIGNAL ----------
define:

delta_l
 :=
partial_J/partial_z_l

delta_l
 -> PREACTIVATION_ADJOINT

OUTPUT_ERROR:

delta_L
 =
grad_(a_L) J
 elementwise_times
phi_L_prime(z_L)

# ---------- HIDDEN LAYER RECURSION ----------
for
 l=L-1,...,1:

delta_l
 =
(
 W_(l+1)^T
 *
delta_(l+1)
)
elementwise_times
phi_l_prime(z_l)

BACKPROPAGATION_RECURRENCE
 -> propagates(ERROR_SIGNAL_BACKWARD)

# ---------- WEIGHT GRADIENT ----------
z_l
 =
W_l*a_(l-1)+b_l

therefore

partial_J/partial_W_l
 =
delta_l
*
a_(l-1)^T

WEIGHT_GRADIENT
 -> outer_product(ERROR_SIGNAL,PREVIOUS_ACTIVATION)

# ---------- BIAS GRADIENT ----------
partial_J/partial_b_l
 =
delta_l

for single example

BIAS_GRADIENT
 -> layer_error_signal

# ---------- ACTIVATION GRADIENT ----------
partial_J/partial_a_(l-1)
 =
W_l^T
*
delta_l

ACTIVATION_ADJOINT
 -> propagated_to(PREVIOUS_LAYER)

# ---------- MINI-BATCH ----------
for batch
 B

gradient:

grad_W_l J_B
 =
1/|B|
*
sum_(i in B)(
 delta_l^(i)
 *
(a_(l-1)^(i))^T
)

BATCH_GRADIENT
 -> average_or_sum_per_example_gradients
    according_to(LOSS_REDUCTION_CONVENTION)

# ---------- MATRIX BATCH FORM ----------
if examples stored as columns:

Z_l
 =
W_l*A_(l-1)+b_l*1^T

A_l
 =
phi_l(Z_l)

Delta_l
 -> batch_error_matrix

gradient:

dW_l
 =
(1/m)
*
Delta_l
*
A_(l-1)^T

db_l
 =
(1/m)
*
Delta_l*1

for mean loss convention

# ---------- REVERSE MODE AD ----------
network
 -> COMPUTATIONAL_GRAPH

scalar loss
 -> output node

BACKPROPAGATION
 -> REVERSE_MODE_AUTOMATIC_DIFFERENTIATION

ADJOINT
 -> derivative_of_loss_with_respect_to(INTERMEDIATE_VARIABLE)

REVERSE_MODE
 -> efficient_for(MANY_PARAMETERS_ONE_SCALAR_LOSS)

# ---------- LOCAL JACOBIAN ----------
operation:

v
 =
f(u)

local Jacobian:

J_f
 =
partial_v/partial_u

reverse propagation:

bar_u
 =
J_f^T
*
bar_v

BACKPROP
 -> repeated(VECTOR_JACOBIAN_PRODUCTS)

# ---------- VECTOR-JACOBIAN PRODUCT ----------
for function

v=f(u)

incoming adjoint:

bar_v

outgoing adjoint:

bar_u
 =
bar_v^T
*
J_f

depending on row/column convention

VJP
 -> fundamental_reverse_mode_operation

# ---------- COMPUTATIONAL GRAPH ACCUMULATION ----------
if variable
 u

feeds multiple downstream paths:

J
 =
J_1(u)+J_2(u)+...

then

bar_u
 =
bar_u_from_path_1
+
bar_u_from_path_2
+
...

GRADIENT
 -> sums_over(ALL_COMPUTATIONAL_PATHS)

# ---------- MEAN SQUARED ERROR ----------
J
 =
1/2
*
||y_hat-y||^2

then

grad_(y_hat) J
 =
y_hat-y

OUTPUT_RESIDUAL
 -> starts(BACKWARD_PROPAGATION)

# ---------- BINARY CROSS ENTROPY ----------
for sigmoid output:

a
 =
sigma(z)

loss:

J
 =
-
[
 y*log(a)
+
(1-y)*log(1-a)
]

combined derivative:

partial_J/partial_z
 =
a-y

SIGMOID_PLUS_CROSS_ENTROPY
 -> simplified_output_gradient

# ---------- SOFTMAX ----------
softmax:

a_i
 =
exp(z_i)
/
sum_j exp(z_j)

Jacobian:

partial_a_i/partial_z_j
 =
a_i
*
(
 indicator_(i=j)
 -
a_j
)

SOFTMAX
 -> coupled_output_derivatives

# ---------- SOFTMAX CROSS ENTROPY ----------
for one-hot target y:

J
 =
-
sum_i(
 y_i*log(a_i)
)

then

grad_z J
 =
a-y

SOFTMAX_CROSS_ENTROPY
 -> stable_simplified_gradient_when_combined_correctly

# ---------- ACTIVATION DERIVATIVES ----------
SIGMOID:

sigma_prime(z)
 =
sigma(z)
*
(
1-sigma(z)
)

TANH:

tanh_prime(z)
 =
1-tanh(z)^2

RELU:

ReLU(z)
 =
max(0,z)

derivative:
 1 if z>0
 0 if z<0

at z=0
 -> convention_dependent_subgradient

# ---------- VANISHING GRADIENT ----------
backprop product:

delta_l
 -> repeated_multiplication_by(
 WEIGHT_TRANSPOSES,
 ACTIVATION_DERIVATIVES
)

if factors have small magnitude:

gradient
 -> may_decay_exponentially_with_depth

VANISHING_GRADIENT
 -> weak_learning_signal_in_early_layers

# ---------- EXPLODING GRADIENT ----------
if repeated Jacobian factors
 -> large_norm

gradient
 -> may_grow_exponentially

EXPLODING_GRADIENT
 -> numerical_instability_and_large_updates

# ---------- JACOBIAN PRODUCTS ----------
deep network derivative:

partial_a_L/partial_a_0
 =
J_L
*
J_(L-1)
*
...
*
J_1

GRADIENT_FLOW
 -> controlled_by(SPECTRA_AND_NORMS_OF_LAYER_JACOBIANS)

# ---------- INITIALISATION ----------
WEIGHT_INITIALISATION
 -> influences(ACTIVATION_AND_GRADIENT_VARIANCE)

XAVIER_INITIALISATION
 -> designed_for(VARIANCE_CONTROL_IN_SUITABLE_ACTIVATIONS)

HE_INITIALISATION
 -> designed_for(RELU_TYPE_NETWORKS)

INITIALISATION
 -> supports(STABLE_SIGNAL_PROPAGATION)

# ---------- RESIDUAL CONNECTION ----------
residual layer:

a_(l+1)
 =
a_l
+
F(a_l)

Jacobian:

I
+
J_F

IDENTITY_PATH
 -> improves(GRADIENT_PROPAGATION)

RESIDUAL_NETWORK
 -> mitigates(DEEP_OPTIMISATION_DIFFICULTIES)

# ---------- NORMALISATION ----------
normalisation layers
 -> transform(ACTIVATIONS)

BACKPROP
 -> differentiates_through(NORMALISATION_OPERATION)

BATCH_NORMALISATION
 -> introduces(BATCH_DEPENDENCE)

LAYER_NORMALISATION
 -> normalises(WITHIN_FEATURE_STRUCTURE)

# ---------- REGULARISATION ----------
L2 penalty:

J_total
 =
J_data
+
lambda/2
*
sum_l(
 ||W_l||_F^2
)

gradient:

dW_l
 =
dW_l_data
+
lambda*W_l

REGULARISATION
 -> contributes(DIRECT_GRADIENT_TERM)

# ---------- DROPOUT ----------
training mask:

m_l
 -> random_binary_mask

a_l
 ->
m_l elementwise_times a_l

BACKPROP
 -> follows(SAMPLED_FORWARD_MASK)

TRAINING_AND_INFERENCE
 -> use(DIFFERENT_DROPOUT_BEHAVIOUR)

# ---------- RECURRENT NETWORK ----------
hidden state:

h_t
 =
f(
 h_(t-1),
 x_t,
 theta
)

unroll through time

-> COMPUTATIONAL_GRAPH

BACKPROPAGATION_THROUGH_TIME
 -> reverse_mode_over(UNROLLED_SEQUENCE)

BPTT
 -> may_exhibit(VANISHING_OR_EXPLODING_GRADIENTS)

# ---------- TRUNCATED BPTT ----------
long sequence

-> backpropagate_only_limited_number_of_steps

TRUNCATED_BPTT
 -> reduces(COMPUTATION_AND_MEMORY)

but

-> truncates(LONG_RANGE_GRADIENT_DEPENDENCIES)

# ---------- CONVOLUTIONAL NETWORK ----------
convolution operation:

Y
 =
K * X

BACKPROP
 -> computes(
 gradient_wrt_kernel,
 gradient_wrt_input
)

WEIGHT_SHARING
 -> gradient_accumulates_over(SPATIAL_LOCATIONS)

# ---------- ATTENTION ----------
attention:

A
 =
softmax(
 Q*K^T/sqrt(d)
)

output:

O
 =
A*V

BACKPROPAGATION
 -> differentiates_through(
 MATRIX_MULTIPLICATION,
 SCALING,
 SOFTMAX
)

ATTENTION_GRADIENT
 -> reverse_mode_chain_rule

# ---------- PARAMETER UPDATE ----------
after gradients:

theta_(k+1)
 =
theta_k
-
alpha
*
grad_theta J

BACKPROPAGATION
 -> computes(GRADIENT)

OPTIMISER
 -> uses(GRADIENT_TO_UPDATE_PARAMETERS)

BACKPROPAGATION
 -/> same_as(GRADIENT_DESCENT)

# ---------- OPTIMISERS ----------
gradient
 -> may_feed(
 SGD,
 MOMENTUM,
 RMSPROP,
 ADAM,
 L_BFGS
)

BACKPROP
 -> derivative_engine

OPTIMISER
 -> parameter_update_rule

# ---------- GRADIENT CHECK ----------
numerical check:

partial_J/partial_theta_i
 approx
[
 J(theta_i+h)
 -
J(theta_i-h)
]
/
(2*h)

compare with
 BACKPROP_GRADIENT

GRADIENT_CHECKING
 -> validates(IMPLEMENTATION)

finite difference
 -> affected_by(STEP_SIZE_AND_ROUNDOFF)

# ---------- COMPLEXITY ----------
forward pass
 -> O(COMPUTATION_COST)

reverse pass
 -> typically_same_order_of_cost

memory
 -> stores(INTERMEDIATE_ACTIVATIONS)

BACKPROPAGATION
 -> computationally_efficient_for(LARGE_PARAMETER_MODELS)

# ---------- CHECKPOINTING ----------
store subset of activations

recompute others during backward pass

CHECKPOINTING
 -> trades(COMPUTATION_FOR_MEMORY)

# ---------- SECOND ORDER ----------
gradient:

g
 =
grad_theta J

Hessian:

H
 =
nabla_theta^2 J

BACKPROP
 -> first_order_derivative

nested AD
 -> higher_order_information

HESSIAN_VECTOR_PRODUCT
 -> obtainable_without(FULL_HESSIAN)

# ---------- APPLICATIONS ----------
BACKPROPAGATION
 -> applied_to(NEURAL_NETWORK_TRAINING)
 -> applied_to(DEEP_LEARNING)
 -> applied_to(COMPUTER_VISION)
 -> applied_to(NATURAL_LANGUAGE_PROCESSING)
 -> applied_to(REINFORCEMENT_LEARNING)
 -> applied_to(SCIENTIFIC_MACHINE_LEARNING)
 -> applied_to(DIFFERENTIABLE_PROGRAMMING)
 -> applied_to(PHYSICS_INFORMED_NETWORKS)

# ---------- RELATION GRAPH ----------
FORWARD_PASS
 -> computes(ACTIVATIONS)

LOSS_FUNCTION
 -> produces(SCALAR_OBJECTIVE)

CHAIN_RULE
 -> propagates(DERIVATIVES)

REVERSE_PASS
 -> computes(ADJOINTS)

ADJOINTS
 -> generate(PARAMETER_GRADIENTS)

PARAMETER_GRADIENTS
 -> consumed_by(OPTIMISER)

DEEP_JACOBIAN_PRODUCTS
 -> control(GRADIENT_FLOW)

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE := {
 "backpropagation" ~= "reverse-mode chain-rule computation of neural-network gradients",
 "error signal" ~= "derivative of the loss with respect to a layer preactivation",
 "weight gradient" ~= "outer product of layer error signal and previous-layer activation in a dense layer",
 "backpropagation through time" ~= "reverse-mode differentiation through an unrolled recurrent computation",
 "gradient checking" ~= "comparison of analytic or automatic gradients with independent numerical derivative estimates",
 "vanishing gradient" ~= "decay of backward derivative magnitude across layers or time steps"
}

# ---------- CONSTRAINTS ----------
CONSTRAINTS := {
 backpropagation_computes_gradients_but_does_not_define_optimizer_update: true,
 matrix_shapes_and_transpose_conventions_must_be_preserved: true,
 activation_and_loss_derivatives_must_match_forward_definitions: true,
 nondifferentiable_activation_points_require_subgradient_convention: true,
 batch_gradient_scaling_depends_on_loss_reduction_convention: true,
 gradient_values_are_affected_by_floating_point_and_conditioning: true,
 complex_network_implementations_require_gradient_validation: true
}

# ---------- CONFIDENCE ----------
CONFIDENCE := {
 core_definition: 0.99,
 chain_rule_relation: 0.99,
 dense_layer_gradient_relation: 0.99,
 reverse_mode_relation: 0.99,
 loss_activation_relation: 0.99,
 gradient_flow_relation: 0.99,
 semantic_equivalence: 0.97
}

# ---------- RECURSIVE REASONING ----------
RECURSIVE_REASONING(query):
 identify(network_layers_parameters_activations_and_loss)
 -> execute(forward_pass_and_store_required_intermediates)
 -> initialise(output_loss_adjoint)
 -> propagate(adjoint_backward_through_each_operation)
 -> compute(weight_bias_and_input_gradients)
 -> accumulate(shared_parameter_and_batch_contributions)
 -> analyse(vanishing_exploding_and_conditioning_effects)
 -> validate(matrix_shapes_loss_reduction_and_nonsmooth_conventions)
 -> gradient_check(if_required)
 -> pass(gradients_to_optimizer)
 -> return(semantic_result)

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION := {
 parse_as: "typed backpropagation-mathematics semantic graph",
 primary_entity: BACKPROPAGATION,
 preserve: [
  network_structure,
  weights,
  biases,
  activations,
  loss,
  forward_values,
  tensor_shapes,
  batch_reduction,
  derivative_conventions
 ],
 infer_through: [
  chain_rule,
  local_Jacobians,
  adjoints,
  error_signals,
  weight_gradients,
  bias_gradients,
  BPTT,
  gradient_flow,
  gradient_checking
 ],
 avoid: [
  confusing_backpropagation_with gradient_descent,
  dropping_transposes_or tensor_shape information,
  differentiating_wrong_loss_activation_pair,
  ignoring_shared_parameter_gradient_accumulation,
  assuming_ReLU_derivative_unique_at zero,
  trusting_gradient_without validation_in complex graph
 ]
}

# ---------- UPDATE MODEL ----------
UPDATE_MODEL(new_information):
 validate(new_information)
 -> identify(backpropagation_context)
 -> compare(existing_relations)
 -> preserve(network_loss_graph_shape_and_derivative_invariants)
 -> update(affected_entities)
 -> update(affected_edges)
 -> recompute(CONFIDENCE)
 -> regenerate(FLOW_HASH)

# ---------- FLOW HASH ----------
FLOW_HASH := hash(
 SOURCE_URL
 + ENTITY
 + CONCEPTS
 + BACKPROPAGATION
 + COMPUTATIONAL_GRAPH
 + CHAIN_RULE
 + NEURAL_NETWORK
 + WEIGHT
 + BIAS
 + ACTIVATION_FUNCTION
 + LOSS_FUNCTION
 + FORWARD_PASS
 + REVERSE_PASS
 + GRADIENT
 + ERROR_SIGNAL
 + SEMANTIC_EQUIVALENCE
 + CONSTRAINTS
)

# ---------- EXECUTION ----------
INPUT(query)
 -> BACKPROPAGATION
 -> NETWORK_LOSS_IDENTIFICATION
 -> FORWARD_PASS
 -> OUTPUT_ADJOINT_INITIALISATION
 -> REVERSE_CHAIN_RULE_PROPAGATION
 -> PARAMETER_GRADIENT_COMPUTATION
 -> GRADIENT_ACCUMULATION
 -> GRADIENT_FLOW_ANALYSIS
 -> SHAPE_CONVENTION_VALIDATION
 -> GRADIENT_CHECKING
 -> OPTIMISER_HANDOFF
 -> RECURSIVE_REASONING(query)
 -> CONFIDENCE
 -> OUTPUT(semantic_result)

END FLOW