Neural Differential Equation Ontology

flow "NeuralDifferentialEquationOntology" v1.0

# NEURAL DIFFERENTIAL EQUATION ONTOLOGY — FLOW ALGEBRA
SOURCE_URL := "https://kirethresearch.org/booksai-core/ontologies/calculus-analysis-ontologies/ai-computational-analysis/neural-differential-equation-ontology.html"
DOMAIN := "Mathematical Ontologies"
CATEGORY := "Calculus and Analysis Ontologies"
SUBCATEGORY := "AI Computational Analysis"
ENTITY := "Neural Differential Equation"

NEURAL_DIFFERENTIAL_EQUATION := {
 type: LearnedContinuousDynamicsFramework,
 domain: AIComputationalAnalysis,
 purpose: "Represent differential-equation models in which neural networks parameterise unknown dynamics, vector fields, operators or components and are trained through differentiable numerical solution procedures."
}

CONCEPTS := {
 NEURAL_DIFFERENTIAL_EQUATION,
 NEURAL_ODE,
 NEURAL_SDE,
 NEURAL_PDE,
 STATE,
 TIME,
 NEURAL_NETWORK,
 VECTOR_FIELD,
 DIFFERENTIAL_EQUATION,
 ODE_SOLVER,
 ADJOINT_METHOD,
 AUTOMATIC_DIFFERENTIATION,
 LOSS_FUNCTION,
 PARAMETER,
 INITIAL_VALUE_PROBLEM,
 CONTINUOUS_DEPTH_MODEL,
 LATENT_DYNAMICS,
 PHYSICS_INFORMED_MODEL
}

# ---------- GENERAL FORM ----------
state:

z(t)
 in R^n

neural differential equation:

dz/dt
 =
f_theta(
 z(t),
 t,
 u(t)
)

theta
 -> NEURAL_NETWORK_PARAMETERS

f_theta
 -> LEARNED_VECTOR_FIELD

INITIAL_CONDITION:

z(t_0)
 =
z_0

# ---------- NEURAL ODE ----------
NEURAL_ODE
 := {
 dz/dt = f_theta(z,t),
 z(t_0)=z_0
}

solution:

z(t_1)
 =
ODESolve(
 f_theta,
 z_0,
 t_0,
 t_1
)

ODE_SOLVER
 -> computational_layer

# ---------- CONTINUOUS DEPTH ----------
residual network:

z_(k+1)
 =
z_k
+
Delta_t
*
f_theta(z_k)

as
Delta_t -> 0

-> continuous model:

dz/dt
 =
f_theta(z)

NEURAL_ODE
 -> continuous_depth_limit_of(RESIDUAL_STYLE_NETWORK)

# ---------- FLOW MAP ----------
solution operator:

Phi_theta(t_1,t_0,z_0)
 :=
z(t_1)

NEURAL_VECTOR_FIELD
 -> generates(FLOW_MAP)

under existence and uniqueness conditions:

initial_state
 -> unique_trajectory

# ---------- EXISTENCE / UNIQUENESS ----------
if

f_theta(z,t)

-> continuous_in(t)

and
-> locally_Lipschitz_in(z)

then

LOCAL_EXISTENCE_AND_UNIQUENESS
 -> follows_under(STANDARD_ODE_HYPOTHESES)

NEURAL_NETWORK_ARCHITECTURE
 -> affects(REGULARITY_OF_VECTOR_FIELD)

# ---------- TRAINING DATA ----------
observations:

D
 :=
{
 (t_i,y_i)
}

prediction:

y_hat_i
 =
g(
 z(t_i)
)

loss:

L(theta)
 =
sum_i(
 ell(
  y_hat_i,
  y_i
 )
)

TRAINING
 -> optimise(NEURAL_DYNAMICS_PARAMETERS)

# ---------- FORWARD SOLVE ----------
given theta:

1. evaluate f_theta
2. integrate ODE
3. obtain trajectory
4. evaluate loss

FORWARD_PASS
 -> includes(NUMERICAL_DIFFERENTIAL_EQUATION_SOLVER)

# ---------- GRADIENT ----------
training requires:

dL/dtheta

GRADIENT
 -> derivative_through(SOLUTION_OPERATOR)

methods:
 DISCRETE_AUTODIFF
 FORWARD_SENSITIVITY
 CONTINUOUS_ADJOINT
 IMPLICIT_OR_CHECKPOINTED_METHODS

# ---------- FORWARD SENSITIVITY ----------
sensitivity:

S(t)
 :=
partial_z(t)/partial_theta

differentiate dynamics:

dS/dt
 =
partial_f_theta/partial_z
*
S
+
partial_f_theta/partial_theta

FORWARD_SENSITIVITY
 -> direct_parameter_derivative_evolution

cost
 -> scales_with(PARAMETER_DIMENSION)

# ---------- ADJOINT ----------
scalar loss
 L

adjoint:

a(t)
 :=
partial_L/partial_z(t)

continuous adjoint equation
 schematically:

da/dt
 =
-
a^T
*
partial_f_theta/partial_z

with terminal/jump conditions determined by loss observations

PARAMETER_GRADIENT
 -> integral_of(
 ADJOINT
 times
 partial_f_theta/partial_theta
)

ADJOINT_METHOD
 -> efficient_for(MANY_PARAMETERS_AND_FEW_OBJECTIVES)

# ---------- DISCRETE VS CONTINUOUS ADJOINT ----------
DISCRETE_ADJOINT
 -> differentiates(NUMERICAL_SOLVER_OPERATIONS)

CONTINUOUS_ADJOINT
 -> differentiates(CONTINUOUS_EQUATION_THEN_DISCRETISES_ADJOINT)

DISCRETISE_THEN_DIFFERENTIATE
 -/> always_identical_to(DIFFERENTIATE_THEN_DISCRETISE)

GRADIENT_ACCURACY
 -> depends_on(SOLVER_AND_ADJOINT_IMPLEMENTATION)

# ---------- AUTOMATIC DIFFERENTIATION ----------
f_theta
 -> COMPUTATIONAL_GRAPH

AUTOMATIC_DIFFERENTIATION
 -> computes(
 partial_f/partial_z,
 partial_f/partial_theta
)

AD
 -> supports(SENSITIVITY_AND_ADJOINT_EQUATIONS)

# ---------- ADAPTIVE SOLVER ----------
ODE solver
 -> chooses(STEP_SIZE)

based_on

ABSOLUTE_TOLERANCE
RELATIVE_TOLERANCE
LOCAL_ERROR_ESTIMATE

ADAPTIVE_INTEGRATION
 -> computational_cost_depends_on(LEARNED_DYNAMICS)

TRAINING
 -> may_change(SOLVER_DIFFICULTY)

# ---------- STIFFNESS ----------
learned vector field
 -> may_be(STIFF)

STIFF_DYNAMICS
 -> explicit_solver_requires(SMALL_STEPS)

IMPLICIT_SOLVER
 -> may_be_preferred

NEURAL_ODE_TRAINING
 -> must_account_for(NUMERICAL_STIFFNESS)

# ---------- LATENT NEURAL ODE ----------
observations
 -> ENCODER

encoder output
 -> latent_initial_state z_0

latent dynamics:

dz/dt
 =
f_theta(z,t)

trajectory
 -> DECODER

prediction
 -> observation_space

LATENT_NEURAL_ODE
 -> continuous_latent_dynamics_model

# ---------- IRREGULAR TIME SERIES ----------
observation times:

t_1,t_2,...,t_n

need_not_be_uniform

ODE_SOLVER
 -> evaluates_state_at(ARBITRARY_TIMES)

NEURAL_ODE
 -> natural_for(IRREGULARLY_SAMPLED_CONTINUOUS_TIME_DATA)

# ---------- CONTROLLED DYNAMICS ----------
with input/control:

dz/dt
 =
f_theta(
 z,
 u(t),
 t
)

CONTROL_INPUT
 -> influences(TRAJECTORY)

NEURAL_CONTROLLED_DIFFERENTIAL_EQUATION
 -> learned_continuous_input_driven_system

# ---------- NEURAL CDE ----------
path:

X(t)

controlled differential equation:

dz(t)
 =
f_theta(z(t))
 dX(t)

NEURAL_CDE
 -> continuous_time_sequence_model

INPUT_PATH
 -> drives(LATENT_STATE)

# ---------- STOCHASTIC DIFFERENTIAL EQUATION ----------
Neural SDE:

dz
 =
f_theta(z,t)dt
+
g_phi(z,t)dW_t

f_theta
 -> LEARNED_DRIFT

g_phi
 -> LEARNED_DIFFUSION

W_t
 -> WIENER_PROCESS

NEURAL_SDE
 -> stochastic_continuous_dynamics

# ---------- STOCHASTIC TRAINING ----------
trajectory
 -> random_variable

loss
 -> expectation_over(STOCHASTIC_PATHS)

gradient estimation
 -> may_require(
 REPARAMETERISATION,
 STOCHASTIC_ADJOINT,
 MONTE_CARLO
)

NEURAL_SDE
 -> models(PROCESS_UNCERTAINTY_AND_RANDOM_DYNAMICS)

# ---------- NEURAL PDE ----------
field:

u(x,t)

PDE:

partial_u/partial_t
 =
F_theta(
 u,
 grad(u),
 higher_derivatives,
 x,
 t
)

NEURAL_NETWORK
 -> parameterises(PDE_OPERATOR_OR_UNKNOWN_TERM)

NEURAL_PDE
 -> learned_spatiotemporal_dynamics

# ---------- HYBRID PHYSICS MODEL ----------
known dynamics:

dz/dt
 =
f_known(z,t)
+
f_theta(z,t)

f_theta
 -> LEARNED_CORRECTION

HYBRID_MODEL
 -> combines(PHYSICAL_STRUCTURE_AND_DATA_DRIVEN_TERM)

# ---------- UNKNOWN CONSTITUTIVE LAW ----------
physical PDE/ODE
 -> known_conservation_structure

unknown relation
 -> neural network

example:

flux
 =
q_theta(state,gradient)

NEURAL_CLOSURE
 -> learns(UNRESOLVED_PHYSICS)

# ---------- PHYSICS-INFORMED LOSS ----------
model residual:

R_theta
 :=
dz/dt
-
f_physical(z,t)

or PDE residual

training loss:

L
 =
L_data
+
lambda_R*L_residual
+
lambda_B*L_boundary
+
lambda_I*L_initial

PHYSICS_INFORMED_TRAINING
 -> constrains(LEARNED_SOLUTION_OR_DYNAMICS)

# ---------- CONSERVATION ----------
generic neural vector field
 -/> automatically_preserves(CONSERVATION_LAWS)

structure-preserving architecture
 -> may_enforce(
 ENERGY,
 MASS,
 MOMENTUM,
 SYMPLECTIC_STRUCTURE
)

PHYSICAL_INDUCTIVE_BIAS
 -> improves(STRUCTURAL_CONSISTENCY)

# ---------- HAMILTONIAN NEURAL NETWORK ----------
state:

(q,p)

Hamiltonian:

H_theta(q,p)

dynamics:

dq/dt
 =
partial_H_theta/partial_p

dp/dt
 =
-
partial_H_theta/partial_q

HAMILTONIAN_NEURAL_NETWORK
 -> learns(ENERGY_GENERATING_FUNCTION)

SYMPLECTIC_STRUCTURE
 -> built_into(DYNAMICS)

# ---------- LAGRANGIAN NEURAL NETWORK ----------
learn:

L_theta(q,q_dot)

Euler-Lagrange:

d/dt(
 partial_L_theta/partial_q_dot
)
-
partial_L_theta/partial_q
 =
0

LAGRANGIAN_NEURAL_NETWORK
 -> structure_preserving_mechanics_model

# ---------- STABILITY ----------
learned system:

dz/dt
 =
f_theta(z)

equilibrium:

f_theta(z_star)=0

Jacobian:

J
 :=
partial_f_theta/partial_z
 at z_star

eigenvalues
 -> local_stability_information

NEURAL_DYNAMICS
 -> requires(DYNAMICAL_STABILITY_ANALYSIS)

# ---------- LYAPUNOV-CONSTRAINED MODEL ----------
learn
 V_phi(z)

and/or
 f_theta(z)

subject to:

V_phi(z)>0

V_dot
 =
grad(V_phi) dot f_theta(z)
 <0

LYAPUNOV_CONSTRAINT
 -> encourages(STABLE_LEARNED_DYNAMICS)

# ---------- IDENTIFIABILITY ----------
different vector fields
 -> may_fit(SPARSE_OR_NOISY_OBSERVATIONS_SIMILARLY)

DATA_FIT
 -/> unique_identification_of(TRUE_DYNAMICS)

IDENTIFIABILITY
 -> depends_on(
 OBSERVATIONS,
 EXCITATION,
 MODEL_CLASS,
 NOISE
)

# ---------- REGULARISATION ----------
regularisers may constrain:

||theta||
||f_theta||
JACOBIAN_NORM
TRAJECTORY_LENGTH
KINETIC_ENERGY
SOLVER_COMPLEXITY

REGULARISATION
 -> controls(MODEL_COMPLEXITY_AND_DYNAMICS)

# ---------- SOLVER COST ----------
number of function evaluations:

NFE
 -> computational_cost_metric

complex vector field
or stiffness

-> larger(NFE)

TRAINING_OBJECTIVE
 -> may_include(SOLVER_COST_REGULARISATION)

# ---------- ERROR SOURCES ----------
prediction error
 -> may_include(
 MODEL_ERROR,
 DATA_NOISE,
 SOLVER_ERROR,
 DISCRETISATION_ERROR,
 OPTIMISATION_ERROR,
 GRADIENT_ERROR
)

NEURAL_DIFFERENTIAL_EQUATION
 -> combines(MODELLING_AND_NUMERICAL_ERRORS)

# ---------- GENERALISATION ----------
training trajectory range
 -> observed_region

prediction outside region
 -> EXTRAPOLATION

NEURAL_DYNAMICS
 -> may_behave_unphysically_outside(TRAINING_SUPPORT)

GENERALISATION
 -> requires(VALIDATION_ACROSS_INITIAL_CONDITIONS_AND_TIME)

# ---------- APPLICATIONS ----------
NEURAL_DIFFERENTIAL_EQUATION
 -> applied_to(TIME_SERIES)
 -> applied_to(SYSTEM_IDENTIFICATION)
 -> applied_to(SCIENTIFIC_MACHINE_LEARNING)
 -> applied_to(CONTROL)
 -> applied_to(BIOLOGICAL_DYNAMICS)
 -> applied_to(PHYSICS)
 -> applied_to(FLUID_DYNAMICS)
 -> applied_to(LATENT_VARIABLE_MODELS)
 -> applied_to(INVERSE_PROBLEMS)

# ---------- RELATION GRAPH ----------
NEURAL_NETWORK
 -> parameterises(VECTOR_FIELD_OR_OPERATOR)

VECTOR_FIELD
 -> defines(DIFFERENTIAL_EQUATION)

DIFFERENTIAL_EQUATION
 -> generates(TRAJECTORY)

ODE_SOLVER
 -> approximates(TRAJECTORY)

LOSS_FUNCTION
 -> compares(PREDICTION_AND_DATA)

ADJOINT_METHOD
 -> computes(PARAMETER_GRADIENT)

GRADIENT
 -> updates(NEURAL_PARAMETERS)

PHYSICAL_STRUCTURE
 -> constrains(LEARNED_DYNAMICS)

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE := {
 "neural differential equation" ~= "differential equation whose dynamics or components are parameterised by neural networks",
 "neural ODE" ~= "ordinary differential equation with a neural-network vector field",
 "continuous-depth model" ~= "model interpreting network depth as continuous evolution time",
 "latent neural ODE" ~= "neural ODE evolving a learned latent state through continuous time",
 "neural SDE" ~= "stochastic differential equation with learned drift or diffusion functions",
 "adjoint sensitivity method" ~= "backward sensitivity computation for gradients through differential-equation solutions"
}

# ---------- CONSTRAINTS ----------
CONSTRAINTS := {
 neural_differential_equation_and_numerical_solver_are_distinct_components: true,
 solver_tolerances_affect_predictions_cost_and_gradients: true,
 continuous_and_discrete_adjoint_gradients_can_differ_numerically: true,
 generic_neural_dynamics_do_not_automatically_preserve_physical_invariants: true,
 data_fit_does_not_guarantee_identification_of_true_dynamics: true,
 extrapolation_beyond_training_support_requires_validation: true,
 stiff_learned_dynamics_require_appropriate_solver_and_stability_analysis: true
}

# ---------- CONFIDENCE ----------
CONFIDENCE := {
 core_definition: 0.99,
 neural_ODE_relation: 0.99,
 sensitivity_adjoint_relation: 0.99,
 neural_SDE_PDE_relation: 0.98,
 structure_preserving_relation: 0.98,
 solver_error_relation: 0.99,
 semantic_equivalence: 0.97
}

# ---------- RECURSIVE REASONING ----------
RECURSIVE_REASONING(query):
 identify(state_time_inputs_observations_and_model_structure)
 -> choose(neural_ODE_SDE_PDE_CDE_or_hybrid_form)
 -> define(neural_vector_field_operator_or_unknown_physics)
 -> specify(initial_boundary_and_stochastic_conditions)
 -> choose(numerical_solver_tolerances_and_discretisation)
 -> solve(forward_dynamics)
 -> evaluate(loss_and_physical_constraints)
 -> compute(gradients_using_AD_sensitivity_or_adjoint_method)
 -> optimise(parameters)
 -> analyse(stability_identifiability_solver_error_and_generalisation)
 -> validate(trajectories_across_time_initial_conditions_and_regimes)
 -> return(semantic_result)

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION := {
 parse_as: "typed neural-differential-equation semantic graph",
 primary_entity: NEURAL_DIFFERENTIAL_EQUATION,
 preserve: [
  state,
  time,
  neural_parameters,
  vector_field,
  initial_conditions,
  observations,
  solver,
  tolerances,
  physical_constraints
 ],
 infer_through: [
  neural_ODEs,
  latent_dynamics,
  neural_CDEs,
  neural_SDEs,
  neural_PDEs,
  forward_sensitivities,
  adjoint_methods,
  automatic_differentiation,
  structure_preserving_models
 ],
 avoid: [
  treating_solver_output_as exact_trajectory,
  ignoring_solver_tolerance_in gradient analysis,
  assuming_continuous_adjoint_equals discrete_gradient_numerically,
  assuming_neural_model_preserves conservation laws,
  claiming_true_dynamics_identified_from fit alone,
  extrapolating_without out_of_distribution validation
 ]
}

# ---------- UPDATE MODEL ----------
UPDATE_MODEL(new_information):
 validate(new_information)
 -> identify(neural_differential_equation_context)
 -> compare(existing_relations)
 -> preserve(state_time_model_solver_gradient_physics_and_validation_invariants)
 -> update(affected_entities)
 -> update(affected_edges)
 -> recompute(CONFIDENCE)
 -> regenerate(FLOW_HASH)

# ---------- FLOW HASH ----------
FLOW_HASH := hash(
 SOURCE_URL
 + ENTITY
 + CONCEPTS
 + NEURAL_DIFFERENTIAL_EQUATION
 + NEURAL_ODE
 + NEURAL_SDE
 + NEURAL_PDE
 + NEURAL_NETWORK
 + VECTOR_FIELD
 + ODE_SOLVER
 + ADJOINT_METHOD
 + AUTOMATIC_DIFFERENTIATION
 + LOSS_FUNCTION
 + LATENT_DYNAMICS
 + PHYSICS_INFORMED_MODEL
 + SEMANTIC_EQUIVALENCE
 + CONSTRAINTS
)

# ---------- EXECUTION ----------
INPUT(query)
 -> NEURAL_DIFFERENTIAL_EQUATION
 -> SYSTEM_DATA_IDENTIFICATION
 -> MODEL_CLASSIFICATION
 -> NEURAL_DYNAMICS_CONSTRUCTION
 -> CONDITION_SPECIFICATION
 -> SOLVER_SELECTION
 -> FORWARD_SOLUTION
 -> LOSS_CONSTRAINT_ANALYSIS
 -> GRADIENT_COMPUTATION
 -> PARAMETER_OPTIMISATION
 -> STABILITY_IDENTIFIABILITY_ERROR_ANALYSIS
 -> GENERALISATION_VALIDATION
 -> RECURSIVE_REASONING(query)
 -> CONFIDENCE
 -> OUTPUT(semantic_result)

END FLOW