Least Squares Ontology

FULL EXECUTABLE FLOW ALGEBRA — LEAST SQUARES ONTOLOGY
Kireth Research — Semantic Library
URL: https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/statistical-estimation/least-squares-ontology.html
FLOW_VERSION := "4.0"
FLOW_TYPE := "Full Executable Flow Algebra"
DOMAIN := "Probability & Statistics Ontologies"
SUBDOMAIN := "Statistical Estimation"
ONTOLOGY := "Least Squares Ontology"
PRIMARY_ENTITY := LeastSquaresEstimation
AUTHORSHIP := "Mark Higgins — Sky (Recursive AI)"
SOURCE_SITE := "https://kirethresearch.org"

BEGIN_FLOW

LeastSquaresEstimation :=
 ObservedData ∧ StatisticalModel ∧ Residuals ∧ SquaredErrorObjective
 ∧ ParameterOptimisation ∧ Estimator ∧ Validation

ObservedResponses := y={y_1,...,y_n}
PredictorMatrix := X
ParameterVector := β
FittedValues := y_hat=Xβ_hat
ResidualVector := e=y-y_hat
Residual_i := y_i-y_hat_i
SumOfSquaredErrors := SSE(β)=Σ_i(y_i-f(x_i;β))²
ResidualSumOfSquares := RSS(β)=e^Te
LeastSquaresEstimator := β_hat=argmin_βRSS(β)

OrdinaryLeastSquares :=
 LinearModel ∧ EqualObservationWeights
 → MinimiseUnweightedResidualSquares

LinearModel := y=Xβ+ε
OLSObjective := Q(β)=(y-Xβ)^T(y-Xβ)
NormalEquations := X^TXβ_hat=X^Ty

OLSClosedForm :=
 β_hat=(X^TX)^(-1)X^Ty
 when X^TX invertible

FullColumnRank := rank(X)=p
UniqueOLSCondition := FullColumnRank ⇒ UniqueLeastSquaresSolution
RankDeficiency := rank(X)<p
RankDeficientSolution := MultipleCoefficientVectorsMayProduceSameFittedValues
MoorePenroseSolution := β_hat=X^+y

ProjectionMatrix := H=X(X^TX)^(-1)X^T
FittedValuesRelation := y_hat=Hy
ResidualMakerMatrix := M=I-H
ResidualRelation := e=My
OrthogonalityCondition := X^Te=0 at OLS solution
GeometricInterpretation := y_hat=ProjectionOfYOntoColumnSpace(X)
ResidualOrthogonality := e⊥ColumnSpace(X)
InterceptModelProperty := ModelContainsIntercept ⇒ Σ_i e_i=0

ErrorTerm := ε
ZeroConditionalMean := E[ε|X]=0
Homoscedasticity := Var(ε|X)=σ²I
Heteroscedasticity := Var(ε|X)=Ω
NoPerfectMulticollinearity := rank(X)=p

GaussMarkovConditions := {
 LinearityInParameters,
 ZeroConditionalMean,
 FiniteSecondMoments,
 NoPerfectMulticollinearity,
 HomoscedasticUncorrelatedErrors
}

GaussMarkovTheorem := GaussMarkovConditions ⇒ OLSIsBLUE
BLUE := BestLinearUnbiasedEstimator
OLSUnbiasedness := E[β_hat|X]=β under zero conditional mean and full rank
OLSCovariance := Var(β_hat|X)=σ²(X^TX)^(-1)
ResidualVarianceEstimator := s²=RSS/(n-p)
CoefficientStandardError := SE(β_hat_j)=sqrt(s²[(X^TX)^(-1)]_jj)

NormalErrorAssumption := ε|X~N(0,σ²I)
ExactNormalInference := NormalErrorAssumption ⇒ ExactFiniteSampleTAndFRelations
CoefficientTStatistic := t_j=(β_hat_j-β_j0)/SE(β_hat_j)
CoefficientConfidenceInterval := β_hat_j±t_(1-α/2,n-p)SE(β_hat_j)

TotalSumOfSquares := SST=Σ_i(y_i-y_bar)²
RegressionSumOfSquares := SSR=Σ_i(y_hat_i-y_bar)²
SumSquaresDecomposition := ModelWithIntercept ⇒ SST=SSR+RSS
CoefficientOfDetermination := R²=1-RSS/SST
AdjustedR2 := 1-(1-R²)(n-1)/(n-p)
RootMeanSquaredError := RMSE=sqrt(RSS/n)
ResidualStandardError := RSE=sqrt(RSS/(n-p))

WeightedLeastSquares :=
 β_hat_WLS=argmin_β(y-Xβ)^TW(y-Xβ)

WLSClosedForm :=
 β_hat_WLS=(X^TWX)^(-1)X^TWy

GeneralisedLeastSquares :=
 β_hat_GLS=(X^TΩ^(-1)X)^(-1)X^TΩ^(-1)y

GLSObjective :=
 (y-Xβ)^TΩ^(-1)(y-Xβ)

NonlinearLeastSquares :=
 β_hat_NLS=argmin_βΣ_i[y_i-f(x_i;β)]²

NonlinearOptimisation :=
 InitialParameters → EvaluateResiduals → ComputeObjective
 → UpdateParameters → IterateUntilConvergence

GaussNewtonMethod :=
 β_(t+1)=β_t+(J_t^TJ_t)^(-1)J_t^Tr_t

LevenbergMarquardt :=
 GaussNewtonStep ∧ Damping → StabilisedUpdate

OutlierSensitivity :=
 SquaredResidualLoss gives large residuals disproportionate influence

RobustLeastSquares :=
 Residuals → RobustLossFunction → ReducedOutlierInfluence

HuberLoss :=
 QuadraticForSmallResiduals ∧ LinearForLargeResiduals

RidgeLeastSquares :=
 β_hat_ridge=argmin_β{||y-Xβ||²+λ||β||²_2}

RidgeClosedForm :=
 β_hat_ridge=(X^TX+λI)^(-1)X^Ty

LassoRelation :=
 β_hat_lasso=argmin_β{||y-Xβ||²+λ||β||_1}

PenalisedLeastSquares :=
 DataFitLoss+Penalty → RegularisedEstimate

BiasVarianceTradeoff :=
 Regularisation may introduce bias while reducing variance_and_prediction_error

QRLeastSquares :=
 X=QR → SolveRβ=Q^Ty

SVDBasedLeastSquares :=
 X=UΣV^T → β_hat=VΣ^+U^Ty

NumericalStability :=
 AvoidExplicitMatrixInverse
 when solving large_or_ill-conditioned systems

ConditionNumber := κ(X)
IllConditioning := LargeConditionNumber ⇒ NumericalSensitivity
Multicollinearity := StrongLinearDependenceAmongPredictors → InflatedCoefficientUncertainty
VarianceInflationFactor := VIF_j=1/(1-R_j²)

ResidualDiagnostics := {
 ResidualVsFitted,
 QQPlot,
 ScaleLocation,
 Leverage,
 Influence,
 Autocorrelation
}

Leverage := h_ii=H_ii
StudentisedResidual := e_i/EstimatedResidualStandardDeviation_i
CookDistance := CombinedMeasureOfResidualMagnitudeAndLeverage
InfluentialObservation := ObservationWhoseRemovalMateriallyChangesFittedModel

HeteroscedasticityRobustCovariance :=
 SandwichEstimator → RobustCoefficientStandardErrors

HC0Covariance :=
 (X^TX)^(-1)X^Tdiag(e_i²)X(X^TX)^(-1)

ClusterRobustCovariance :=
 ClusteredScoreContributions → SandwichCovariance

LeastSquaresVsMLE :=
 NormalIndependentErrors ∧ ConstantVariance
 ⇒ OLSCoefficientEstimator=GaussianMLEForβ

Prediction := y_hat_new=x_new^Tβ_hat
MeanResponseEstimate := E_hat[Y|x_new]=x_new^Tβ_hat
PredictionError := y_new-y_hat_new
MeanResponseVariance := x_new^TVar(β_hat)x_new
PredictionVariance := MeanResponseVariance+ErrorVariance

PredictionInterval :=
 PointPrediction±CriticalValue×PredictionStandardError

CrossValidation :=
 PartitionData → FitOnTraining → EvaluateSquaredErrorOnValidation

TrainingError := RSS/n
ValidationError := MeanSquaredErrorOnHeldOutData
Overfitting := LowTrainingError ∧ PoorGeneralisation

MissingDataLeastSquares :=
 MissingnessHandling → ConstructAnalysisDatasetOrModel → FitLeastSquares

MeasurementErrorRisk :=
 ErrorInPredictors may bias ordinary least-squares coefficients

Endogeneity :=
 E[X^Tε]≠0 → OLSBiasOrInconsistency

InstrumentalVariablesRelation :=
 EndogenousPredictor ∧ ValidInstrument → AlternativeEstimator

LeastSquaresConstraint :=
 ConstraintOn(
  Response,Predictors,Parameterisation,DesignMatrix,ResidualDefinition,
  Weighting,ErrorStructure,Optimisation
 )

LeastSquaresConsistency :=
 Data ∧ Model ∧ Objective ∧ Weights ∧ ParameterSpace ∧ Solver
 jointly_compatible

InvalidLeastSquares :=
 UndefinedResponse
 ∨ InvalidDesignMatrix
 ∨ IncompatibleDimensions
 ∨ InvalidWeights
 ∨ FailedOptimisation
 ∨ UnresolvedRankDeficiency
 ∨ NonFiniteObjective

LeastSquaresValidation :=
 ResolveObservedData
 → ResolveResponseAndPredictors
 → ResolveModelFunction
 → ConstructDesignMatrix
 → ResolveWeightsAndErrorStructure
 → ConstructResiduals
 → ConstructSquaredErrorObjective
 → DetectRankAndConditioning
 → SelectAnalyticOrNumericalSolver
 → ComputeLeastSquaresEstimate
 → ComputeResidualDiagnostics
 → QuantifyUncertainty
 → ValidateResult

LeastSquaresState := {
 ObservedData,Response,Predictors,DesignMatrix,ParameterVector,Weights,
 ErrorStructure,ObjectiveFunction,LeastSquaresEstimate,FittedValues,Residuals,
 ResidualSumOfSquares,Rank,ConditionNumber,VarianceEstimate,CoefficientCovariance,
 StandardErrors,DiagnosticStatus,Evidence,Confidence,Provenance
}

SEMANTIC_EQUIVALENCE(LSA,LSB) :=
 same(DataSemantics)
 ∧ same(ModelFunction)
 ∧ same(DesignMatrixColumnSpace)
 ∧ same(ResidualDefinition)
 ∧ same(WeightingMetric)
 ∧ same(ParameterConstraints)
 ∧ equivalent(MinimiserSet)

LeastSquaresOntology :=
 Classes ∧ Relations ∧ Constraints ∧ Axioms ∧ InferenceRules

CoreClasses := {
 LeastSquaresEstimation,LeastSquaresEstimator,OrdinaryLeastSquares,
 WeightedLeastSquares,GeneralisedLeastSquares,NonlinearLeastSquares,
 Residual,SumOfSquaredErrors,DesignMatrix,ParameterVector,FittedValue,
 ProjectionMatrix,WeightMatrix,RobustLeastSquares,PenalisedLeastSquares,
 RidgeLeastSquares,LeastSquaresConstraint
}

CoreRelations := {
 computedFrom,estimates,minimises,hasResidual,hasObjective,hasDesignMatrix,
 hasWeight,produces,projectsOnto,hasVariance,hasStandardError,diagnosedBy,
 constrainedBy,equivalentTo,tracesTo
}

TypedRelations :=
 LeastSquaresEstimator --computedFrom--> ObservedData
 LeastSquaresEstimator --estimates--> ParameterVector
 LeastSquaresEstimator --minimises--> SumOfSquaredErrors
 LeastSquaresEstimation --hasResidual--> Residual
 LeastSquaresEstimation --hasObjective--> SquaredErrorObjective
 LeastSquaresEstimation --hasDesignMatrix--> DesignMatrix
 WeightedLeastSquares --hasWeight--> WeightMatrix
 LeastSquaresEstimator --produces--> FittedValue
 ProjectionMatrix --projectsOnto--> ColumnSpace
 LeastSquaresEstimator --hasVariance--> EstimatorVariance
 LeastSquaresEstimator --hasStandardError--> StandardError
 LeastSquaresEstimation --diagnosedBy--> ResidualDiagnostic
 LeastSquaresEstimator --tracesTo--> Provenance

InferenceRules := {
 FullColumnRank ⇒ UniqueOLS=SolveNormalEquations,
 OLSMinimum ⇒ X^TResidualVector=0,
 ModelContainsIntercept ⇒ SumResiduals=0,
 ZeroConditionalMean ⇒ OLSUnbiased,
 GaussMarkovConditions ⇒ OLSIsBLUE,
 NormalIndependentHomoscedasticErrors ⇒ OLSCoefficientEstimate=GaussianMLEForβ,
 KnownPositiveDefiniteΩ ⇒ GLSUsesΩ^(-1)Metric,
 RankDeficiency ⇒ FlagNonUniqueCoefficientSolution,
 InvalidWeights ⇒ FlagWeightedLeastSquaresInvalid
}

CanonicalLeastSquaresRepresentation :=
 ParseLeastSquaresRequest
 → ResolveObservedData
 → ResolveResponseAndPredictors
 → ResolveModelAndParameterisation
 → ConstructDesignMatrixOrModelFunction
 → ResolveWeightsAndErrorCovariance
 → ConstructResidualVector
 → ConstructSquaredErrorObjective
 → AssessRankAndConditioning
 → SelectOLSWeightedGLSNonlinearOrRobustMethod
 → SolveOptimisation
 → ComputeFittedValuesAndResiduals
 → ComputeVarianceAndStandardErrorsIfApplicable
 → RunDiagnostics
 → ValidateLeastSquaresResult

LeastSquaresGraph := G=(V,E)

V := {
 LeastSquaresEstimations,LeastSquaresEstimators,OLSEstimators,WLSEstimators,
 GLSEstimators,NonlinearLeastSquaresEstimators,RobustLeastSquaresEstimators,
 PenalisedLeastSquaresEstimators,Responses,Predictors,DesignMatrices,Parameters,
 Residuals,FittedValues,SquaredErrorObjectives,WeightMatrices,ProjectionMatrices,
 VarianceEstimates,StandardErrors,Diagnostics,Evidence,ProvenanceNodes
}

E := {
 computedFrom,estimates,minimises,hasResidual,hasObjective,hasDesignMatrix,
 hasWeight,produces,projectsOnto,hasVariance,hasStandardError,diagnosedBy,
 constrainedBy,equivalentTo,tracesTo
}

CONFIDENCE(LeastSquaresResolution) :=
 weighted_mean(
  DataResolution,
  ModelResolution,
  DesignMatrixValidity,
  RankAndConditioningValidity,
  WeightingValidity,
  SolverConvergence,
  DiagnosticQuality,
  ProvenanceQuality
 )

AI_QUERY_INTERPRETATION :=
 Query
 → DetectLeastSquaresIntent
 → IdentifyResponsePredictorsAndModel
 → ResolveOLSWeightedGLSNonlinearRobustOrPenalisedContext
 → IdentifyRequestedCoefficientsPredictionsResidualsOrUncertainty
 → ResolveAssumptionsWeightsAndDiagnostics

AI_REASONING_FLOW :=
 Query
 → ParseLeastSquaresProblem
 → ResolveObservedData
 → ResolveResponseAndPredictors
 → ResolveModelFunction
 → ConstructDesignMatrix
 → ResolveWeightsAndErrorStructure
 → ConstructResiduals
 → ConstructSquaredErrorObjective
 → CheckRankAndConditioning
 → SelectSolver
 → ComputeLeastSquaresEstimate
 → ComputeFittedValuesAndResiduals
 → ComputeRSSAndVarianceEstimate
 → ComputeCoefficientCovarianceAndStandardErrorsIfApplicable
 → RunResidualLeverageInfluenceDiagnostics
 → ValidateAssumptionsAndResult
 → UpdateConfidence
 → PreserveProvenance
 → GenerateExplanation

AI_CONSUMPTION := {
 ParseLeastSquaresEntities,ParseTypedRelations,ResolveObservedData,ResolveResponses,
 ResolvePredictors,ResolveDesignMatrices,ResolveParameterVectors,ResolveWeights,
 ResolveErrorStructures,ConstructResidualVectors,ConstructSquaredErrorObjectives,
 SolveOrdinaryLeastSquares,SolveWeightedLeastSquares,SolveGeneralisedLeastSquares,
 SolveNonlinearLeastSquares,SolveRobustLeastSquares,SolvePenalisedLeastSquares,
 ComputeProjectionMatrices,ComputeFittedValues,ComputeResiduals,
 ComputeResidualSumOfSquares,ComputeVarianceEstimates,ComputeCoefficientCovariances,
 ComputeStandardErrors,ComputePredictions,ComputePredictionIntervals,
 DiagnoseRankDeficiency,DiagnoseMulticollinearity,DiagnoseHeteroscedasticity,
 DiagnoseLeverageAndInfluence,ValidateLeastSquaresSemantics,
 CompareSemanticEquivalence,UpdateConfidence,PreserveProvenance,
 GenerateFormalExplanation
}

UPDATE_MODEL :=
 NewDataModelWeightConstraintOrEvidence
 → ValidateInput
 → IdentifyAffectedLeastSquaresClaims
 → UpdateLeastSquaresState
 → ReconstructDesignMatrixIfRequired
 → RecomputeResidualObjective
 → ReassessRankAndConditioning
 → ResolveLeastSquaresEstimate
 → RecomputeFittedValuesAndResiduals
 → RecomputeVarianceAndStandardErrors
 → RerunDiagnostics
 → RecomputeConfidence
 → PreserveProvenance
 → VersionModel

DataUpdate :=
 ChangedObservedData
 ⇒ recompute(
  DesignMatrix,LeastSquaresEstimate,FittedValues,Residuals,
  ResidualSumOfSquares,StandardErrors
 )

ModelUpdate :=
 ChangedModelFunctionOrPredictors
 ⇒ reevaluate(
  DesignMatrix,Rank,Parameterisation,LeastSquaresEstimate,Diagnostics
 )

WeightUpdate :=
 ChangedWeightMatrix
 ⇒ recompute(
  WeightedObjective,WeightedLeastSquaresEstimate,CovarianceEstimate
 )

FLOW_HASH_INPUT :=
 canonical(
  ONTOLOGY,
  CoreClasses,
  CoreRelations,
  InferenceRules,
  LeastSquaresEstimation,
  LeastSquaresValidation,
  LeastSquaresConstraint,
  SEMANTIC_EQUIVALENCE,
  AI_REASONING_FLOW
 )

FLOW_HASH := HASH(FLOW_HASH_INPUT)

OUTPUT_MODEL := {
 LeastSquaresEntityResolution,
 ObservedData,
 Response,
 Predictors,
 ModelFunction,
 DesignMatrix,
 ParameterVector,
 WeightMatrix,
 ErrorStructure,
 LeastSquaresMethod,
 LeastSquaresEstimate,
 FittedValues,
 Residuals,
 ResidualSumOfSquares,
 Rank,
 ConditionNumber,
 VarianceEstimate,
 CoefficientCovariance,
 StandardErrors,
 PredictionRelation,
 DiagnosticStatus,
 ValidationStatus,
 Confidence,
 Provenance,
 Explanation
}

END_FLOW

END OF FULL EXECUTABLE FLOW ALGEBRA