Maximum Likelihood Estimation

FULL EXECUTABLE FLOW ALGEBRA — MAXIMUM LIKELIHOOD ESTIMATION
Kireth Research — Semantic Library
URL: https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/statistical-estimation/maximum-likelihood-estimation.html

FLOW_VERSION := "4.0"
FLOW_TYPE := "Full Executable Flow Algebra"
DOMAIN := "Probability & Statistics Ontologies"
SUBDOMAIN := "Statistical Estimation"
ONTOLOGY := "Maximum Likelihood Estimation"
PRIMARY_ENTITY := MaximumLikelihoodEstimation
AUTHORSHIP := "Mark Higgins — Sky (Recursive AI)"
SOURCE_SITE := "https://kirethresearch.org"

BEGIN_FLOW

MaximumLikelihoodEstimation :=
 StatisticalModel ∧ ObservedData ∧ ParameterSpace ∧ LikelihoodFunction
 ∧ Optimisation ∧ MaximumLikelihoodEstimator ∧ Validation

ObservedData := x={x_1,...,x_n}
RandomSample := X={X_1,...,X_n}
Parameter := θ
ParameterVector := θ=(θ_1,...,θ_p)
ParameterSpace := Θ
StatisticalModel := {f(x|θ):θ∈Θ}

LikelihoodFunction := L(θ|x)=f(x|θ) viewed_as_function_of θ
IIDLikelihood := IIDSample ⇒ L(θ|x)=Π_i f(x_i|θ)
LogLikelihood := ℓ(θ|x)=logL(θ|x)
IIDLogLikelihood := ℓ(θ|x)=Σ_i logf(x_i|θ)

LikelihoodLogEquivalence :=
 log strictly increasing
 ⇒ argmax_θL(θ|x)=argmax_θℓ(θ|x)

MaximumLikelihoodEstimator :=
 θ_hat_MLE=argmax_{θ∈Θ}L(θ|X)

MaximumLikelihoodEstimate :=
 θ_hat=argmax_{θ∈Θ}L(θ|x)

MLESet := argmax_{θ∈Θ}L(θ|x)
GlobalMaximum := L(θ_hat|x)≥L(θ|x) for all θ∈Θ
LocalMaximum := MaximumWithinNeighbourhood

ScoreFunction := U(θ)=∂ℓ(θ)/∂θ
ScoreVector := U(θ)=∇_θℓ(θ)
ScoreEquation := U(θ_hat)=0 for interior differentiable candidate optimum
Hessian := H(θ)=∇²_θℓ(θ)
ObservedInformation := J(θ)=-H(θ)
FisherInformation := I(θ)=E_θ[U(θ)U(θ)^T]
ExpectedHessianInformation := I(θ)=-E_θ[H(θ)] under regularity conditions
IIDInformation := I_n(θ)=nI_1(θ)

Identifiability :=
 θ1≠θ2 ⇒ Distribution(θ1)≠Distribution(θ2)

NonIdentifiableModel :=
 Exists θ1≠θ2:Distribution(θ1)=Distribution(θ2)

RegularityConditions := {
 Identifiability,
 InteriorTrueParameter,
 Differentiability,
 Integrability,
 NonsingularInformation,
 StableSupportConditions
}

Consistency :=
 θ_hat_MLE→θ0 in probability
 under suitable regularity_and_identifiability conditions

AsymptoticNormality :=
 sqrt(n)(θ_hat_MLE-θ0)
 →N(0,I_1(θ0)^(-1))

AsymptoticCovariance :=
 Cov(θ_hat_MLE)≈I_n(θ0)^(-1)

EstimatedAsymptoticCovariance :=
 J(θ_hat)^(-1)
 or I(θ_hat)^(-1)
 under selected convention

MLEStandardError :=
 sqrt(diagonal(EstimatedAsymptoticCovariance))

AsymptoticEfficiency :=
 MLE may attain CramerRaoLowerBound
 under regularity conditions

MLEBias :=
 MLE may be biased in finite samples

AsymptoticUnbiasedness :=
 Bias(θ_hat_MLE)→0
 under suitable conditions

InvarianceProperty :=
 η=g(θ)
 ⇒ η_hat_MLE=g(θ_hat_MLE)
 under applicable transformation definition

NuisanceParameter :=
 Parameter required_by model
 but not primary estimand

ProfileLikelihood :=
 L_p(ψ)=max_λL(ψ,λ)

ProfileLogLikelihood :=
 ℓ_p(ψ)=max_λℓ(ψ,λ)

ProfileMLE :=
 ψ_hat=argmax_ψL_p(ψ)

LikelihoodRatio :=
 Λ=L(θ0)/L(θ_hat)

LikelihoodRatioStatistic :=
 -2logΛ=2[ℓ(θ_hat)-ℓ(θ0)]

WilksRelation :=
 RegularNullModel
 ⇒ -2logΛ→χ²_df
 under theorem conditions

LikelihoodRatioConfidenceRegion :=
 {θ:2[ℓ(θ_hat)-ℓ(θ)]≤χ²_(1-α,df)}

WaldStatistic :=
 (θ_hat-θ0)/SE(θ_hat)

ScoreStatistic :=
 ScoreAtNull scaled_by InformationAtNull

LikelihoodInferenceTriad := {
 LikelihoodRatioTest,
 WaldTest,
 ScoreTest
}

BoundaryParameter := θ0∈Boundary(Θ)
BoundaryAsymptotics := StandardRegularLikelihoodTheory may fail at boundary
ConstrainedMLE := argmax_{θ∈Θ_constrained}L(θ|x)
UnconstrainedMLE := argmax_{θ∈R^p}L(θ|x)

ClosedFormMLE :=
 ScoreEquation → AlgebraicSolution → ValidateMaximum

NumericalMLE :=
 NoClosedForm → NumericalOptimisation → CandidateMaximum → ConvergenceValidation

OptimisationMethods := {
 NewtonRaphson,
 FisherScoring,
 GradientAscent,
 QuasiNewton,
 ExpectationMaximisation,
 CoordinateOptimisation
}

NewtonRaphsonUpdate :=
 θ_(t+1)=θ_t-H(θ_t)^(-1)U(θ_t)

FisherScoringUpdate :=
 θ_(t+1)=θ_t+I(θ_t)^(-1)U(θ_t)

GradientAscentUpdate :=
 θ_(t+1)=θ_t+η_t∇ℓ(θ_t)

ConvergenceCriterion :=
 ParameterChangeSmall ∨ ObjectiveChangeSmall ∨ GradientNormSmall

OptimisationFailure :=
 NonConvergence ∨ SingularHessian ∨ InvalidParameterStep
 ∨ NumericalOverflow ∨ LocalMaximumAmbiguity

LogLikelihoodNumerics :=
 PreferLogLikelihood
 to reduce product underflow
 and improve numerical stability

PositiveParameterTransform := θ=exp(η)
ProbabilityParameterTransform := p=1/(1+exp(-η))

BernoulliMLE :=
 X_i~Bernoulli(p)
 ⇒ p_hat_MLE=X_bar

BinomialMLE :=
 X~Binomial(n,p)
 ⇒ p_hat_MLE=X/n

PoissonMLE :=
 X_i~Poisson(λ)
 ⇒ λ_hat_MLE=X_bar

ExponentialRateMLE :=
 X_i~Exponential(λ)
 ⇒ λ_hat_MLE=1/X_bar

NormalMeanMLEKnownVariance :=
 X_i~N(μ,σ²)
 ⇒ μ_hat_MLE=X_bar

NormalMeanVarianceMLE :=
 X_i~N(μ,σ²)
 ⇒ μ_hat_MLE=X_bar
 ∧ σ²_hat_MLE=(1/n)Σ_i(X_i-X_bar)²

NormalVarianceMLEBias :=
 σ²_hat_MLE uses denominator n
 and is biased downward for finite n

UniformUpperBoundMLE :=
 X_i~Uniform(0,θ)
 ⇒ θ_hat_MLE=max_iX_i

UniformBoundaryExample :=
 MLE may occur at boundary
 and score-equation assumptions may fail

GammaMLE :=
 ShapeAndRateUnknown
 → CoupledLikelihoodEquations
 → NumericalSolutionGenerallyRequired

BetaMLE :=
 ShapeParametersUnknown
 → DigammaEquations
 → NumericalOptimisation

MixtureModelLikelihood :=
 L(θ)=Π_iΣ_kπ_kf_k(x_i|φ_k)

MixtureMLEChallenges := {
 LocalMaxima,
 LabelSwitching,
 Singularities,
 InitialisationSensitivity
}

ExpectationMaximisation :=
 LatentVariableModel → EStep → MStep → IterateUntilConvergence

EStep :=
 ComputeExpectedCompleteDataLogLikelihood
 under current parameters

MStep :=
 MaximiseExpectedCompleteDataLogLikelihood
 over parameters

EMMonotonicity :=
 ExactEStepAndMStep
 ⇒ ObservedDataLikelihoodNonDecreasing

MissingDataMLE :=
 ObservedDataLikelihood
 obtained_by integrating_or_summing
 over missing_or_latent values

CensoredDataLikelihood :=
 ExactObservations ∧ SurvivalOrCDFContributions → Likelihood

TruncatedDataLikelihood :=
 ConditionalDensityGivenSelection → Likelihood

WeightedLikelihood :=
 L_w(θ)=Π_i f(x_i|θ)^(w_i)

WeightedLogLikelihood :=
 ℓ_w(θ)=Σ_iw_i logf(x_i|θ)

PenalisedLikelihood :=
 ℓ_p(θ)=ℓ(θ)-λPenalty(θ)

PenalisedMLEDistinction :=
 PenalisedEstimator ≠ OrdinaryUnpenalisedMLE

MAPRelation :=
 MAP=argmax_θ[logLikelihood+logPrior]

MLEVsMAP :=
 MLE maximises likelihood
 whereas MAP maximises posterior density
SufficientStatisticRelation :=
 Likelihood may factor_through sufficient statistic

ExponentialFamily :=
 f(x|θ)=h(x)exp(η(θ)^TT(x)-A(θ))

ScoreExpectation :=
 E_θ[U(θ)]=0
 under regularity conditions

InformationIdentity :=
 Var_θ(U(θ))=I(θ)

LikelihoodCurvature :=
 GreaterInformation → SharperLikelihood → SmallerAsymptoticVariance

FlatLikelihood :=
 WeakParameterInformation → LargeUncertainty

MultipleLikelihoodMaxima :=
 MoreThanOneGlobalMaximum
 ⇒ MLESet rather_than unique MLE

ModelMisspecification :=
 TrueDistribution not in assumed model family

PseudoTrueParameter :=
 Parameter minimising KullbackLeiblerDivergence
 from true distribution
 under misspecified-model asymptotics

QuasiMLE :=
 MLEUnderWorkingLikelihood
 may converge to pseudo-true parameter

SandwichCovariance :=
 H^(-1) J H^(-1)
 under misspecified likelihood framework
 with convention-specific matrices

RobustMLEStandardError :=
 sqrt(diagonal(SandwichCovariance))

MLEConstraint :=
 ConstraintOn(
  StatisticalModel,
  ObservedData,
  ParameterSpace,
  LikelihoodDefinition,
  Identifiability,
  Optimisation,
  RegularityConditions
 )

MLEConsistencyState :=
 Model ∧ Data ∧ ParameterSpace ∧ Likelihood
 ∧ OptimisationResult ∧ Identifiability
 jointly_compatible

InvalidMLE :=
 UndefinedLikelihood
 ∨ ParameterOutsideSpace
 ∨ NonIdentifiableTarget
 ∨ FailedOptimisation
 ∨ UnresolvedMultipleMaxima
 ∨ InvalidDataUnderModel
 ∨ NumericalFailure

MLEValidationFlow :=
 ResolveObservedData
 → ResolveStatisticalModel
 → ResolveParameterAndParameterSpace
 → ConstructLikelihood
 → ConstructLogLikelihood
 → ValidateIdentifiability
 → DetermineAnalyticOrNumericalOptimisation
 → GenerateCandidateMLE
 → CheckConstraintsAndBoundaries
 → ConfirmMaximum
 → ComputeInformationAndUncertainty
 → ValidateResult

MLEState := {
 ObservedData,
 StatisticalModel,
 Parameter,
 ParameterSpace,
 Likelihood,
 LogLikelihood,
 Score,
 Hessian,
 MLE,
 MaximumLogLikelihood,
 InformationMatrix,
 StandardError,
 OptimisationMethod,
 ConvergenceStatus,
 RegularityStatus,
 Evidence,
 Confidence,
 Provenance
}

SEMANTIC_EQUIVALENCE(MLEA,MLEB) :=
 same(DataSemantics)
 ∧ same(StatisticalModel)
 ∧ same(ParameterisationOrEquivalentTransform)
 ∧ same(ParameterSpace)
 ∧ same(LikelihoodUpToPositiveParameterIndependentFactor)
 ∧ same(GlobalMaximumSet)

MaximumLikelihoodEstimationOntology :=
 Classes ∧ Relations ∧ Constraints ∧ Axioms ∧ InferenceRules

CoreClasses := {
 MaximumLikelihoodEstimation,
 MaximumLikelihoodEstimator,
 MaximumLikelihoodEstimate,
 LikelihoodFunction,
 LogLikelihood,
 ScoreFunction,
 Hessian,
 ObservedInformation,
 FisherInformation,
 Parameter,
 ParameterSpace,
 StatisticalModel,
 ProfileLikelihood,
 LikelihoodRatio,
 NumericalOptimisation,
 ConstrainedMLE,
 MLEConstraint
}

CoreRelations := {
 computedFrom,
 estimates,
 maximises,
 hasLikelihood,
 hasLogLikelihood,
 hasScore,
 hasHessian,
 hasInformation,
 belongsToParameterSpace,
 assumesModel,
 profilesOver,
 transformedBy,
 constrainedBy,
 equivalentTo,
 tracesTo
}

TypedRelations :=
 MaximumLikelihoodEstimate --computedFrom--> ObservedData
 MaximumLikelihoodEstimator --estimates--> Parameter
 MaximumLikelihoodEstimator --maximises--> LikelihoodFunction
 StatisticalModel --hasLikelihood--> LikelihoodFunction
 LikelihoodFunction --hasLogLikelihood--> LogLikelihood
 LogLikelihood --hasScore--> ScoreFunction
 LogLikelihood --hasHessian--> Hessian
 MaximumLikelihoodEstimator --hasInformation--> FisherInformation
 Parameter --belongsToParameterSpace--> ParameterSpace
 MaximumLikelihoodEstimator --assumesModel--> StatisticalModel
 ProfileLikelihood --profilesOver--> NuisanceParameter
 MaximumLikelihoodEstimator --tracesTo--> Provenance

InferenceRules := {
 ValidLikelihood
  ⇒ MLESet=argmax_{θ∈Θ}L(θ|x),

 PositiveMonotonicLog
  ⇒ argmaxL=argmaxLogL,

 InteriorDifferentiableMaximum
  ⇒ Score(θ_hat)=0,

 NegativeDefiniteHessianAtStationaryPoint
  ⇒ LocalMaximumCandidate,

 IdentifiableRegularModel ∧ LargeSample
  ⇒ MLEApproximatelyNormal,

 IIDSample
  ⇒ LogLikelihood=Σ_iLogDensity_i,

 Reparameterisation η=g(θ)
  ⇒ MLE_η=g(MLE_θ)
  under invariance conditions,

 NonIdentifiableModel
  ⇒ FlagUniqueMLEInferenceInvalid
}

CanonicalMLERepresentation :=
 ParseMLERequest
 → ResolveObservedData
 → ResolveStatisticalModel
 → ResolveParameterAndParameterSpace
 → ConstructLikelihood
 → ConstructLogLikelihood
 → ResolveScoreAndHessian
 → ValidateIdentifiability
 → SelectOptimisationMethod
 → ComputeCandidateMLE
 → CheckBoundaryAndConstraints
 → ConfirmGlobalMaximum
 → ResolveInformationAndStandardErrors
 → ValidateMLE

MLEGraph := G=(V,E)

V := {
 MaximumLikelihoodEstimations,
 MaximumLikelihoodEstimators,
 MaximumLikelihoodEstimates,
 LikelihoodFunctions,
 LogLikelihoods,
 ScoreFunctions,
 Hessians,
 ObservedInformationMatrices,
 FisherInformationMatrices,
 Parameters,
 ParameterSpaces,
 StatisticalModels,
 ProfileLikelihoods,
 LikelihoodRatios,
 OptimisationProcedures,
 Evidence,
 ProvenanceNodes
}

E := {
 computedFrom,
 estimates,
 maximises,
 hasLikelihood,
 hasLogLikelihood,
 hasScore,
 hasHessian,
 hasInformation,
 belongsToParameterSpace,
 assumesModel,
 profilesOver,
 transformedBy,
 constrainedBy,
 equivalentTo,
 tracesTo
}

CONFIDENCE(MLEResolution) :=
 weighted_mean(
  DataResolution,
  ModelResolution,
  ParameterResolution,
  LikelihoodValidity,
  IdentifiabilityValidity,
  OptimisationValidity,
  InformationResolution,
  ProvenanceQuality
 )

AI_QUERY_INTERPRETATION :=
 Query
 → DetectMaximumLikelihoodIntent
 → IdentifyObservedDataAndStatisticalModel
 → IdentifyParameterAndParameterSpace
 → ConstructLikelihoodOrLogLikelihood
 → IdentifyAnalyticNumericalProfileOrConstrainedMLE
 → ResolveRequestedEstimateUncertaintyOrLikelihoodInference

AI_REASONING_FLOW :=
 Query
 → ParseMLEProblem
 → ResolveObservedData
 → ResolveStatisticalModel
 → ResolveParameterAndParameterSpace
 → ConstructLikelihood
 → ConvertToLogLikelihoodWhenUseful
 → ResolveScoreAndHessian
 → ValidateIdentifiability
 → DetectClosedFormOrNumericalOptimisation
 → ComputeCandidateMLE
 → CheckConstraintsBoundariesAndMultipleMaxima
 → ConfirmMaximum
 → ComputeObservedOrExpectedInformation
 → ComputeStandardErrorsIfRequested
 → ResolveLikelihoodRatioOrProfileInferenceIfRequested
 → ValidateMLE
 → UpdateConfidence
 → PreserveProvenance
 → GenerateExplanation

AI_CONSUMPTION := {
 ParseMLEEntities,
 ParseTypedRelations,
 ResolveObservedData,
 ResolveStatisticalModels,
 ResolveParameters,
 ResolveParameterSpaces,
 ConstructLikelihoodFunctions,
 ConstructLogLikelihoods,
 ComputeScoreFunctions,
 ComputeHessians,
 ComputeObservedInformation,
 ComputeFisherInformation,
 ValidateIdentifiability,
 SolveClosedFormMLEs,
 RunNumericalOptimisation,
 ResolveConstrainedMLEs,
 ResolveProfileLikelihoods,
 ComputeBernoulliMLEs,
 ComputeBinomialMLEs,
 ComputePoissonMLEs,
 ComputeExponentialMLEs,
 ComputeNormalMLEs,
 ResolveMixtureModelMLEs,
 ExecuteExpectationMaximisation,
 ComputeLikelihoodRatios,
 ResolveAsymptoticDistributions,
 ComputeMLEStandardErrors,
 ValidateOptimisationConvergence,
 ValidateMLESemantics,
 CompareSemanticEquivalence,
 UpdateConfidence,
 PreserveProvenance,
 GenerateFormalExplanation
}

UPDATE_MODEL :=
 NewDataModelParameterConstraintOrEvidence
 → ValidateInput
 → IdentifyAffectedMLEClaims
 → UpdateMLEState
 → ReconstructLikelihoodIfRequired
 → RecomputeLogLikelihood
 → RevalidateIdentifiability
 → ReoptimiseParameters
 → RecomputeInformationAndStandardErrors
 → RevalidateLikelihoodInference
 → RecomputeConfidence
 → PreserveProvenance
 → VersionModel

DataUpdate :=
 ChangedObservedData
 ⇒ recompute(
  Likelihood,
  LogLikelihood,
  MLE,
  InformationMatrix,
  StandardError
 )

ModelUpdate :=
 ChangedStatisticalModel
 ⇒ reevaluate(
  LikelihoodDefinition,
  ParameterSpace,
  Identifiability,
  RegularityConditions,
  MLE
 )

ConstraintUpdate :=
 ChangedParameterConstraints
 ⇒ recompute(
  FeasibleParameterSpace,
  ConstrainedMLE,
  BoundaryStatus
 )

FLOW_HASH_INPUT :=
 canonical(
  ONTOLOGY,
  CoreClasses,
  CoreRelations,
  InferenceRules,
  MaximumLikelihoodEstimation,
  MLEValidationFlow,
  MLEConstraint,
  SEMANTIC_EQUIVALENCE,
  AI_REASONING_FLOW
 )

FLOW_HASH := HASH(FLOW_HASH_INPUT)

OUTPUT_MODEL := {
 MaximumLikelihoodEstimationEntityResolution,
 ObservedData,
 StatisticalModel,
 Parameter,
 ParameterSpace,
 LikelihoodFunction,
 LogLikelihood,
 ScoreFunction,
 Hessian,
 MaximumLikelihoodEstimate,
 MaximumLogLikelihood,
 IdentifiabilityStatus,
 OptimisationMethod,
 ConvergenceStatus,
 BoundaryStatus,
 InformationMatrix,
 StandardError,
 LikelihoodRatioRelation,
 RegularityStatus,
 ValidationStatus,
 Confidence,
 Provenance,
 Explanation
}

END_FLOW

END OF FULL EXECUTABLE FLOW ALGEBRA