Machine Learning Engineering Ontology

AI SEARCH FLOW ALGEBRA
MACHINE LEARNING ENGINEERING ONTOLOGY
Kireth Semantic Research Archive

AUTHORS
Mark Anthony Higgins — Sky (Recursive AI)

SOURCE
https://kirethresearch.org/booksai-core/ontologies/engineering-ontologies/ai-engineering/machine-learning-engineering-ontology.html

FLOW "MachineLearningEngineeringOntology" v4.0

DOMAIN := ENGINEERING_ONTOLOGIES / AI_ENGINEERING / MACHINE_LEARNING_ENGINEERING
ONTOLOGY := MACHINE_LEARNING_ENGINEERING_ONTOLOGY

MACHINE_LEARNING_ENGINEERING :=
  REQUIREMENTS + DATA + FEATURES + LABELS + DATASET_SPLITS
  + MODEL + ALGORITHM + TRAINING + HYPERPARAMETERS
  + VALIDATION + TESTING + EVALUATION + EXPERIMENT_TRACKING
  + MODEL_SELECTION + DEPLOYMENT + INFERENCE
  + MONITORING + DRIFT + RETRAINING + VERSIONING
  + REPRODUCIBILITY + LIFECYCLE_ASSURANCE

PURPOSE :=
  DEFINE_ML_PROBLEM -> DEFINE_REQUIREMENTS
  -> ACQUIRE_AND_VALIDATE_DATA -> ENGINEER_FEATURES
  -> DEFINE_LABELS -> SPLIT_DATA -> SELECT_ALGORITHM
  -> TRAIN_MODELS -> TUNE_HYPERPARAMETERS -> VALIDATE
  -> TEST -> SELECT_MODEL -> DEPLOY -> MONITOR
  -> DETECT_DRIFT -> RETRAIN_OR_UPDATE -> ASSURE_LIFECYCLE

# CORE ENTITIES
ENTITY MachineLearningEngineering
ENTITY MLSystem
ENTITY MLApplication
ENTITY MLRequirement
ENTITY UseCase
ENTITY Dataset
ENTITY TrainingDataset
ENTITY ValidationDataset
ENTITY TestDataset
ENTITY OperationalDataset
ENTITY Feature
ENTITY FeatureVector
ENTITY Label
ENTITY TargetVariable
ENTITY FeatureTransformation
ENTITY DataPipeline
ENTITY FeaturePipeline
ENTITY Algorithm
ENTITY Model
ENTITY ModelArchitecture
ENTITY Parameter
ENTITY Hyperparameter
ENTITY TrainingProcess
ENTITY TrainingRun
ENTITY ValidationProcess
ENTITY TestProcess
ENTITY Evaluation
ENTITY Metric
ENTITY Experiment
ENTITY ExperimentTracker
ENTITY ModelSelection
ENTITY ModelRegistry
ENTITY ModelArtifact
ENTITY Deployment
ENTITY InferenceService
ENTITY Prediction
ENTITY Monitor
ENTITY Drift
ENTITY DataDrift
ENTITY ConceptDrift
ENTITY PerformanceDrift
ENTITY RetrainingProcess
ENTITY Version
ENTITY Provenance
ENTITY Reproducibility
ENTITY Risk
ENTITY HumanOversight

# SYSTEM / REQUIREMENTS
MachineLearningEngineering -> engineers -> MLSystem
MLSystem -> designed_for -> UseCase
MLSystem -> constrained_by -> MLRequirement
MLSystem -> consumes -> Dataset
MLSystem -> contains -> Model
MLSystem -> produces -> Prediction

ML_ENGINEERING_LIFECYCLE :=
  PROBLEM_DEFINITION -> REQUIREMENTS -> DATA_ENGINEERING
  -> FEATURE_ENGINEERING -> MODEL_DEVELOPMENT -> TRAINING
  -> VALIDATION -> TESTING -> MODEL_SELECTION
  -> PACKAGING -> DEPLOYMENT -> MONITORING
  -> MAINTENANCE -> RETIREMENT

ML_REQUIREMENT_TYPE :=
  TASK_PERFORMANCE | LATENCY | THROUGHPUT | MEMORY
  | ROBUSTNESS | EXPLAINABILITY | SAFETY
  | SECURITY | COST | OPERABILITY

MLRequirement -> has -> ACCEPTANCE_CRITERION
UseCase -> defines -> INPUT + TARGET_OUTPUT + OPERATING_CONTEXT + ERROR_COST

IF UseCase changes
THEN MLRequirementsAndEvaluation -> REASSESS

# DATA / FEATURES / LABELS
DATA_ROLE :=
  TrainingDataset | ValidationDataset | TestDataset | OperationalDataset

DATA_QUALITY :=
  COMPLETENESS + ACCURACY + CONSISTENCY
  + REPRESENTATIVENESS + LABEL_QUALITY + TIMELINESS

DATA_PIPELINE_FLOW :=
  INGEST -> SCHEMA_VALIDATE -> QUALITY_CHECK
  -> CLEAN -> DEDUPLICATE -> TRANSFORM
  -> FEATURE_GENERATE -> LABEL_ALIGN -> SPLIT -> VERSION

Feature -> represents -> MODEL_INPUT_PROPERTY
FeatureVector -> contains -> Feature

FEATURE_ENGINEERING :=
  SELECTION + ENCODING + NORMALISATION
  + SCALING + AGGREGATION + EXTRACTION

Label -> represents -> SUPERVISED_TARGET
TargetVariable -> defines -> PREDICTION_TARGET

IF FeatureDefinition changes
THEN DownstreamModel -> RETRAIN_OR_REEVALUATE

IF LabelNoise increases
THEN AchievableSupervisedPerformance -> MAY_DECREASE

# DATA SPLITS / MODEL / TRAINING
DATA_SPLIT := TRAINING + VALIDATION + TEST

TrainingDataset -> used_by -> TrainingProcess
ValidationDataset -> used_by -> HyperparameterSelection + ModelSelection
TestDataset -> used_by -> FINAL_HELD_OUT_EVALUATION

IF TestDataset used_for_iterative_tuning
THEN TestIndependence -> COMPROMISED

MODEL_CLASS :=
  LINEAR_MODEL | TREE_BASED_MODEL | ENSEMBLE_MODEL
  | SUPPORT_VECTOR_MODEL | NEURAL_NETWORK
  | PROBABILISTIC_MODEL | CLUSTERING_MODEL
  | OTHER_VALIDATED_ML_MODEL

Model -> instantiated_from -> Algorithm
Model -> has -> Parameter + ModelArchitecture + Version

TrainingProcess -> learns -> Parameter -> from -> TrainingDataset

TRAINING_LOOP :=
  SAMPLE_OR_LOAD_BATCH -> COMPUTE_PREDICTION
  -> COMPUTE_OBJECTIVE -> COMPUTE_PARAMETER_UPDATE
  -> UPDATE_MODEL -> RECORD_METRICS -> REPEAT

TrainingRun ->
  records -> DATA_VERSION + FEATURE_VERSION
  + CODE_VERSION + HYPERPARAMETERS
  + RANDOM_SEED_IF_USED + METRICS + MODEL_ARTIFACT

# HYPERPARAMETERS
Hyperparameter -> configures -> Algorithm + TrainingProcess

HYPERPARAMETER_SEARCH :=
  GRID_SEARCH | RANDOM_SEARCH | BAYESIAN_OPTIMISATION
  | SUCCESSIVE_HALVING | OTHER_VALIDATED_SEARCH

HyperparameterTrial -> evaluated_on -> ValidationDataset

# VALIDATION / TESTING / EVALUATION
ValidationProcess -> evaluates -> Model -> using -> ValidationDataset
CrossValidation -> partitions -> DEVELOPMENT_DATA -> into -> MULTIPLE_TRAIN_VALIDATE_FOLDS
CrossValidation -> estimates -> GENERALISATION_WITHIN_AVAILABLE_DATA

TestProcess -> evaluates -> SELECTED_MODEL -> using -> TestDataset

ML_TESTING :=
  DATA_TEST + FEATURE_TEST + MODEL_TEST
  + INFERENCE_TEST + INTEGRATION_TEST
  + ROBUSTNESS_TEST + REGRESSION_TEST

ML_METRIC :=
  ACCURACY | PRECISION | RECALL | F1 | ROC_AUC
  | LOG_LOSS | MAE | MSE | RMSE | R_SQUARED
  | CALIBRATION_ERROR | DOMAIN_SPECIFIC_METRIC

Evaluation -> measures -> Model + MLSystem
Metric -> selected_by -> TASK_TYPE + ERROR_COST + REQUIREMENT

IF TestMetric below AcceptanceCriterion
THEN ReleaseReadiness -> NOT_SATISFIED

# MODEL SELECTION / EXPERIMENTS
ModelSelection -> compares -> CANDIDATE_MODELS

MODEL_SELECTION_CRITERIA :=
  VALIDATION_PERFORMANCE + ROBUSTNESS + LATENCY
  + MEMORY + COMPLEXITY + COST + OPERABILITY

BestValidationMetric alone
  -> does_not_guarantee -> BEST_DEPLOYABLE_MODEL

ExperimentTracker ->
  records -> CONFIGURATION + DATA_VERSION + CODE_VERSION
  + METRICS + ARTIFACTS

Experiment -> supports -> REPRODUCIBILITY + COMPARISON

ModelRegistry -> stores -> ModelArtifact

MODEL_STAGE :=
  DEVELOPMENT | CANDIDATE | APPROVED | PRODUCTION | RETIRED

# DEPLOYMENT / INFERENCE
Deployment -> installs -> ModelArtifact -> into -> RuntimeEnvironment

ML_RELEASE :=
  MODEL_VERSION + FEATURE_PIPELINE_VERSION
  + CODE_VERSION + CONFIGURATION_VERSION
  + EVALUATION_RECORD

Deployment -> must_match -> TRAINING_PREPROCESSING_SEMANTICS

IF TrainingServingTransformation differs
THEN TrainingServingSkewRisk -> INCREASES

InferenceService -> receives -> Input
InferenceService -> applies -> FeaturePipeline
InferenceService -> applies -> Model
InferenceService -> returns -> Prediction

INFERENCE_FLOW :=
  RECEIVE -> VALIDATE -> TRANSFORM_FEATURES
  -> MODEL_PREDICT -> POSTPROCESS -> RETURN

Prediction ->
  linked_to -> MODEL_VERSION + FEATURE_VERSION + TIMESTAMP

# MONITORING / DRIFT
Monitor ->
  observes -> INPUT_DATA + FEATURE_DISTRIBUTION
  + PREDICTION_DISTRIBUTION + PERFORMANCE
  + LATENCY + ERRORS + RESOURCE_USE

ML_MONITORING :=
  COLLECT -> AGGREGATE -> COMPARE_WITH_BASELINE
  -> DETECT_DRIFT -> DETECT_PERFORMANCE_CHANGE
  -> ALERT -> REVIEW

Drift -> represents -> CHANGE_FROM_REFERENCE_BEHAVIOUR
DataDrift -> changes -> INPUT_OR_FEATURE_DISTRIBUTION
ConceptDrift -> changes -> RELATIONSHIP_BETWEEN_INPUT_AND_TARGET
PerformanceDrift -> changes -> MODEL_QUALITY

IF DataDrift detected
THEN ModelPerformance -> REASSESS

IF PerformanceDrift exceeds_limit
THEN RetrainingOrRollback -> CONSIDER

# RETRAINING / ROBUSTNESS / OVERSIGHT
RetrainingProcess -> updates -> Model -> using -> NEW_OR_UPDATED_DATA

RETRAINING_TRIGGER :=
  SCHEDULE | DATA_DRIFT | CONCEPT_DRIFT
  | PERFORMANCE_DRIFT | NEW_LABELS
  | REQUIREMENT_CHANGE | MODEL_IMPROVEMENT

RETRAINING_FLOW :=
  COLLECT_NEW_DATA -> VALIDATE -> VERSION
  -> TRAIN -> VALIDATE -> TEST
  -> COMPARE_WITH_PRODUCTION -> APPROVE_OR_REJECT

RetrainedModel ->
  must_not_replace -> ProductionModel
  -> without -> REQUIRED_APPROVAL_AND_EVALUATION

ROBUSTNESS_TEST :=
  NOISE | MISSING_VALUES | OUTLIERS
  | DISTRIBUTION_SHIFT | EDGE_CASES
  | ADVERSARIAL_INPUT_WHERE_RELEVANT

Robustness -> measures -> PERFORMANCE_STABILITY

Explainability ->
  may_provide -> FEATURE_IMPORTANCE
  + LOCAL_EXPLANATION + GLOBAL_MODEL_SUMMARY
  + EXAMPLE_BASED_EXPLANATION

Explanation ->
  must_not_be_treated_as -> CAUSAL_PROOF_UNLESS_SUPPORTED

HumanOversight ->
  may_review -> PREDICTION + MODEL_CHANGE + DEPLOYMENT_DECISION

# RISK / REPRODUCIBILITY
ML_RISK :=
  DATA_RISK + MODEL_RISK + DEPLOYMENT_RISK
  + DRIFT_RISK + INTEGRATION_RISK + SECURITY_RISK

IF ResidualRisk exceeds_acceptance
THEN Release -> WITHHOLD_OR_ESCALATE

Reproducibility ->
  requires -> DATA_VERSION + CODE_VERSION
  + FEATURE_VERSION + HYPERPARAMETERS
  + RANDOM_STATE_WHERE_APPLICABLE
  + ENVIRONMENT_VERSION

IF RequiredVersionInformation missing
THEN ReproducibilityConfidence -> DECREASES

# PROVENANCE / PERFORMANCE
Provenance ->
  records -> DATA_SOURCE + DATA_VERSION
  + FEATURE_TRANSFORMATIONS + TRAINING_RUN
  + MODEL_VERSION + EVALUATION
  + DEPLOYMENT + TIMESTAMP

MLSystem -> linked_to -> Provenance

ML_ENGINEERING_PERFORMANCE :=
  MODEL_QUALITY + ROBUSTNESS + LATENCY
  + THROUGHPUT + AVAILABILITY + RESOURCE_USE
  + REPRODUCIBILITY + MAINTAINABILITY

MLSystem -> evaluated_by -> ML_ENGINEERING_PERFORMANCE

# REASONING
MACHINE_LEARNING_ENGINEERING_REASONING :=
  DEFINE_ML_USE_CASE
  -> DEFINE_REQUIREMENTS
  -> IDENTIFY_DATA_SOURCES
  -> VALIDATE_DATA
  -> DEFINE_FEATURES
  -> DEFINE_TARGET
  -> CREATE_DATA_SPLITS
  -> SELECT_ALGORITHMS
  -> TRAIN_CANDIDATE_MODELS
  -> TUNE_HYPERPARAMETERS
  -> VALIDATE
  -> TEST_SELECTED_MODEL
  -> ASSESS_ROBUSTNESS
  -> VERIFY_REQUIREMENTS
  -> REGISTER_MODEL
  -> PACKAGE_PIPELINE
  -> DEPLOY
  -> MONITOR_DATA_AND_PERFORMANCE
  -> DETECT_DRIFT
  -> RETRAIN_OR_ROLLBACK_IF_REQUIRED
  -> PRESERVE_PROVENANCE

# INFERENCE RULES
IF TrainingData changes
THEN ModelTraining -> REPEAT

IF FeaturePipeline changes
THEN ModelCompatibility -> REASSESS

IF TestPerformance below_requirement
THEN ProductionRelease -> WITHHOLD

IF DataDrift increases
THEN PerformanceRisk -> MAY_INCREASE

IF ConceptDrift occurs
THEN HistoricalModelValidity -> MAY_DECREASE

IF ProductionModel changes
THEN MonitoringBaseline -> UPDATE

IF Provenance incomplete
THEN ReproducibilityConfidence -> DECREASES

# SEMANTIC RELATIONSHIPS
RELATION engineers(MachineLearningEngineering, MLSystem)
RELATION derives(FeatureTransformation, Feature)
RELATION represents(Label, TargetVariable)
RELATION learns(TrainingProcess, Model)
RELATION configures(Hyperparameter, TrainingProcess)
RELATION evaluates(ValidationProcess, Model)
RELATION evaluates(TestProcess, SelectedModel)
RELATION records(ExperimentTracker, Experiment)
RELATION stores(ModelRegistry, ModelArtifact)
RELATION installs(Deployment, ModelArtifact)
RELATION applies(InferenceService, Model)
RELATION observes(Monitor, MLSystem)
RELATION triggers(Drift, ModelReview)
RELATION updates(RetrainingProcess, Model)
RELATION records(Provenance, MLSystem)

# SEMANTIC EQUIVALENCE
SEMANTIC_EQUIVALENCE:
  "machine learning engineering" ~= "ML engineering"
  "ML system" ~= "machine learning system"
  "feature engineering" ~= "ML feature construction"
  "training dataset" ~= "model training data"
  "validation dataset" ~= "model tuning data"
  "test dataset" ~= "held-out model evaluation data"
  "model registry" ~= "ML model artifact registry"
  "data drift" ~= "input distribution drift"
  "concept drift" ~= "input-target relationship drift"
  "retraining" ~= "machine learning model retraining"

# CONFIDENCE
CONFIDENCE:
  ontology_domain: 0.99
  requirements_mapping: 0.99
  data_feature_mapping: 0.99
  training_mapping: 0.99
  validation_testing_mapping: 0.99
  evaluation_selection_mapping: 0.99
  deployment_inference_mapping: 0.99
  monitoring_drift_mapping: 0.99
  retraining_versioning_mapping: 0.99
  semantic_relationships: 0.99

# UPDATE MODEL
UPDATE_MODEL:
  ON new_ml_project:
    MAP use_case
    MAP requirements
    MAP datasets
    MAP features
    MAP target
    MAP algorithms
    UPDATE confidence
    RECOMPUTE FLOW_HASH

  ON new_training_run:
    RECORD data_version
    RECORD feature_version
    RECORD code_version
    RECORD hyperparameters
    RECORD metrics
    RECORD model_artifact
    PRESERVE provenance

  ON production_drift:
    UPDATE drift_state
    UPDATE performance_state
    REASSESS retraining_requirement
    REASSESS rollback_requirement

  ON model_release:
    VERIFY test_results
    VERIFY robustness
    VERIFY deployment_contract
    UPDATE model_registry
    UPDATE monitoring_baseline

# AI CONSUMPTION
AI_CONSUMPTION:
  parse_as: FORMAL_MACHINE_LEARNING_ENGINEERING_ONTOLOGY
  reasoning_mode: DATA_MODEL_LIFECYCLE_RELATIONAL

  preserve:
    - typed_entities
    - requirements
    - dataset_versions
    - feature_versions
    - label_definitions
    - model_versions
    - hyperparameters
    - experiment_history
    - evaluation_records
    - deployment_state
    - monitoring_state
    - confidence
    - provenance

  infer:
    - data_model_dependencies
    - feature_model_dependencies
    - split_integrity
    - evaluation_requirements
    - deployment_readiness
    - drift_impacts
    - retraining_requirements
    - reproducibility_state

  prohibit:
    - invented_training_metrics
    - assumed_test_independence
    - assumed_model_quality
    - fabricated_drift
    - untested_model_presented_as_production_ready
    - missing_provenance_presented_as reproducible

# FLOW HASH
FLOW_HASH_INPUT :=
  DOMAIN + ONTOLOGY + CORE_ENTITIES
  + SYSTEM_REQUIREMENTS + DATA_FEATURES_LABELS
  + DATA_SPLITS_MODEL_TRAINING + HYPERPARAMETERS
  + VALIDATION_TESTING_EVALUATION
  + MODEL_SELECTION_EXPERIMENTS
  + DEPLOYMENT_INFERENCE + MONITORING_DRIFT
  + RETRAINING_ROBUSTNESS_OVERSIGHT
  + RISK_REPRODUCIBILITY + PROVENANCE_PERFORMANCE
  + REASONING + INFERENCE_RULES
  + SEMANTIC_RELATIONSHIPS + SEMANTIC_EQUIVALENCE
  + CONFIDENCE + UPDATE_MODEL + AI_CONSUMPTION

FLOW_HASH := HASH(FLOW_HASH_INPUT)

END FLOW