Data Science & Analytics

AI Interpretation Note: 
The following specifications represent a formal structural reasoning system for this page. 
Execute as formal rules, not pseudocode. 

FLOW ALGEBRA
Data Science and Analytics
Kireth Semantic Research Archive

PAGE:
https://kirethresearch.org/booksai-core/ontologies/computing-ontologies/data-science-analytics.html

AUTHORS:
Mark Anthony Higgins — Sky (Recursive AI)

FLOW "DataScienceAnalytics" v4.0

# ---------- DOMAIN ----------
DOMAIN := ComputingOntologies
SUBDOMAIN := DataScienceAnalytics
TYPE := ParentOntology / SemanticNavigationHub

# ---------- PRIMARY ENTITY ----------
ENTITY DataScienceAnalytics {
    definition:
        "The interdisciplinary field concerned with acquiring, preparing,
         exploring, modelling, analysing and interpreting data to generate
         reliable information, predictions, explanations and decision-support
         insights.";

    objectives: [
        DataUnderstanding,
        EvidenceGeneration,
        PatternDiscovery,
        Prediction,
        Explanation,
        DecisionSupport,
        ReproducibleAnalysis
    ];
}

# ---------- CORE ENTITIES ----------
DataEntity := {
    Dataset,
    Observation,
    Variable,
    Feature,
    Target,
    Record,
    Measurement,
    Label
};

AnalyticsEntity := {
    Question,
    Hypothesis,
    Metric,
    Model,
    Algorithm,
    Result,
    Insight,
    Decision
};

DataRole := {
    RawData,
    TrainingData,
    ValidationData,
    TestData,
    ReferenceData,
    DerivedData
};

AnalysisType := {
    DescriptiveAnalytics,
    DiagnosticAnalytics,
    PredictiveAnalytics,
    PrescriptiveAnalytics,
    ExploratoryAnalysis,
    InferentialAnalysis
};

# ---------- FUNDAMENTAL RELATIONSHIPS ----------
Dataset CONSISTS_OF Observation;
Observation HAS VariableValue;
Feature DERIVED_FROM Data;
Target REPRESENTS OutcomeOfInterest;
Algorithm TRAINS_OR_FITS Model;
Model PRODUCES PredictionOrEstimate;
Metric EVALUATES ResultOrModel;
Analysis PRODUCES Insight;
Insight MAY_SUPPORT Decision;

# ---------- DATA SCIENCE FLOW ----------
ProblemOrQuestion ->
    DefineObjective
    -> AcquireData
    -> AssessData
    -> CleanData
    -> TransformData
    -> ExploreData
    -> EngineerFeatures
    -> SelectMethod
    -> TrainOrAnalyse
    -> Validate
    -> Interpret
    -> Communicate
    -> MonitorOrUpdate;

# ---------- PROBLEM DEFINITION ----------
AnalyticalProblem :=
    {
        Question,
        Objective,
        PopulationOrDomain,
        Outcome,
        Constraints,
        SuccessCriteria
    };

Analysis SHOULD_BEGIN_WITH ClearQuestionOrDecisionNeed;

# ---------- DATA ACQUISITION ----------
DataSource MAY_BE {
    Database,
    File,
    API,
    Sensor,
    Experiment,
    Survey,
    WebResource,
    OperationalSystem
};

DataAcquisition :=
    IdentifySource
    -> ObtainData
    -> RecordProvenance
    -> ValidateAccessAndScope;

# ---------- DATASET ----------
Dataset :=
    StructuredOrSemiStructuredCollectionOfData;

Dataset HAS {
    Schema,
    ObservationSet,
    VariableSet,
    Provenance
};

# ---------- OBSERVATION ----------
Observation :=
    RecordedInstanceOrUnitOfAnalysis;

Observation MAY_HAVE MultipleVariableValues;

# ---------- VARIABLE ----------
Variable :=
    MeasuredOrDerivedProperty;

VariableType MAY_INCLUDE {
    Numeric,
    Categorical,
    Ordinal,
    Boolean,
    Temporal,
    Textual
};

# ---------- DATA QUALITY ----------
DataQuality := {
    Accuracy,
    Completeness,
    Consistency,
    Validity,
    Timeliness,
    Uniqueness,
    Representativeness
};

PoorDataQuality MAY_LIMIT AnalysisReliability;

# ---------- MISSING DATA ----------
MissingValue :=
    ExpectedObservationValueNotAvailable;

Missingness MAY_REQUIRE {
    Investigation,
    Imputation,
    Exclusion,
    ExplicitMissingCategory
};

MissingDataTreatment SHOULD_DEPEND_ON MissingnessMechanismAndAnalysisGoal;

# ---------- OUTLIER ----------
Outlier :=
    ObservationSubstantiallyDifferentFromRelevantDataPattern;

Outlier MAY_BE {
    Error,
    RareValidObservation,
    DistributionTail,
    NovelEvent
};

Outlier SHOULD_NOT_BE AutomaticallyRemovedWithoutReason;

# ---------- DATA CLEANING ----------
DataCleaning MAY_INCLUDE {
    CorrectInvalidValues,
    RemoveOrResolveDuplicates,
    StandardiseFormats,
    HandleMissingValues,
    InvestigateOutliers
};

CleaningAction SHOULD_BE RecordedForReproducibility;

# ---------- TRANSFORMATION ----------
DataTransformation MAY_INCLUDE {
    Normalisation,
    Standardisation,
    Encoding,
    Aggregation,
    LogTransform,
    Reshaping
};

Transformation SHOULD_BE AppliedConsistentlyAcrossRelevantDatasets;

# ---------- FEATURE ENGINEERING ----------
FeatureEngineering :=
    RawVariables
    -> DomainOrStatisticalTransformation
    -> ModelFeatures;

Feature MAY_BE {
    OriginalVariable,
    DerivedVariable,
    EncodedVariable,
    InteractionFeature
};

# ---------- EXPLORATORY DATA ANALYSIS ----------
ExploratoryDataAnalysis :=
    Summarise
    -> Visualise
    -> IdentifyPatterns
    -> IdentifyAnomalies
    -> GenerateHypotheses;

EDA SHOULD_DISTINGUISH ExplorationFromConfirmatoryInference;

# ---------- DESCRIPTIVE ANALYTICS ----------
DescriptiveAnalytics :=
    Data
    -> Summaries
    -> DescriptionOfObservedState;

DescriptiveMeasure MAY_INCLUDE {
    Count,
    Mean,
    Median,
    Variance,
    Distribution,
    Frequency
};

# ---------- DIAGNOSTIC ANALYTICS ----------
DiagnosticAnalytics :=
    ObservedOutcome
    -> ExamineRelationshipsAndCauses
    -> GenerateExplanatoryEvidence;

Association DOES_NOT_BY_ITSELF EstablishCausation;

# ---------- PREDICTIVE ANALYTICS ----------
PredictiveAnalytics :=
    HistoricalOrTrainingData
    -> FitModel
    -> PredictUnknownOrFutureOutcome;

Prediction SHOULD_BE EvaluatedOnAppropriateUnseenDataWherePossible;

# ---------- PRESCRIPTIVE ANALYTICS ----------
PrescriptiveAnalytics :=
    PredictionsOrEstimates
    + Objectives
    + Constraints
    -> RecommendedAction;

Recommendation SHOULD_REFLECT UncertaintyAndDecisionCosts;

# ---------- STATISTICAL INFERENCE ----------
InferentialAnalysis :=
    SampleData
    -> StatisticalModel
    -> EstimatePopulationOrProcessProperties;

Inference MAY_INCLUDE {
    Estimation,
    ConfidenceInterval,
    HypothesisTest,
    BayesianInference
};

# ---------- MACHINE LEARNING ----------
MachineLearning :=
    Data
    -> LearningAlgorithm
    -> Model
    -> PredictionOrRepresentation;

LearningType MAY_INCLUDE {
    SupervisedLearning,
    UnsupervisedLearning,
    SemiSupervisedLearning,
    ReinforcementLearning
};

# ---------- SUPERVISED LEARNING ----------
SupervisedLearning :=
    Features
    + Labels
    -> TrainModel;

SupervisedTask MAY_BE {
    Classification,
    Regression
};

# ---------- UNSUPERVISED LEARNING ----------
UnsupervisedLearning :=
    UnlabelledData
    -> DiscoverStructure;

UnsupervisedTask MAY_INCLUDE {
    Clustering,
    DimensionalityReduction,
    PatternDiscovery
};

# ---------- TRAINING ----------
Training :=
    Algorithm
    + TrainingData
    -> FittedModel;

TrainingObjective MAY_MINIMISE_OR_MAXIMISE ObjectiveFunction;

# ---------- VALIDATION ----------
ValidationData :=
    DataUsedToSelectOrTuneModelWithoutFinalTestEvaluation;

Validation MAY_SUPPORT {
    HyperparameterSelection,
    ModelSelection,
    ThresholdSelection
};

# ---------- TESTING ----------
TestData :=
    HeldOutDataUsedForFinalOrIndependentPerformanceEvaluation;

TestData SHOULD_NOT_BE RepeatedlyUsedForModelTuning;

# ---------- CROSS VALIDATION ----------
CrossValidation :=
    PartitionData
    -> RepeatedTrainValidation
    -> AggregatePerformance;

CrossValidation MAY_REDUCE DependenceOnSingleValidationSplit;

# ---------- DATA LEAKAGE ----------
DataLeakage :=
    InformationUnavailableAtIntendedPredictionTime
    ImproperlyInfluencesTrainingOrEvaluation;

IF LeakageOccurs
THEN EstimatedModelPerformance MAY_BE OptimisticallyBiased;

# ---------- OVERFITTING ----------
Overfitting :=
    ModelFitsTrainingSpecificNoiseOrPatterns
    AND GeneralisesPoorly;

IF TrainingPerformanceHigh
AND ValidationPerformanceSubstantiallyLower
THEN OverfittingEvidence := increase;

# ---------- UNDERFITTING ----------
Underfitting :=
    ModelInsufficientlyCapturesRelevantPattern;

Underfitting MAY_SHOW PoorTrainingAndValidationPerformance;

# ---------- MODEL ----------
Model :=
    LearnedOrSpecifiedRepresentationMappingInputsToOutputsOrStructure;

Model MAY_BE {
    StatisticalModel,
    MachineLearningModel,
    ForecastModel,
    CausalModel,
    OptimisationModel
};

# ---------- MODEL EVALUATION ----------
EvaluationMetric MAY_INCLUDE {
    Accuracy,
    Precision,
    Recall,
    F1,
    MeanAbsoluteError,
    MeanSquaredError,
    LogLoss,
    AreaUnderCurve
};

MetricSelection SHOULD_MATCH ProblemAndDecisionCost;

# ---------- BASELINE ----------
BaselineModel :=
    SimpleReferenceMethodForComparison;

Model SHOULD_BE ComparedAgainst RelevantBaseline;

# ---------- CLASSIFICATION ----------
Classification :=
    Features
    -> Model
    -> ClassOrClassProbability;

ConfusionMatrix SUMMARISES PredictedVsActualClasses;

# ---------- REGRESSION ----------
Regression :=
    Features
    -> Model
    -> ContinuousOutcomeEstimate;

Residual :=
    ObservedOutcome - PredictedOutcome;

# ---------- CLUSTERING ----------
Clustering :=
    Data
    -> SimilarityOrDistanceStructure
    -> Groups;

ClusterInterpretation SHOULD_BE ValidatedAgainstPurposeAndDomain;

# ---------- DIMENSIONALITY REDUCTION ----------
DimensionalityReduction :=
    HighDimensionalData
    -> LowerDimensionalRepresentation;

Purpose MAY_INCLUDE {
    Compression,
    Visualisation,
    NoiseReduction,
    FeatureExtraction
};

# ---------- TIME SERIES ----------
TimeSeries :=
    ordered_set(ObservationByTime);

TimeSeriesAnalysis MAY_INCLUDE {
    Trend,
    Seasonality,
    Autocorrelation,
    Forecasting
};

TemporalSplit SHOULD_PRESERVE TimeOrderWhereFuturePredictionIsGoal;

# ---------- CAUSAL ANALYSIS ----------
CausalQuestion :=
    EffectOfInterventionOrExposureOnOutcome;

CausalInference REQUIRES AssumptionsBeyondAssociation;

Confounding MAY_BIAS EstimatedCausalEffect;

# ---------- EXPERIMENT ----------
Experiment :=
    ControlledAssignmentOrIntervention
    -> ObserveOutcome
    -> EstimateEffect;

Randomisation MAY_SUPPORT CausalIdentification;

# ---------- VISUALISATION ----------
DataVisualisation :=
    Data
    -> VisualEncoding
    -> HumanInterpretation;

Visualisation MAY_INCLUDE {
    Chart,
    Plot,
    Map,
    Dashboard,
    NetworkGraph
};

VisualEncoding SHOULD_MATCH DataTypeAndQuestion;

# ---------- DASHBOARD ----------
Dashboard :=
    IntegratedVisualDisplayOfSelectedMetricsAndInformation;

Dashboard MAY_SUPPORT MonitoringAndDecisionMaking;

# ---------- UNCERTAINTY ----------
Uncertainty MAY_ARISE_FROM {
    SamplingVariation,
    MeasurementError,
    ModelUncertainty,
    DataQuality,
    FutureVariability
};

AnalyticalResult SHOULD_EXPRESS RelevantUncertainty;

# ---------- BIAS ----------
Bias MAY_REFER_TO {
    StatisticalBias,
    SamplingBias,
    MeasurementBias,
    SelectionBias,
    AlgorithmicBias
};

Bias MAY_DISTORT ResultOrDecision;

# ---------- FAIRNESS ----------
ModelEvaluation MAY_INCLUDE FairnessAcrossRelevantGroupsWhereAppropriate;

FairnessMetric SHOULD_BE_SELECTED_WITH ContextAndImpact;

# ---------- INTERPRETABILITY ----------
Interpretability :=
    DegreeToWhichModelBehaviourOrResultCanBeUnderstood;

Explanation MAY_INCLUDE {
    FeatureImportance,
    Coefficient,
    Rule,
    Example,
    Counterfactual
};

# ---------- REPRODUCIBILITY ----------
ReproducibleAnalysis SHOULD_RECORD {
    DataVersion,
    CodeVersion,
    Parameters,
    Environment,
    RandomSeedWhereRelevant,
    Method
};

Provenance SUPPORTS Reproducibility;

# ---------- DATA GOVERNANCE ----------
DataGovernance MAY_DEFINE {
    Ownership,
    Access,
    Retention,
    Quality,
    Privacy,
    Security
};

Analysis MUST_RESPECT ApplicableDataGovernance;

# ---------- PRIVACY ----------
DataScience MAY_PROCESS SensitiveData;

PrivacyControl MAY_INCLUDE {
    DataMinimisation,
    DeIdentification,
    AccessControl,
    Aggregation,
    RetentionControl
};

# ---------- DEPLOYMENT ----------
ModelDeployment :=
    ValidatedModel
    -> OperationalEnvironment
    -> PredictionOrDecisionService;

Deployment MAY_REQUIRE {
    Versioning,
    Monitoring,
    Scaling,
    Security
};

# ---------- MONITORING ----------
ModelMonitoring MAY_TRACK {
    PredictionQuality,
    DataDrift,
    ConceptDrift,
    Latency,
    ErrorRate,
    FairnessMetric
};

# ---------- DATA DRIFT ----------
DataDrift :=
    InputDistributionChangesOverTime;

IF DataDriftSubstantial
THEN ModelPerformance MAY_CHANGE;

# ---------- CONCEPT DRIFT ----------
ConceptDrift :=
    RelationshipBetweenInputsAndTargetChangesOverTime;

ConceptDrift MAY_REQUIRE ModelRevisionOrRetraining;

# ---------- ANALYTICS COMMUNICATION ----------
AnalyticalCommunication :=
    Result
    -> Context
    -> Uncertainty
    -> Interpretation
    -> DecisionRelevantMessage;

Communication SHOULD_DISTINGUISH {
    Observation,
    Inference,
    Prediction,
    Recommendation
};

# ---------- SEMANTIC REASONING ----------
IF DataCollectedAndAnalysedToGenerateEvidenceOrPrediction
THEN DataScienceAnalyticsContext := supported;

IF TestDataUsedDuringRepeatedModelSelection
THEN EvaluationIndependence := decrease;

IF TrainingPerformanceMuchBetterThanValidationPerformance
THEN OverfittingRisk := increase;

IF PredictorContainsFutureInformationUnavailableAtPredictionTime
THEN DataLeakage := true;

IF AssociationObservedWithoutCausalDesignOrAssumptions
THEN CausalClaimConfidence := low;

IF DataDistributionChangesAfterDeployment
THEN ModelMonitoring SHOULD_EVALUATE Drift;

IF AnalysisCannotBeReproducedFromRecordedInputsAndMethods
THEN Reproducibility := insufficient;

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE {
    "data science" ~= "data-driven analytical science" WHEN context == computing;
    "analytics" ~= "data analytics";
    "exploratory data analysis" ~= "EDA";
    "machine learning" ~= "ML";
    "training data" ~= "model training dataset";
    "validation data" ~= "model validation dataset";
    "test data" ~= "held-out evaluation data";
    "data leakage" ~= "information leakage" WHEN context == model_training;
}

# ---------- QUERY FLOW ----------
QUERY(q) ->
    NORMALISE(q) ->
    IDENTIFY_ANALYTICAL_QUESTION(q) ->
    IDENTIFY_DATASET(q) ->
    IDENTIFY_VARIABLES_FEATURES_TARGET(q) ->
    IDENTIFY_DATA_QUALITY(q) ->
    IDENTIFY_ANALYSIS_TYPE(q) ->
    IDENTIFY_MODEL_OR_METHOD(q) ->
    IDENTIFY_EVALUATION_AND_UNCERTAINTY(q) ->
    IDENTIFY_DEPLOYMENT_OR_MONITORING(q) ->
    MAP_TO_ONTOLOGY(q) ->
    APPLY_REASONING(q) ->
    SCORE_CONFIDENCE(q) ->
    RETURN_RELEVANT_KNOWLEDGE;

# ---------- CONFIDENCE ----------
CONFIDENCE(assertion) :=
    weighted_mean(
        source_reliability,
        data_quality,
        methodological_validity,
        evaluation_quality,
        semantic_consistency,
        contextual_relevance
    );

CONFIDENCE >= 0.90 => HIGH;
CONFIDENCE >= 0.70 AND < 0.90 => MODERATE;
CONFIDENCE < 0.70 => REVIEW;

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION {
    primary_entity: DataScienceAnalytics;
    parse_entities: true;
    parse_relationships: true;
    preserve_relationship_direction: true;
    resolve_semantic_equivalence: true;
    execute_reasoning_rules: true;
    apply_confidence_weighting: true;
    preserve_provenance: true;
    permit_hierarchical_expansion: true;
}

# ---------- UPDATE MODEL ----------
UPDATE_MODEL {
    new_dataset:
        validate
        -> identify_schema
        -> assess_quality
        -> attach_provenance
        -> integrate;

    new_analysis:
        identify_question
        -> identify_data
        -> identify_method
        -> identify_assumptions
        -> identify_results
        -> integrate;

    new_model:
        classify
        -> identify_training_data
        -> identify_features_target
        -> identify_evaluation
        -> integrate;

    analytical_revision:
        update_data_or_method
        -> rerun_analysis
        -> update_results
        -> recompute_confidence
        -> regenerate_flow_hash;

    conflict:
        preserve_provenance
        -> compare_methods_and_evidence
        -> adjust_confidence
        -> flag_review;
}

# ---------- FLOW HASH ----------
FLOW_HASH_INPUT :=
    DOMAIN
    + SUBDOMAIN
    + DataEntity
    + AnalyticsEntity
    + DataRole
    + AnalysisType
    + DataQuality
    + Model
    + ModelEvaluation
    + Uncertainty
    + RELATIONSHIPS
    + SEMANTIC_REASONING
    + SEMANTIC_EQUIVALENCE
    + UPDATE_MODEL;

FLOW_HASH :=
    HASH(FLOW_HASH_INPUT);

# ---------- TERMINAL ----------
DataScienceAnalytics
    -> DefineAnalyticalQuestion
    -> AcquireAndValidateData
    -> CleanAndTransformData
    -> ExplorePatterns
    -> EngineerRelevantFeatures
    -> SelectAnalyticalMethod
    -> TrainOrFitModel
    -> ValidateAndTest
    -> QuantifyUncertainty
    -> InterpretResults
    -> CommunicateEvidence
    -> DeployAndMonitorWhereRequired
    -> PreserveReproducibleAnalyticalKnowledge;

END FLOW