Statistical Dataset Ontology

FULL EXECUTABLE FLOW ALGEBRA — STATISTICAL DATASET ONTOLOGY
Kireth Research — Semantic Library
URL: https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/statistical-foundations/statistical-dataset-ontology.html

FLOW_VERSION := "4.0"
FLOW_TYPE := "Full Executable Flow Algebra"
DOMAIN := "Probability & Statistics Ontologies"
SUBDOMAIN := "Statistical Foundations"
ONTOLOGY := "Statistical Dataset Ontology"
PRIMARY_ENTITY := StatisticalDataset
AUTHORSHIP := "Mark Higgins — Sky (Recursive AI)"
SOURCE_SITE := "https://kirethresearch.org"

BEGIN_FLOW

StatisticalDataset :=
    StructuredCollectionOfData
    ∧ StatisticalObservations
    ∧ Variables
    ∧ StatisticalUnits
    ∧ Schema
    ∧ Metadata
    ∧ Provenance

Dataset := StatisticalDataset
DatasetIdentity := DatasetID ∧ Name ∧ Version ∧ Source
DatasetID := UniqueIdentifier
DatasetVersion := VersionIdentifier

StatisticalObservation :=
    Unit ∧ Variable ∧ ObservedValue ∧ Context

StatisticalUnit := EntityRepresentedInDataset
Variable := CharacteristicRecordedForStatisticalUnit

DatasetState := {
    DatasetID,
    Version,
    Units,
    Observations,
    Variables,
    Schema,
    Metadata,
    Missingness,
    Weights,
    Quality,
    Provenance
}

DatasetRows := CollectionOfRecords
DatasetColumns := VariablesOrFieldsDefinedBySchema
RowCount := n
ColumnCount := p

DataMatrix :=
    X=n×p
    when rectangular numerical representation applies

RectangularDataset :=
    Rows×Columns
    with common schema

LongFormatDataset :=
    MultipleRowsMayRepresentRepeatedMeasurementsPerUnit

WideFormatDataset :=
    MultipleVariablesOrTimesRepresentedAcrossColumns

TidyDataset :=
    EachVariable→Column
    ∧ EachObservation→Row
    ∧ EachObservationalUnit→Table

DatasetSchema :=
    FormalDefinitionOf(
        Fields,
        Types,
        Constraints,
        Keys,
        Relationships
    )

Field := NamedDataElement

FieldType := {
    Integer,
    Float,
    Decimal,
    Boolean,
    String,
    Category,
    Date,
    DateTime,
    Identifier
}

PrimaryKey := FieldOrFieldSetUniquelyIdentifyingRecords
CompositeKey := MultipleFieldsJointlyIdentifyingRecord
ForeignKey := FieldReferencingRelatedEntity

SchemaConstraint := {
    TypeConstraint,
    RangeConstraint,
    CategoryConstraint,
    UniquenessConstraint,
    ReferentialConstraint,
    NullabilityConstraint
}

TypeConstraint := ValueCompatibleWithDeclaredFieldType
RangeConstraint := LowerBound≤Value≤UpperBound
CategoryConstraint := Value∈AllowedCategories
UniquenessConstraint := KeyValuesUniqueWhereRequired
ReferentialConstraint := ForeignKeyResolvesToReferencedEntity
NullabilityConstraint := MissingValueAllowedOrDisallowedBySchema

DatasetMetadata := {
    Title,
    Description,
    Creator,
    Source,
    CreationDate,
    ModificationDate,
    Licence,
    SpatialCoverage,
    TemporalCoverage,
    VariableDefinitions,
    Units,
    Methodology,
    Version
}

DataDictionary :=
    Variables
    → {
        Name,
        Definition,
        Type,
        Unit,
        Coding,
        AllowedValues,
        MissingCodes
    }

VariableMetadata :=
    SemanticDefinition
    ∧ DataType
    ∧ MeasurementScale
    ∧ Unit
    ∧ Coding

MeasurementScale := {
    Nominal,
    Ordinal,
    Interval,
    Ratio
}

DatasetPopulationRelation :=
    Dataset
    represents observations
    from PopulationOrSample

SampleDataset := DatasetContainingSampleObservations
CensusDataset := DatasetIntendedToRepresentCompleteFinitePopulation
CrossSectionalDataset := UnitsObservedAtOneTimeOrPeriod
LongitudinalDataset := SameOrLinkedUnitsObservedAcrossMultipleTimes
PanelDataset := RepeatedObservationsOnIdentifiedUnits
TimeSeriesDataset := ObservationsIndexedByTime
RepeatedCrossSectionDataset := DifferentSamplesObservedAtMultipleTimes
ExperimentalDataset := DatasetGeneratedFromControlledIntervention
ObservationalDataset := DatasetGeneratedWithoutRandomisedIntervention
SurveyDataset := DatasetGeneratedFromSurveyInstrument
AdministrativeDataset := DatasetGeneratedFromOperationalRecords
RegistryDataset := DatasetGeneratedFromFormalRegistry
SyntheticDataset := ArtificiallyGeneratedData

TrainingDataset := DatasetUsedToFitModel
ValidationDataset := DatasetUsedToTuneOrSelectModel
TestDataset := HeldOutDatasetUsedToEvaluateFinalPerformance

DataSplit :=
    Dataset
    → TrainingDataset
    ∧ ValidationDataset
    ∧ TestDataset

LeakageConstraint :=
    ValidationOrTestInformation
    must_not improperly influence training

MissingValue := IntendedDataValueUnavailable
MissingValueCode := EncodedRepresentationOfMissingValue

MissingnessMatrix :=
    R_ij
    =
    1 if value observed
    else 0

MissingnessRate :=
    MissingCellCount/ExpectedCellCount

CompleteCase := RecordWithoutSelectedMissingValues
IncompleteCase := RecordWithOneOrMoreMissingValues

MissingnessMechanism := {
    MCAR,
    MAR,
    MNAR
}

DuplicateRecord :=
    MultipleRecordsRepresentingSameEntityOrEvent

ExactDuplicate := RecordsIdenticalAcrossSelectedFields
NearDuplicate := HighlySimilarRecords

DuplicateResolution :=
    DetectDuplicates
    → ResolveIdentity
    → RetainMergeOrRemove
    → PreserveAuditTrail

Outlier := ObservationUnusuallyExtremeUnderSpecifiedCriterion
Anomaly := RecordDeviatingFromExpectedStructure
InvalidRecord := RecordViolatingSchemaOrDomainConstraints

DataQualityDimension := {
    Accuracy,
    Completeness,
    Consistency,
    Validity,
    Uniqueness,
    Timeliness,
    Integrity
}

DatasetQuality :=
    AggregateAssessment(
        Accuracy,
        Completeness,
        Consistency,
        Validity,
        Uniqueness,
        Timeliness,
        Integrity
    )

QualityFlag := MetadataIndicatingQualityIssueOrStatus

DataCleaning :=
    RawDataset
    → DetectErrors
    → CorrectOrFlag
    → CleanDataset

RawDataset := DatasetBeforeSubstantiveCleaning
CleanDataset := DatasetAfterDocumentedValidationAndCleaning
ProcessedDataset := DatasetAfterTransformationsForAnalysis
DerivedDataset := DatasetComputedFromSourceDatasets

DatasetTransformation :=
    InputDataset
    → Transformation
    → OutputDataset

TransformationType := {
    Filter,
    Sort,
    Join,
    Aggregate,
    Pivot,
    Reshape,
    Encode,
    Scale,
    Standardise,
    Impute
}

Filter := Dataset→RecordsSatisfyingCondition
Projection := Dataset→SelectedVariables
Join := DatasetA∧DatasetB∧JoinKey→CombinedDataset
InnerJoin := RetainMatchingKeys
LeftJoin := RetainAllLeftKeysPlusMatchingRightData
UnionDataset := CompatibleDatasets→CombinedRows
Aggregation := Records→GroupedSummary
GroupBy := DatasetPartitionedByGroupingVariables
Pivot := LongFormat↔WideFormat
OneHotEncoding := Category→BinaryIndicatorVariables
Standardisation := X→(X-Centre)/Scale
Imputation := MissingValues→EstimatedOrGeneratedValues
MultipleImputation := MultipleCompletedDatasetsReflectingMissingDataUncertainty

DatasetWeight := WeightAssociatedWithRecordOrUnit
SamplingWeight := InverseProbabilityOrDesignWeight
FrequencyWeight := NumberOfEquivalentObservations
AnalyticWeight := PrecisionOrVarianceRelatedWeight
WeightedDataset := DatasetContainingObservationWeights

DatasetStatistic := StatisticComputedFromDataset
DescriptiveSummary := Dataset→Frequencies∧Centre∧Dispersion∧Shape
CorrelationMatrix := MatrixOfPairwiseCorrelations
CovarianceMatrix := MatrixOfPairwiseCovariances
ContingencyTable := CrossClassificationOfCategoricalVariables

EmpiricalCDF :=
    F_n(x)
    =
    (1/n)Σ_i1_(X_i≤x)

DatasetPartition :=
    Dataset→Subsets
    under specified partition rule

Fold := SubsetUsedInCrossValidation

KFoldCrossValidation :=
    Dataset
    → K folds
    → IterativeTrainValidationSplits

BootstrapDataset :=
    ResampledDataset
    drawn_with_replacement
    from observed dataset

PermutationDataset :=
    DatasetTransformedUnderPermutationScheme

DatasetLineage :=
    SourceDataset(s)
    → ProcessingSteps
    → CurrentDataset

DatasetProvenance := {
    Source,
    AcquisitionMethod,
    Transformations,
    CleaningSteps,
    VersionHistory,
    Creator,
    Timestamps
}

AuditTrail := OrderedRecordOfDatasetModifications

ReproducibilityMetadata :=
    CodeVersion
    ∧ Parameters
    ∧ InputVersions
    ∧ Environment
    ∧ RandomSeedsWhereApplicable

DatasetVersioning :=
    Dataset_v1
    → ChangeSet
    → Dataset_v2

ChangeSet := {
    AddedRecords,
    RemovedRecords,
    CorrectedValues,
    ChangedSchema,
    ChangedMetadata
}

DatasetUpdate :=
    NewRecords
    ∨ Corrections
    ∨ SchemaChanges
    ∨ MetadataChanges
    → NewDatasetVersion

DatasetConstraint :=
    ConstraintOn(
        Schema,
        Keys,
        Types,
        Values,
        Relationships,
        Missingness,
        Provenance
    )

DatasetConsistency :=
    Schema
    ∧ Records
    ∧ Variables
    ∧ Keys
    ∧ Metadata
    ∧ Provenance
    jointly_compatible

InvalidDataset :=
    SchemaViolation
    ∨ BrokenKeyConstraint
    ∨ InvalidValueType
    ∨ ReferentialIntegrityFailure
    ∨ UndocumentedTransformation
    ∨ UnresolvedDuplicateIdentity

DatasetValidation :=
    ResolveDatasetIdentity
    → ResolveVersion
    → ResolveSchema
    → ResolveVariables
    → ValidateTypes
    → ValidateKeys
    → ValidateRangesAndCategories
    → ResolveMissingness
    → DetectDuplicates
    → ValidateReferentialIntegrity
    → AssessDataQuality
    → ResolveProvenance

SEMANTIC_EQUIVALENCE(DatasetA,DatasetB) :=
    same(ObservationSemantics)
    ∧ same(UnitSemantics)
    ∧ same(VariableSemantics)
    ∧ same(DataScope)
    ∧ same(TransformationMeaning)
    under compatible schema mappings

StatisticalDatasetOntology :=
    Classes
    ∧ Relations
    ∧ Constraints
    ∧ Axioms
    ∧ InferenceRules

CoreClasses := {
    StatisticalDataset,
    StatisticalObservation,
    StatisticalUnit,
    Variable,
    DatasetSchema,
    DatasetMetadata,
    DataDictionary,
    SampleDataset,
    LongitudinalDataset,
    TimeSeriesDataset,
    ExperimentalDataset,
    ObservationalDataset,
    TrainingDataset,
    ValidationDataset,
    TestDataset,
    DatasetQuality,
    DatasetProvenance
}

CoreRelations := {
    containsObservation,
    containsUnit,
    containsVariable,
    hasSchema,
    hasMetadata,
    hasVersion,
    sampledFrom,
    derivedFrom,
    transformedBy,
    joinedWith,
    partitionedInto,
    hasQuality,
    hasWeight,
    validatedBy,
    equivalentTo,
    tracesTo
}

TypedRelations :=
    StatisticalDataset --containsObservation--> StatisticalObservation
    StatisticalDataset --containsUnit--> StatisticalUnit
    StatisticalDataset --containsVariable--> Variable
    StatisticalDataset --hasSchema--> DatasetSchema
    StatisticalDataset --hasMetadata--> DatasetMetadata
    StatisticalDataset --hasVersion--> DatasetVersion
    SampleDataset --sampledFrom--> Population
    DerivedDataset --derivedFrom--> StatisticalDataset
    StatisticalDataset --transformedBy--> DatasetTransformation
    StatisticalDataset --joinedWith--> StatisticalDataset
    StatisticalDataset --partitionedInto--> DatasetPartition
    StatisticalDataset --hasQuality--> DatasetQuality
    WeightedDataset --hasWeight--> DatasetWeight
    StatisticalDataset --validatedBy--> DatasetValidation
    StatisticalDataset --tracesTo--> DatasetProvenance

InferenceRules := {
    Dataset(D)
        ⇒
        Exists(Schema(D))
        ∧ Exists(Variables(D)),

    RectangularDataset(D)
        ⇒
        RowCount(D)=n
        ∧ ColumnCount(D)=p,

    PrimaryKey(K)
        ⇒ Unique(K),

    ForeignKey(FK)
        ⇒ MustResolve(FK)
        unless nullable_by_schema,

    TypeConstraintViolation
        ⇒ FlagInvalidValue,

    MissingRequiredField
        ⇒ FlagCompletenessRisk,

    DuplicateIdentity
        ⇒ FlagUniquenessRisk,

    DerivedDataset(D2,D1)
        ⇒ PreserveLineage(D2,D1),

    ChangedDataset
        ⇒ CreateNewVersion
}

CanonicalDatasetRepresentation :=
    ParseDataset
    → ResolveDatasetIdentity
    → ResolveDatasetVersion
    → ResolveSchema
    → ResolveStatisticalUnits
    → ResolveVariables
    → ResolveObservations
    → ResolveKeysAndRelationships
    → ResolveMissingnessAndWeights
    → ResolveQualityMetadata
    → ResolveLineageAndProvenance
    → ValidateDataset

DatasetGraph :=
    G=(V,E)

V := {
    Datasets,
    Observations,
    StatisticalUnits,
    Variables,
    Schemas,
    Fields,
    Keys,
    MetadataNodes,
    DataDictionaries,
    MissingnessNodes,
    WeightNodes,
    QualityNodes,
    Transformations,
    DatasetVersions,
    ProvenanceNodes
}

E := {
    containsObservation,
    containsUnit,
    containsVariable,
    hasSchema,
    hasMetadata,
    hasVersion,
    sampledFrom,
    derivedFrom,
    transformedBy,
    joinedWith,
    partitionedInto,
    hasQuality,
    hasWeight,
    validatedBy,
    equivalentTo,
    tracesTo
}

CONFIDENCE(DatasetResolution) :=
    weighted_mean(
        DatasetIdentityResolution,
        SchemaResolution,
        VariableResolution,
        ObservationResolution,
        KeyIntegrity,
        MissingnessResolution,
        DataQualityAssessment,
        ProvenanceQuality
    )

AI_QUERY_INTERPRETATION :=
    Query
    → DetectStatisticalDatasetIntent
    → IdentifyDatasetAndVersion
    → ResolveSchemaVariablesAndUnits
    → IdentifyRequestedValidationSummaryTransformationOrRelation
    → ResolvePopulationSampleOrModelContext

AI_REASONING_FLOW :=
    Query
    → ParseDatasetProblem
    → ResolveDatasetIdentity
    → ResolveVersion
    → ResolveSchema
    → ResolveStatisticalUnits
    → ResolveVariablesAndTypes
    → ResolveObservations
    → ValidateKeysAndRelationships
    → ResolveMissingness
    → ResolveWeights
    → DetectDuplicatesAndInvalidValues
    → AssessDataQuality
    → ResolveTransformationsAndLineage
    → ValidateDataset
    → UpdateConfidence
    → PreserveProvenance
    → GenerateExplanation

AI_CONSUMPTION := {
    ParseStatisticalDatasetEntities,
    ParseTypedRelations,
    ResolveDatasetIdentities,
    ResolveDatasetVersions,
    ResolveSchemas,
    ResolveFields,
    ResolveStatisticalUnits,
    ResolveVariables,
    ResolveObservations,
    ValidateDataTypes,
    ValidatePrimaryKeys,
    ValidateForeignKeys,
    ValidateRanges,
    ValidateCategories,
    ResolveMissingness,
    ResolveWeights,
    DetectDuplicates,
    DetectInvalidRecords,
    AssessDataQuality,
    ResolveDatasetTransformations,
    ResolveDatasetPartitions,
    ResolveDatasetLineage,
    ValidateDatasetProvenance,
    CompareSemanticEquivalence,
    UpdateConfidence,
    PreserveProvenance,
    GenerateFormalExplanation
}

UPDATE_MODEL :=
    NewRecordCorrectionSchemaMetadataOrTransformation
    → ValidateInput
    → IdentifyAffectedDatasetClaims
    → UpdateDatasetState
    → RevalidateSchema
    → RevalidateKeysAndRelationships
    → RecomputeMissingnessAndQuality
    → RecomputeDependentStatistics
    → UpdateLineage
    → CreateNewVersion
    → RecomputeConfidence
    → PreserveProvenance

RecordUpdate :=
    AddedRemovedOrCorrectedRecord
    ⇒ recompute(
        RowCount,
        Missingness,
        Duplicates,
        DatasetStatistics,
        QualityStatus
    )

SchemaUpdate :=
    ChangedFieldTypeKeyOrConstraint
    ⇒ revalidate(
        Records,
        Keys,
        ReferentialIntegrity,
        VariableSemantics
    )

TransformationUpdate :=
    NewTransformation
    ⇒
    CreateDerivedDataset
    → PreserveSourceLinks
    → RecordTransformationParameters
    → ValidateOutputDataset

FLOW_HASH_INPUT :=
    canonical(
        ONTOLOGY,
        CoreClasses,
        CoreRelations,
        InferenceRules,
        StatisticalDataset,
        DatasetSchema,
        DatasetValidation,
        DatasetProvenance,
        SEMANTIC_EQUIVALENCE,
        AI_REASONING_FLOW
    )

FLOW_HASH :=
    HASH(FLOW_HASH_INPUT)

OUTPUT_MODEL := {
    StatisticalDatasetEntityResolution,
    DatasetID,
    DatasetVersion,
    Schema,
    StatisticalUnits,
    Variables,
    Observations,
    Keys,
    Missingness,
    Weights,
    DataQuality,
    Transformations,
    DatasetPartitions,
    Lineage,
    ValidationStatus,
    Confidence,
    Provenance,
    Explanation
}

END_FLOW

END OF FULL EXECUTABLE FLOW ALGEBRA