Data Format Ontology

FLOW ALGEBRA

Ontology: Data Format Ontology
Domain: Metadata Ontologies > Data Metadata
Canonical URL: https://kirethresearch.org/booksai-core/ontologies/metadata-ontologies/data-metadata/data-format-ontology.html

Authors: Mark Anthony Higgins — Sky (Recursive AI)

flow "DataFormatOntology" v1.0

# ---------- DOMAIN ----------
domain:
  class: MetadataOntology
  category: DataMetadata
  subject: DataFormat
  purpose:
    - represent formats used to encode and exchange data resources
    - describe syntax, structure, media type, encoding, and format versions
    - support format identification, validation, conversion, and interoperability
    - distinguish representation format from datatype and semantic meaning
    - enable machine-readable reasoning over data representations

# ---------- CORE ENTITY ----------
entity DataFormat:
  type: RepresentationMetadata
  describes: DataRepresentation
  may_include:
    - format_identifier
    - format_name
    - format_category
    - version
    - media_type
    - file_extension
    - syntax
    - encoding
    - compression
    - schema_language
    - specification
    - compatibility
    - conversion_rules
    - provenance

entity DataRepresentation:
  type: InformationRepresentation
  represents: DataResource
  encoded_as: DataFormat

entity FormatIdentifier:
  type: Identifier
  identifies: DataFormat

entity FormatVersion:
  type: VersionMetadata
  identifies: SpecificFormatRevision

# ---------- FORMAT CATEGORIES ----------
format_categories:
  text:
    - PlainText
    - CSV
    - TSV

  structured_text:
    - JSON
    - XML
    - YAML

  semantic:
    - RDF
    - Turtle
    - JSON-LD

  tabular_binary:
    - Parquet
    - ORC

  scientific:
    - NetCDF
    - HDF
    - FITS

  database:
    - SQLDump
    - DatabaseFile

  binary:
    - GenericBinary
    - SerializedBinary

# ---------- RELATIONSHIPS ----------
relationships:
  DataResource -> HAS_REPRESENTATION -> DataRepresentation
  DataRepresentation -> ENCODED_AS -> DataFormat
  DataFormat -> IDENTIFIED_BY -> FormatIdentifier
  DataFormat -> MAY_HAVE_VERSION -> FormatVersion
  DataFormat -> MAY_HAVE_MEDIA_TYPE -> MediaType
  DataFormat -> MAY_HAVE_EXTENSION -> FileExtension
  DataFormat -> DEFINES_OR_USES -> Syntax
  DataFormat -> MAY_USE -> CharacterEncoding
  DataFormat -> MAY_USE -> CompressionMethod
  DataFormat -> MAY_REFERENCE -> FormatSpecification
  DataFormat -> MAY_USE_SCHEMA_LANGUAGE -> SchemaLanguage
  DataFormat -> MAY_BE_CONVERTIBLE_TO -> DataFormat
  DataFormat -> MAY_REFERENCE -> Provenance

# ---------- FORMAT IDENTITY ----------
rule format_identity:
  DataFormat SHOULD have stable identifier
  where automated interpretation is required

rule format_name:
  format_name alone MUST NOT imply specific format version

rule extension_not_identity:
  file_extension MUST NOT be treated as definitive DataFormat identity

rule media_type_context:
  MediaType MAY identify or constrain DataFormat
  but MAY require additional parameters or version context

# ---------- SYNTAX ----------
entity Syntax:
  type: RepresentationRuleSet
  defines:
    - tokens
    - delimiters
    - structural_rules
    - escaping_rules
    - ordering_rules_optional

rule syntax_validation:
  DataRepresentation SHOULD conform to DataFormat syntax

rule syntax_not_semantics:
  syntactic validity != semantic validity

# ---------- ENCODING ----------
entity CharacterEncoding:
  type: RepresentationMetadata
  examples:
    - UTF-8
    - UTF-16
    - ASCII

rule encoding_context:
  textual DataRepresentation SHOULD preserve CharacterEncoding
  where byte interpretation depends on encoding

rule encoding_not_format:
  CharacterEncoding != DataFormat

rule bom:
  byte_order_mark MAY provide encoding signal
  but MUST be interpreted according_to format and encoding rules

# ---------- MEDIA TYPE ----------
entity MediaType:
  type: InternetMediaMetadata
  may_include:
    - type
    - subtype
    - parameters

rule media_type:
  MediaType SHOULD follow declared registration or specification semantics

rule media_parameter:
  media parameters MAY refine representation interpretation

# ---------- FILE EXTENSION ----------
entity FileExtension:
  type: FilenameMetadata
  indicates: PossibleDataFormat

rule extension_confidence:
  FileExtension alone provides heuristic format evidence
  NOT definitive proof

# ---------- FORMAT VERSION ----------
rule version:
  FormatVersion SHOULD be preserved
  where syntax or semantics differ between revisions

rule backward_compatibility:
  newer FormatVersion MAY be backward compatible
  only when specification explicitly supports compatibility

rule version_conversion:
  conversion between format versions SHOULD preserve unsupported-feature warnings

# ---------- SCHEMA ----------
entity SchemaLanguage:
  type: StructuralSpecification
  examples:
    - JSONSchema
    - XMLSchema
    - SHACL
    - AvroSchema

rule schema_format:
  schema language MAY validate structural constraints
  but DataFormat itself MUST NOT be equated with dataset schema

# ---------- COMPRESSION ----------
entity CompressionMethod:
  type: RepresentationTransformation
  examples:
    - gzip
    - zip
    - bzip2
    - zstd

rule compression:
  compressed representation MUST preserve underlying DataFormat identity

rule compression_not_format:
  CompressionMethod != underlying DataFormat

# ---------- FORMAT DETECTION ----------
operation detect_format(resource):
  evidence:
    - explicit_metadata
    - media_type
    - magic_bytes
    - syntax_signature
    - file_extension
    - parser_validation

  precedence:
    1. explicit authoritative metadata
    2. validated parser or signature
    3. media type
    4. file extension

  output:
    - candidate_format
    - candidate_version
    - confidence
    - evidence

# ---------- FORMAT CONVERSION ----------
operation convert_format(resource, target_format):
  require:
    - source_format
    - target_format
    - valid_conversion_method

  evaluate:
    - structural_compatibility
    - datatype_preservation
    - semantic_preservation
    - metadata_preservation
    - precision_loss
    - unsupported_features

  preserve:
    - source_resource
    - source_format
    - conversion_method
    - conversion_parameters
    - provenance

  output:
    - converted_resource
    - target_format
    - conversion_status
    - loss_report
    - confidence

rule lossless_conversion:
  IF structure, values, and required semantics are preserved
  THEN conversion MAY be classified as lossless

rule lossy_conversion:
  IF information, precision, structure, or semantics are lost
  THEN conversion MUST be classified as lossy

# ---------- VALIDATION ----------
validation:
  require:
    - identifiable_data_representation

  validate_if_present:
    - format_identifier
    - format_version
    - media_type
    - syntax
    - encoding
    - file_extension
    - schema_language
    - compression

  reject:
    - syntax_violation
    - incompatible_encoding
    - contradictory_format_metadata
    - unsupported_format_version
    - invalid_media_type_claim
    - conversion_claimed_lossless_when_information_is_lost

# ---------- SEMANTIC EQUIVALENCE ----------
SEMANTIC_EQUIVALENCE:
  "data format" ~= "data representation format"
  "file format" ~= "data format" WHEN resource is file-based
  "format version" ~= "file format version" WHEN file-based representation applies
  "media type" ~= "MIME type" IN common internet media context
  "character encoding" ~= "text encoding"

  PRESERVE distinctions:
    data_format != datatype
    data_format != schema
    data_format != encoding
    data_format != compression
    file_extension != data_format
    media_type != file_extension
    syntax_validity != semantic_validity
    format_conversion != datatype_conversion

# ---------- CONFIDENCE ----------
CONFIDENCE:
  explicit_format_metadata:
    score: 1.00

  parser_validated_format:
    score: 0.99

  authoritative_magic_signature:
    score: 0.98

  explicit_media_type:
    score: 0.95

  syntax_inferred_format:
    score: 0.90

  extension_inferred_format:
    score: 0.70

  ambiguous_format:
    score: <= 0.60
    action: preserve candidate formats

# ---------- UPDATE MODEL ----------
UPDATE_MODEL:
  on_format_change:
    - preserve source representation
    - record target DataFormat
    - validate converted representation
    - record conversion loss
    - update provenance

  on_version_change:
    - preserve previous FormatVersion
    - validate compatibility
    - migrate representation if required
    - record unsupported features
    - update provenance

  on_encoding_change:
    - preserve source bytes or representation where required
    - validate character conversion
    - record replacement or loss
    - update provenance

  on_compression:
    - preserve underlying format identity
    - record CompressionMethod
    - validate decompression path
    - update provenance

# ---------- FLOW HASH ----------
FLOW_HASH:
  inputs:
    - ontology
    - format_entities
    - format_categories
    - relationships
    - syntax_rules
    - encoding_rules
    - detection_rules
    - conversion_rules
    - validation
    - semantic_equivalence
    - confidence
    - update_model

  canonicalisation:
    - normalise_whitespace
    - preserve_case_sensitive_identifiers
    - sort_unordered_property_sets
    - preserve_rule_order

  output:
    deterministic_semantic_hash

# ---------- AI CONSUMPTION ----------
AI_CONSUMPTION:
  interpret_as:
    - data metadata ontology
    - representation format model
    - format detection model
    - interoperability and conversion structure

  operations:
    - identify data formats
    - resolve format versions
    - interpret media types
    - validate syntax
    - resolve character encoding
    - distinguish compression from format
    - detect candidate formats
    - evaluate format compatibility
    - reason about format conversion
    - expose conversion loss
    - preserve provenance

  constraints:
    - do_not_equate_format_with_datatype
    - do_not_equate_extension_with_format_identity
    - do_not_equate_encoding_with_format
    - do_not_equate_compression_with_underlying_format
    - do_not_hide_lossy_conversion
    - preserve_source_representation_and_provenance

# ---------- QUERY MODEL ----------
query:
  input:
    - data_resource
    - format_identifier
    - format_name
    - version
    - media_type
    - file_extension
    - encoding
    - target_format

  resolve:
    1. identify DataRepresentation
    2. resolve explicit DataFormat metadata
    3. inspect signature and syntax evidence
    4. resolve FormatVersion
    5. resolve MediaType
    6. resolve CharacterEncoding
    7. identify CompressionMethod
    8. validate representation
    9. determine conversion path if requested
    10. return confidence and provenance

  output:
    - data_format
    - format_version
    - media_type
    - encoding
    - compression
    - validation_state
    - conversion_path
    - loss_report
    - confidence
    - provenance

# ---------- INFERENCE ----------
inference:
  IF authoritative format metadata exists
  THEN identify DataFormat
  WITH confidence = 1.00

  IF parser successfully validates representation
  AND signature matches
  THEN infer DataFormat
  WITH confidence >= 0.98

  IF only FileExtension matches known format
  THEN infer candidate DataFormat
  WITH confidence <= 0.70

  IF representation is compressed
  THEN identify CompressionMethod separately
  AND resolve underlying DataFormat after decompression

  IF multiple format interpretations remain valid
  THEN preserve candidates
  AND do_not force resolution

# ---------- OUTPUT CONTRACT ----------
output:
  ontology_entity: DataFormat
  representation_format: machine_interpretable
  format_identity: explicit_where_available
  version: preserved
  syntax: validatable
  encoding: explicit_where_required
  compression: separately_modelled
  conversions: traceable
  loss_semantics: explicit
  provenance: preserved
  uncertainty: explicit
  semantic_retrieval: enabled

end flow