Testing Ontology

AI SEARCH

FLOW ALGEBRA — TESTING ONTOLOGY
Semantic Library
Version: 4.0
Canonical Page:
https://kirethresearch.org/booksai-core/entity-graph/computational-entities/software-entities/testing-ontology.html

FLOW TESTING_ONTOLOGY {

  META {
    DOMAIN = "Entity Graph"
    SUBDOMAIN = "Computational Entities / Software Entities"
    ONTOLOGY = "Testing Ontology"
    PURPOSE = "Formalise software testing entities, test levels, test cases, inputs, expected and observed results, coverage, defects, environments, automation, verification, validation and testing relationships."
    LOCALE = "en-GB"
  }

  ENTITY_TYPES {
    SOFTWARE_TESTING
    TEST
    TEST_ACTIVITY
    TEST_LEVEL
    UNIT_TEST
    COMPONENT_TEST
    INTEGRATION_TEST
    SYSTEM_TEST
    ACCEPTANCE_TEST
    REGRESSION_TEST
    SMOKE_TEST
    SANITY_TEST
    PERFORMANCE_TEST
    LOAD_TEST
    STRESS_TEST
    SECURITY_TEST
    USABILITY_TEST
    COMPATIBILITY_TEST
    RELIABILITY_TEST
    TEST_CASE
    TEST_SUITE
    TEST_SCENARIO
    TEST_PLAN
    TEST_STRATEGY
    TEST_PROCEDURE
    TEST_STEP
    TEST_INPUT
    TEST_DATA
    TEST_FIXTURE
    PRECONDITION
    POSTCONDITION
    EXPECTED_RESULT
    OBSERVED_RESULT
    TEST_ORACLE
    ASSERTION
    TEST_RESULT
    PASS
    FAIL
    SKIPPED
    BLOCKED
    INCONCLUSIVE
    REQUIREMENT
    ACCEPTANCE_CRITERION
    SOFTWARE_COMPONENT
    SOFTWARE_SYSTEM
    SOURCE_CODE
    FUNCTION
    CLASS
    API
    INTERFACE
    BUILD
    VERSION
    RELEASE
    TEST_ENVIRONMENT
    CONFIGURATION
    DEPENDENCY
    MOCK
    STUB
    FAKE
    TEST_DOUBLE
    AUTOMATED_TEST
    MANUAL_TEST
    TEST_RUNNER
    TEST_FRAMEWORK
    PIPELINE
    COVERAGE
    CODE_COVERAGE
    STATEMENT_COVERAGE
    BRANCH_COVERAGE
    FUNCTION_COVERAGE
    REQUIREMENT_COVERAGE
    DEFECT
    BUG
    FAILURE
    ERROR
    SEVERITY
    PRIORITY
    REPRODUCTION
    TRACE
    LOG
    METRIC
    PERFORMANCE_METRIC
    TEST_REPORT
    EVIDENCE
    CONFIDENCE_SCORE
    PROVENANCE_RECORD
  }

  RELATIONSHIPS {
    IS_A
    INSTANCE_OF
    PART_OF
    CONTAINS
    TESTS
    TARGETS
    VERIFIES
    VALIDATES
    COVERS
    TRACES_TO
    HAS_TEST_CASE
    HAS_TEST_STEP
    HAS_INPUT
    HAS_TEST_DATA
    HAS_FIXTURE
    HAS_PRECONDITION
    HAS_POSTCONDITION
    HAS_EXPECTED_RESULT
    HAS_OBSERVED_RESULT
    USES_ORACLE
    ASSERTS
    PRODUCES
    HAS_RESULT
    PASSES
    FAILS
    SKIPS
    BLOCKS
    EXECUTES_ON
    USES
    DEPENDS_ON
    MOCKS
    STUBS
    AUTOMATES
    DETECTS
    REVEALS
    REPRODUCES
    ASSOCIATED_WITH
    HAS_SEVERITY
    HAS_PRIORITY
    HAS_COVERAGE
    HAS_METRIC
    HAS_VERSION
    HAS_CONFIGURATION
    PRECEDES
    FOLLOWS
    DERIVED_FROM
    INFERRED_FROM
    HAS_EVIDENCE
    HAS_CONFIDENCE
    HAS_PROVENANCE
    SEMANTICALLY_EQUIVALENT_TO
  }

  INPUTS {
    TEST_PLANS
    TEST_STRATEGIES
    REQUIREMENTS
    ACCEPTANCE_CRITERIA
    TEST_CASE_DEFINITIONS
    TEST_SUITE_DEFINITIONS
    TEST_DATA
    TEST_FIXTURES
    SOURCE_CODE
    BUILD_METADATA
    VERSION_DATA
    RELEASE_DATA
    TEST_ENVIRONMENT_DATA
    CONFIGURATION_DATA
    DEPENDENCY_DATA
    AUTOMATION_SCRIPTS
    TEST_RUNNER_OUTPUT
    TEST_RESULTS
    COVERAGE_REPORTS
    PERFORMANCE_RESULTS
    DEFECT_RECORDS
    FAILURE_LOGS
    TRACE_DATA
    TEST_REPORTS
    ONTOLOGY_TERMS
  }

  NORMALISE {
    STANDARDISE_TEST_IDENTIFIERS
    NORMALISE_TEST_LEVELS
    NORMALISE_TEST_TYPES
    NORMALISE_TEST_CASE_IDENTIFIERS
    NORMALISE_TEST_SUITE_IDENTIFIERS
    NORMALISE_REQUIREMENT_IDENTIFIERS
    NORMALISE_BUILD_VERSION_AND_RELEASE_IDENTIFIERS
    NORMALISE_ENVIRONMENT_NAMES
    NORMALISE_RESULT_STATUSES
    NORMALISE_COVERAGE_METRICS
    NORMALISE_PERFORMANCE_METRICS
    NORMALISE_DEFECT_IDENTIFIERS
    NORMALISE_SEVERITY_AND_PRIORITY
    NORMALISE_TIMESTAMPS
    RESOLVE_TEST_TARGET_AND_RESULT_IDENTITIES
    ALIGN_ONTOLOGY_TERMS
    REMOVE_DUPLICATES
    VALIDATE_METADATA
    PRESERVE_PROVENANCE
  }

  PROCESS TESTING_PIPELINE {
    TEST_OBJECTIVE
      -> TEST_DESIGN
      -> TEST_CASE_AND_DATA_PREPARATION
      -> ENVIRONMENT_SETUP
      -> TEST_EXECUTION
      -> OBSERVATION
      -> ORACLE_COMPARISON
      -> RESULT_CLASSIFICATION
      -> DEFECT_OR_EVIDENCE_LINKING
      -> REPORTING
  }

  PROCESS TEST_DESIGN_REASONING {
    REQUIREMENT_OR_RISK_OR_COMPONENT
      -> TEST_CONDITION

    TEST_CONDITION
      -> TEST_SCENARIO

    TEST_SCENARIO
      -> TEST_CASES

    TEST_CASE
      -> INPUTS_PRECONDITIONS_STEPS_EXPECTED_RESULTS
  }

  PROCESS EXECUTION_REASONING {
    TEST_CASE + TEST_ENVIRONMENT
      -> PRECONDITION_CHECK

    VALID_PRECONDITIONS
      -> TEST_STEPS

    TEST_STEPS
      -> SYSTEM_UNDER_TEST

    SYSTEM_UNDER_TEST
      -> OBSERVED_RESULT

    OBSERVED_RESULT
      -> ORACLE_COMPARISON
  }

  PROCESS RESULT_REASONING {
    EXPECTED_RESULT + OBSERVED_RESULT
      -> COMPARISON

    MATCH
      -> PASS

    MISMATCH
      -> FAIL

    TEST_NOT_EXECUTED_BY_POLICY
      -> SKIPPED

    EXTERNAL_DEPENDENCY_PREVENTS_EXECUTION
      -> BLOCKED

    INSUFFICIENT_EVIDENCE
      -> INCONCLUSIVE
  }

  PROCESS DEFECT_REASONING {
    FAILED_TEST
      -> FAILURE_EVIDENCE

    FAILURE_EVIDENCE
      -> REPRODUCTION_ATTEMPT

    REPRODUCIBLE_FAILURE
      -> DEFECT_CANDIDATE

    DEFECT_CANDIDATE
      -> DIAGNOSTIC_AND_ROOT_CAUSE_ANALYSIS

    CONFIRMED_DEFECT
      -> ISSUE_OR_BUG_RECORD
  }

  PROCESS REGRESSION_REASONING {
    SOFTWARE_CHANGE
      -> CHANGE_IMPACT

    CHANGE_IMPACT
      -> AFFECTED_TEST_SET

    AFFECTED_TEST_SET
      -> REGRESSION_TEST_EXECUTION

    REGRESSION_RESULTS
      -> CHANGE_VALIDATION
  }

  PROCESS COVERAGE_REASONING {
    TEST_EXECUTION
      -> EXECUTED_PROGRAM_ELEMENTS

    EXECUTED_PROGRAM_ELEMENTS
      -> STATEMENT_BRANCH_FUNCTION_OR_OTHER_COVERAGE

    TEST_TO_REQUIREMENT_TRACEABILITY
      -> REQUIREMENT_COVERAGE

    COVERAGE_METRICS
      -> COVERAGE_PROFILE
  }

  PROCESS PERFORMANCE_TEST_REASONING {
    WORKLOAD_PROFILE + TEST_ENVIRONMENT
      -> PERFORMANCE_TEST_EXECUTION

    PERFORMANCE_TEST_EXECUTION
      -> LATENCY
      -> THROUGHPUT
      -> RESOURCE_USAGE
      -> ERROR_RATE

    OBSERVED_METRICS + ACCEPTANCE_THRESHOLDS
      -> PERFORMANCE_RESULT
  }

  PROCESS AUTOMATION_REASONING {
    AUTOMATED_TEST
      -> TEST_SCRIPT_OR_FRAMEWORK

    TEST_SCRIPT_OR_FRAMEWORK
      -> TEST_RUNNER

    TEST_RUNNER
      -> EXECUTION_ENVIRONMENT

    EXECUTION
      -> MACHINE_READABLE_RESULTS

    MACHINE_READABLE_RESULTS
      -> TEST_REPORT
  }

  PROCESS TRACEABILITY_REASONING {
    REQUIREMENT
      -> TEST_CASE

    TEST_CASE
      -> TEST_EXECUTION

    TEST_EXECUTION
      -> TEST_RESULT

    TEST_RESULT
      -> DEFECT_IF_APPLICABLE

    TRACEABILITY_CHAIN
      -> VERIFICATION_AND_COVERAGE_EVIDENCE
  }

  RULES {
    IF TEST_RESULT_IS_RECORDED
    THEN PRESERVE TEST_CASE_BUILD_VERSION_ENVIRONMENT_CONFIGURATION_AND_TIME_CONTEXT

    IF TEST_PASSES
    THEN DO_NOT_ASSERT ABSENCE_OF_ALL_DEFECTS

    IF TEST_FAILS
    THEN DO_NOT AUTOMATICALLY ASSERT SOFTWARE_DEFECT_WITHOUT TEST_ORACLE_ENVIRONMENT_AND_REPRODUCTION_CONTEXT

    IF COVERAGE_IS_REPORTED
    THEN PRESERVE COVERAGE_TYPE_SCOPE_AND_DENOMINATOR

    IF PERFORMANCE_RESULT_IS_REPORTED
    THEN PRESERVE WORKLOAD_HARDWARE_ENVIRONMENT_VERSION_AND_CONFIGURATION_CONTEXT

    IF TEST_IS_BLOCKED
    THEN DISTINGUISH BLOCKED_EXECUTION_FROM_FAILED_ASSERTION

    IF TEST_DOUBLE_IS_USED
    THEN PRESERVE REPLACED_DEPENDENCY_AND_TEST_DOUBLE_TYPE

    IF SEMANTIC_IDENTIFIERS_DIFFER
    AND TESTING_REFERENT_IS_SAME
    THEN APPLY SEMANTIC_EQUIVALENCE
  }

  CONFIDENCE {
    CONFIDENCE_SCORE =
      WEIGHT(
        TEST_CASE_SPECIFICATION_QUALITY,
        TEST_ORACLE_CERTAINTY,
        ENVIRONMENT_REPRODUCIBILITY,
        BUILD_AND_VERSION_CERTAINTY,
        EXECUTION_TRACE_COMPLETENESS,
        TEST_DATA_QUALITY,
        DEFECT_REPRODUCIBILITY,
        PROVENANCE_COMPLETENESS
      )

    LEVELS {
      HIGH
      MODERATE
      LOW
      INSUFFICIENT
    }

    IF TEST_ORACLE_IS_AMBIGUOUS
    THEN REDUCE_RESULT_INTERPRETATION_CONFIDENCE

    IF ENVIRONMENT_OR_BUILD_IS_UNKNOWN
    THEN REDUCE_REPRODUCIBILITY_CONFIDENCE

    IF REPEATED_INDEPENDENT_TEST_RUNS_CONFIRM_RESULT
    THEN INCREASE_CONFIDENCE_SCORE
  }

  SEMANTIC_EQUIVALENCE {
    EQUIVALENCE_KEY =
      NORMALISE(
        ENTITY_TYPE,
        TEST_IDENTIFIER,
        TEST_TARGET,
        BUILD_OR_VERSION,
        ENVIRONMENT,
        CONFIGURATION,
        TEST_DATA_CONTEXT
      )

    IF EQUIVALENCE_KEY(A) == EQUIVALENCE_KEY(B)
    THEN A SEMANTICALLY_EQUIVALENT_TO B

    ELSE PRESERVE_DISTINCT_ENTITIES
  }

  REASONING_CONSTRAINTS {
    TEST_CASE != TEST_EXECUTION
    TEST_EXECUTION != TEST_RESULT
    EXPECTED_RESULT != OBSERVED_RESULT
    TEST_PASS != PROOF_OF_CORRECTNESS
    TEST_FAIL != CONFIRMED_SOFTWARE_DEFECT
    BLOCKED != FAILED
    SKIPPED != PASSED
    CODE_COVERAGE != REQUIREMENT_COVERAGE
    HIGH_CODE_COVERAGE != HIGH_TEST_EFFECTIVENESS
    UNIT_TEST != INTEGRATION_TEST
    VERIFICATION != VALIDATION
    TEST_DOUBLE != REAL_DEPENDENCY
    PERFORMANCE_RESULT_IS_CONTEXT_DEPENDENT
    TEST_RESULTS_ARE_BUILD_VERSION_ENVIRONMENT_AND_DATA_DEPENDENT
    DEFECT_SEVERITY != DEFECT_PRIORITY
    PROVENANCE_MUST_BE_PRESERVED
    UNCERTAINTY_MUST_NOT_BE_DISCARDED
  }

  VALIDATION {
    CHECK_ENTITY_TYPES
    CHECK_RELATION_DOMAINS_AND_RANGES
    CHECK_TEST_IDENTIFIER
    CHECK_TEST_LEVEL_AND_TYPE
    CHECK_TEST_TARGET
    CHECK_TEST_CASE_DEFINITION
    CHECK_INPUTS_AND_TEST_DATA
    CHECK_PRECONDITIONS_AND_POSTCONDITIONS
    CHECK_EXPECTED_RESULT
    CHECK_TEST_ORACLE
    CHECK_OBSERVED_RESULT
    CHECK_RESULT_STATUS
    CHECK_BUILD_VERSION_AND_RELEASE
    CHECK_ENVIRONMENT_AND_CONFIGURATION
    CHECK_TEST_DOUBLE_RELATIONS
    CHECK_COVERAGE
    CHECK_PERFORMANCE_CONTEXT
    CHECK_DEFECT_RELATIONS
    CHECK_TRACEABILITY
    CHECK_PROVENANCE
    CHECK_CONFIDENCE
    CHECK_SEMANTIC_CONSISTENCY
  }

  OUTPUTS {
    NORMALISED_TESTING_ENTITIES
    TEST_PLAN_AND_STRATEGY_RELATIONS
    TEST_CASE_RELATIONS
    TEST_SUITE_RELATIONS
    TEST_INPUT_AND_DATA_RELATIONS
    TEST_ORACLE_RELATIONS
    TEST_EXECUTION_RELATIONS
    TEST_RESULT_RELATIONS
    DEFECT_AND_FAILURE_RELATIONS
    REGRESSION_RELATIONS
    COVERAGE_RELATIONS
    PERFORMANCE_TEST_RELATIONS
    TEST_AUTOMATION_RELATIONS
    REQUIREMENT_TRACEABILITY_RELATIONS
    TEST_REPORT_RELATIONS
    CONFIDENCE_SCORES
    EVIDENCE_LINKS
    PROVENANCE_RECORDS
    EXPLANATION_PATHS
  }

  UPDATE_MODEL {
    ON TEST_DEFINITION_OR_SOFTWARE_CHANGE {
      NORMALISE
      RESOLVE_TEST_AND_TARGET_IDENTITIES
      UPDATE_TEST_CASES
      UPDATE_TEST_SUITES
      UPDATE_TRACEABILITY
      UPDATE_REGRESSION_SCOPE
      RECOMPUTE_COVERAGE_RELATIONS
      UPDATE_CONFIDENCE
      PRESERVE_VERSION_HISTORY
    }

    ON NEW_TEST_EXECUTION_OR_DEFECT_DATA {
      LINK_EVIDENCE
      UPDATE_TEST_RESULTS
      UPDATE_COVERAGE
      UPDATE_PERFORMANCE_METRICS
      REASSESS_DEFECT_RELATIONS
      UPDATE_REPORTS
      UPDATE_CONFIDENCE
      RECORD_PROVENANCE
    }

    ON ONTOLOGY_REVISION {
      REMAP_TERMS
      CHECK_SEMANTIC_EQUIVALENCE
      IDENTIFY_DEPRECATED_TERMS
      MIGRATE_RELATIONSHIPS
      PRESERVE_PRIOR_VERSION_REFERENCES
    }
  }

  FLOW_HASH {
    HASH_INPUT =
      CANONICALISE(
        ENTITY_TYPES,
        RELATIONSHIPS,
        RULES,
        CONFIDENCE,
        SEMANTIC_EQUIVALENCE,
        REASONING_CONSTRAINTS,
        UPDATE_MODEL
      )

    FLOW_HASH = SHA256(HASH_INPUT)
  }

  AI_CONSUMPTION {
    INDEX {
      ONTOLOGY
      ENTITY_TYPES
      RELATIONSHIPS
      RULES
      OUTPUTS
    }

    RETRIEVE {
      BY_TEST
      BY_TEST_TYPE
      BY_TEST_LEVEL
      BY_TEST_CASE
      BY_TEST_SUITE
      BY_REQUIREMENT
      BY_BUILD
      BY_VERSION
      BY_ENVIRONMENT
      BY_RESULT
      BY_COVERAGE
      BY_DEFECT
      BY_PERFORMANCE_METRIC
      BY_CONFIDENCE
    }

    REASON {
      APPLY_RELATIONSHIPS
      APPLY_RULES
      ANALYSE_TEST_DESIGN
      TRACE_TEST_EXECUTION
      CLASSIFY_TEST_RESULTS
      ANALYSE_DEFECT_EVIDENCE
      ANALYSE_REGRESSION_SCOPE
      ANALYSE_COVERAGE
      ANALYSE_PERFORMANCE_TESTS
      TRACE_TEST_AUTOMATION
      BUILD_REQUIREMENT_TEST_TRACEABILITY
      CHECK_SEMANTIC_EQUIVALENCE
      PRESERVE_UNCERTAINTY
      TRACE_EVIDENCE
      TRACE_PROVENANCE
      GENERATE_EXPLANATION_PATH
    }

    RESPONSE_POLICY {
      DISTINGUISH_TEST_CASE_EXECUTION_AND_RESULT
      DISTINGUISH_TEST_FAILURE_FROM_CONFIRMED_DEFECT
      PRESERVE_BUILD_VERSION_ENVIRONMENT_CONFIGURATION_TEST_DATA_AND_TIME_CONTEXT
      DISTINGUISH_CODE_COVERAGE_FROM_REQUIREMENT_COVERAGE
      DISTINGUISH_VERIFICATION_FROM_VALIDATION
      RETURN_CONFIDENCE_WHEN_AVAILABLE
      RETURN_PROVENANCE_WHEN_AVAILABLE
      AVOID_UNSUPPORTED_TESTING_INFERENCE
    }
  }

  TERMINAL {
    RESULT =
      VALIDATED(
        SEMANTICALLY_NORMALISED(
          EVIDENCE_LINKED(
            CONFIDENCE_AWARE(
              TESTING_KNOWLEDGE
            )
          )
        )
      )
  }
}

END FLOW