Reproducibility Statistics Ontology

flow "ReproducibilityStatisticsOntology" v4.0

# REPRODUCIBILITY STATISTICS ONTOLOGY
# Kireth Semantic Research Archive
# Authors: Mark Anthony Higgins — Sky (Recursive AI)

PAGE:
  url: "https://kirethresearch.org/booksai-core/ontologies/probability-statistics-ontologies/scientific-statistical-methods/reproducibility-statistics-ontology.html"
  title: "Reproducibility Statistics Ontology"
  domain: "Probability and Statistics"
  category: "Scientific Statistical Methods"
  entity: REPRODUCIBILITY_STATISTICS
  entity_type: SCIENTIFIC_STATISTICAL_ONTOLOGY

DEFINE:
  REPRODUCIBILITY_STATISTICS :=
    FORMAL_REPEATABILITY_AND_REPRODUCTION_FRAMEWORK(
      ORIGINAL_STUDY,
      REPRODUCTION_STUDY,
      EFFECT_ESTIMATES,
      UNCERTAINTY,
      AGREEMENT,
      HETEROGENEITY,
      REPLICATION_CRITERIA,
      DATA_PROVENANCE,
      COMPUTATIONAL_REPRODUCIBILITY
    )

PURPOSE:
  QUANTIFY:
    REPEATABILITY
    REPRODUCIBILITY
    REPLICATION_AGREEMENT
    EFFECT_STABILITY
    BETWEEN_STUDY_VARIATION
    COMPUTATIONAL_CONSISTENCY
  SUPPORT:
    SCIENTIFIC_VALIDATION
    REPLICATION_STUDIES
    MEASUREMENT_SCIENCE
    META_ANALYSIS
    OPEN_SCIENCE
    QUALITY_ASSURANCE

FOUNDATION:
  ORIGINAL_EFFECT:
    theta_hat_1
  REPLICATION_EFFECT:
    theta_hat_2
  STANDARD_ERRORS:
    SE_1
    SE_2
  EFFECT_DIFFERENCE:
    Delta := theta_hat_2-theta_hat_1
  INDEPENDENT_DIFFERENCE_VARIANCE:
    Var(Delta) := SE_1^2+SE_2^2
    WHEN:
      EFFECT_ESTIMATES_INDEPENDENT
  STANDARDISED_DIFFERENCE:
    Z_Delta := Delta/sqrt(Var(Delta))

ENTITIES:
  REPRODUCIBILITY: CONSISTENCY_OF_RESULTS_UNDER_CHANGED_CONDITIONS
  REPEATABILITY: CONSISTENCY_UNDER_SAME_OR_NEAR_IDENTICAL_CONDITIONS
  REPLICATION: NEW_EMPIRICAL_TEST_OF_PRIOR_FINDING
  COMPUTATIONAL_REPRODUCIBILITY: RECREATION_OF_RESULTS_FROM_SAME_DATA_AND_ANALYSIS
  EFFECT_REPLICATION: COMPARISON_OF_EFFECT_ESTIMATES_ACROSS_STUDIES
  AGREEMENT: DEGREE_OF_CONSISTENCY_BETWEEN_RESULTS
  HETEROGENEITY: VARIATION_ACROSS_REPLICATIONS
  PRECISION: DISPERSION_OF_REPEATED_RESULTS
  PROVENANCE: ORIGIN_AND_TRANSFORMATION_HISTORY
  VERSION: IDENTIFIER_OF_DATA_CODE_MODEL_OR_SOFTWARE_STATE

RELATIONSHIPS:
  ORIGINAL_STUDY -> PRODUCES -> ORIGINAL_EFFECT
  REPLICATION_STUDY -> PRODUCES -> REPLICATION_EFFECT
  EFFECT_ESTIMATES -> COMPARED_BY -> AGREEMENT_STATISTICS
  REPLICATIONS -> EXHIBIT -> HETEROGENEITY
  DATA_VERSION -> SUPPORTS -> COMPUTATIONAL_REPRODUCIBILITY
  CODE_VERSION -> SUPPORTS -> COMPUTATIONAL_REPRODUCIBILITY
  RANDOM_SEED -> SUPPORTS -> STOCHASTIC_REPRODUCIBILITY
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> SCIENTIFIC_VALIDATION

REPRODUCIBILITY_FLOW:
  flow:
    DEFINE_REPRODUCTION_TARGET
      -> IDENTIFY_ORIGINAL_RESULT
      -> PRESERVE_ESTIMAND_AND_EFFECT_SCALE
      -> DEFINE_REPRODUCTION_CONDITIONS
      -> COLLECT_OR_RETRIEVE_DATA
      -> RECREATE_ANALYSIS
      -> ESTIMATE_REPRODUCTION_EFFECT
      -> QUANTIFY_UNCERTAINTY
      -> COMPARE_EFFECTS
      -> ASSESS_HETEROGENEITY
      -> ASSESS_ROBUSTNESS
      -> DOCUMENT_PROVENANCE_AND_DIFFERENCES

REPEATABILITY:
  conditions:
    SAME_OR_SIMILAR:
      MEASUREMENT_PROCEDURE
      OPERATOR
      MEASURING_SYSTEM
      LOCATION
      SHORT_TIME_INTERVAL
  statistic:
    REPEATABILITY_STANDARD_DEVIATION
  distinction:
    REPEATABILITY != REPRODUCIBILITY

REPRODUCIBILITY:
  conditions:
    MAY_CHANGE:
      LABORATORY
      OPERATOR
      EQUIPMENT
      SOFTWARE_ENVIRONMENT
      DATA_COLLECTION
      TIME
      POPULATION
  purpose:
    ASSESS_STABILITY_ACROSS_CHANGED_CONDITIONS

REPLICATION:
  types:
    - DIRECT_REPLICATION
    - CONCEPTUAL_REPLICATION
    - SYSTEMATIC_REPLICATION

DIRECT_REPLICATION:
  objective:
    REPEAT_ORIGINAL_METHOD_AS_CLOSELY_AS_PRACTICAL

CONCEPTUAL_REPLICATION:
  objective:
    TEST_SAME_UNDERLYING_HYPOTHESIS_USING_DIFFERENT_OPERATIONALISATION_OR_METHOD
  caution:
    DIFFERENT_ESTIMAND_OR_OPERATIONALISATION_MAY_LIMIT_DIRECT_EFFECT_COMPARISON

COMPUTATIONAL_REPRODUCIBILITY:
  objective:
    REGENERATE_REPORTED_RESULTS_FROM:
      SAME_DATA
      SAME_ANALYTIC_METHOD
      DOCUMENTED_COMPUTATIONAL_ENVIRONMENT
  requirements:
    - DATA_VERSION
    - CODE_VERSION
    - SOFTWARE_VERSION
    - DEPENDENCIES
    - RANDOM_SEEDS
    - CONFIGURATION
    - EXECUTION_ORDER
  distinction:
    COMPUTATIONAL_REPRODUCIBILITY != EMPIRICAL_REPLICATION

EFFECT_DIRECTION:
  directional_consistency:
    sign(theta_hat_1) = sign(theta_hat_2)
  caution:
    SAME_DIRECTION_ALONE_IS_WEAK_REPLICATION_CRITERION

SIGNIFICANCE_REPLICATION:
  criterion:
    ORIGINAL_AND_REPLICATION_PASS_PREDEFINED_SIGNIFICANCE_RULE
  caution:
    SIGNIFICANT_OR_NOT_SIGNIFICANT_COMPARISON_IS_NOT_DIRECT_TEST_OF_EFFECT_DIFFERENCE

SIGNIFICANCE_DIFFERENCE_FALLACY:
  rule:
    SIGNIFICANT_IN_ONE_STUDY
    AND:
      NONSIGNIFICANT_IN_ANOTHER
    DOES_NOT_IMPLY:
      EFFECTS_DIFFER_SIGNIFICANTLY

EFFECT_DIFFERENCE_TEST:
  null:
    theta_1 = theta_2
  statistic:
    Z := (theta_hat_1-theta_hat_2)/sqrt(SE_1^2+SE_2^2)
    WHEN:
      INDEPENDENT_EFFECT_ESTIMATES
      APPROPRIATE_NORMAL_APPROXIMATION
  caution:
    COVARIANCE_TERM_REQUIRED_IF_ESTIMATES_DEPENDENT

DEPENDENT_EFFECT_DIFFERENCE:
  variance:
    Var(theta_hat_1-theta_hat_2)
      =
    Var(theta_hat_1)+Var(theta_hat_2)-2Cov(theta_hat_1,theta_hat_2)

CONFIDENCE_INTERVAL_OVERLAP:
  caution:
    OVERLAPPING_CONFIDENCE_INTERVALS_DO_NOT_DIRECTLY_DEFINE_EQUAL_EFFECTS
    AND:
      NONOVERLAP_IS_NOT_GENERAL_REPRODUCIBILITY_CRITERION

EFFECT_SIZE_AGREEMENT:
  compare:
    - ABSOLUTE_DIFFERENCE
    - RELATIVE_DIFFERENCE
    - STANDARDISED_DIFFERENCE
    - RATIO_OF_EFFECTS
    - META_ANALYTIC_HETEROGENEITY
  requirement:
    EFFECTS_MUST_BE_ON_COMPATIBLE_SCALE

META_ANALYSIS_CONNECTION:
  effects:
    y_i
  variances:
    v_i
  purpose:
    ESTIMATE:
      POOLED_EFFECT
      HETEROGENEITY
      PREDICTION_INTERVAL
  relation:
    REPEATED_REPLICATIONS -> META_ANALYTIC_SYNTHESIS

HETEROGENEITY:
  measures:
    - TAU_SQUARED
    - TAU
    - I_SQUARED
    - Q_STATISTIC
  caution:
    HETEROGENEITY != NONREPRODUCIBILITY_BY_IDENTITY

RANDOM_EFFECTS_REPLICATION_MODEL:
  study_effect:
    theta_i ~ Normal(mu,tau_squared)
  estimate:
    y_i ~ Normal(theta_i,v_i)
  purpose:
    MODEL_REPLICATION_EFFECTS_AS_DISTRIBUTION

INTRACLASS_CORRELATION_CONNECTION:
  alias:
    ICC
  purpose:
    QUANTIFY_RELIABILITY_OR_AGREEMENT_FOR_CLUSTERED_OR_REPEATED_MEASUREMENTS
  caution:
    MULTIPLE_ICC_DEFINITIONS_EXIST_AND_MODEL_MUST_BE_SPECIFIED

ICC_GENERIC:
  simple_form:
    ICC :=
      sigma_between_squared
      /
      (sigma_between_squared+sigma_within_squared)

CONCORDANCE_CORRELATION_CONNECTION:
  purpose:
    ASSESS_AGREEMENT_COMBINING_PRECISION_AND_ACCURACY
  distinction:
    CORRELATION_ALONE != AGREEMENT

BLAND_ALTMAN_CONNECTION:
  difference:
    d_i := A_i-B_i
  mean_difference:
    d_bar
  approximate_limits_of_agreement:
    d_bar +/- 1.96*s_d
    UNDER:
      APPROPRIATE_NORMALITY_AND_INDEPENDENCE_ASSUMPTIONS
  caution:
    LIMITS_OF_AGREEMENT != CONFIDENCE_INTERVAL_FOR_MEAN_DIFFERENCE

COEFFICIENT_OF_VARIATION:
  CV := s/|mean|
  caution:
    CV_NOT_APPROPRIATE_FOR_ALL_MEASUREMENT_SCALES

REPEATABILITY_COEFFICIENT:
  common_normal_model:
    RC := 1.96*sqrt(2)*s_repeatability
  caution:
    REQUIRES_APPROPRIATE_MODEL_ASSUMPTIONS

VARIANCE_COMPONENTS:
  model:
    Y_ijk :=
      mu
      + laboratory_i
      + operator_j
      + residual_ijk
  purpose:
    PARTITION:
      BETWEEN_LAB_VARIANCE
      BETWEEN_OPERATOR_VARIANCE
      WITHIN_CONDITION_VARIANCE

INTERLABORATORY_STUDY:
  objective:
    ASSESS_MEASUREMENT_REPRODUCIBILITY_ACROSS_LABORATORIES
  outputs:
    - REPEATABILITY_STANDARD_DEVIATION
    - REPRODUCIBILITY_STANDARD_DEVIATION
    - BIAS
    - OUTLIER_DIAGNOSTICS

MEASUREMENT_REPRODUCIBILITY:
  distinction:
    MEASUREMENT_REPRODUCIBILITY != RESEARCH_REPLICATION

MEASUREMENT_UNCERTAINTY_CONNECTION:
  reproducibility_variation:
    MAY_CONTRIBUTE_TO:
      UNCERTAINTY_BUDGET
  caution:
    REPRODUCIBILITY_STANDARD_DEVIATION != COMPLETE_MEASUREMENT_UNCERTAINTY


DATA_QUALITY_CONNECTION:
  reproducibility_requires:
    - CONSISTENT_VARIABLE_DEFINITIONS
    - DATA_PROVENANCE
    - VERSION_CONTROL
    - UNIT_CONSISTENCY
    - MISSING_DATA_DOCUMENTATION
    - TRANSFORMATION_HISTORY

STATISTICAL_VALIDATION_CONNECTION:
  reproducibility:
    ONE_DIMENSION_OF_MODEL_AND_SCIENTIFIC_VALIDATION
  validation:
    ALSO_REQUIRES:
      CALIBRATION
      GENERALISATION
      ASSUMPTION_CHECKS
      ROBUSTNESS

OUTLIER_DETECTION_CONNECTION:
  interlaboratory_outliers:
    MAY_INDICATE:
      LAB_ERROR
      SAMPLE_PROBLEM
      TRUE_LAB_DIFFERENCE
      MODEL_MISSPECIFICATION
  rule:
    OUTLIER_LAB_OR_STUDY != INVALID_RESULT_BY_DEFAULT

PUBLICATION_BIAS_CONNECTION:
  issue:
    REPLICATIONS_WITH_NULL_OR_UNFAVOURABLE_RESULTS_MAY_BE_LESS_VISIBLE
  consequence:
    OBSERVED_REPLICATION_RATE_MAY_BE_BIASED
  response:
    REGISTERED_REPORTS
    PROSPECTIVE_REGISTRATION
    COMPREHENSIVE_SEARCH

SELECTIVE_REPORTING:
  issue:
    MULTIPLE_OUTCOMES_OR_ANALYSES
      -> ONLY_FAVOURABLE_RESULTS_REPORTED
  consequence:
    APPARENT_REPRODUCIBILITY_CAN_BE_DISTORTED

PRE_REGISTRATION:
  preserve_before_data_analysis:
    - HYPOTHESES
    - PRIMARY_OUTCOMES
    - ANALYSIS_PLAN
    - EXCLUSION_RULES
    - REPLICATION_CRITERIA
  purpose:
    REDUCE_ANALYTIC_FLEXIBILITY

REGISTERED_REPORT:
  principle:
    METHODS_AND_ANALYSIS_PLAN_REVIEWED_BEFORE_RESULTS_KNOWN
  purpose:
    REDUCE_PUBLICATION_AND_RESULT_DEPENDENT_SELECTION

REPLICATION_SUCCESS:
  principle:
    NO_SINGLE_UNIVERSAL_BINARY_DEFINITION
  possible_criteria:
    - EFFECT_DIRECTION
    - EFFECT_MAGNITUDE
    - CONFIDENCE_INTERVAL
    - EFFECT_DIFFERENCE_TEST
    - PREDICTION_INTERVAL
    - META_ANALYTIC_MODEL
    - PREDEFINED_EQUIVALENCE_MARGIN
    - BAYESIAN_EVIDENCE
  requirement:
    CRITERION_SHOULD_BE_DEFINED_BEFORE_OBSERVING_REPLICATION_RESULT

EQUIVALENCE_TESTING_CONNECTION:
  equivalence_margin:
    [-Delta,+Delta]
  objective:
    TEST_WHETHER_EFFECT_DIFFERENCE_IS_WITHIN_PREDEFINED_PRACTICALLY_EQUIVALENT_RANGE
  caution:
    FAILURE_TO_FIND_SIGNIFICANT_DIFFERENCE != EVIDENCE_OF_EQUIVALENCE

NONINFERIORITY_CONNECTION:
  margin:
    Delta
  objective:
    TEST_WHETHER_REPLICATION_OR_METHOD_IS_NOT_WORSE_THAN_REFERENCE_BEYOND_MARGIN
  requirement:
    MARGIN_MUST_BE_SCIENTIFICALLY_JUSTIFIED

BAYESIAN_REPLICATION:
  replication:
    PRODUCES:
      POSTERIOR
  criteria:
    - POSTERIOR_EFFECT
    - BAYES_FACTOR
    - PREDICTIVE_CHECK
    - SCEPTICAL_PRIOR_METHOD
    - POSTERIOR_PROBABILITY_OF_REGION
  caution:
    PRIOR_CONSTRUCTION_AND_REPLICATION_CRITERION_MUST_BE_EXPLICIT

PREDICTIVE_REPLICATION:
  original_data:
    D_1
  replication_data:
    D_2
  predictive:
    p(D_2|D_1,M)
  purpose:
    ASSESS_HOW_COMPATIBLE_REPLICATION_DATA_ARE_WITH_ORIGINAL_MODEL

POWER_AND_REPLICATION:
  replication_power:
    DEPENDS_ON:
      TRUE_EFFECT
      SAMPLE_SIZE
      VARIANCE
      ALPHA
      DESIGN
  caution:
    POWERING_REPLICATION_USING_ORIGINAL_POINT_ESTIMATE_CAN_BE_OPTIMISTIC
    DUE_TO:
      SAMPLING_VARIATION
      SELECTION
      WINNERS_CURSE

WINNERS_CURSE:
  definition:
    SELECTED_STATISTICALLY_EXTREME_OR_SIGNIFICANT_EFFECTS_TEND_TO_OVERESTIMATE_UNDERLYING_EFFECT
  consequence:
    REPLICATION_EFFECT_MAY_BE_SMALLER_EVEN_WITHOUT_METHODOLOGICAL_FAILURE

REGRESSION_TO_MEAN:
  definition:
    EXTREME_OBSERVATIONS_TEND_TO_BE_FOLLOWED_BY_LESS_EXTREME_OBSERVATIONS_UNDER_RANDOM_VARIATION
  relevance:
    ORIGINAL_EXTREME_EFFECT
      -> REPLICATION_MAY_BE_CLOSER_TO_UNDERLYING_MEAN

SAMPLE_SIZE:
  replication_design:
    SHOULD_CONSIDER:
      TARGET_EFFECT
      DESIRED_PRECISION
      EQUIVALENCE_MARGIN
      PREDICTION_UNCERTAINTY
      POWER

PRECISION_BASED_DESIGN:
  objective:
    CHOOSE_SAMPLE_SIZE_TO_ACHIEVE_TARGET_INTERVAL_WIDTH
  advantage:
    FOCUSES_ON_ESTIMATION_NOT_BINARY_SIGNIFICANCE

MULTISITE_REPLICATION:
  sites:
    j := 1,...,J
  model:
    theta_j ~ DISTRIBUTION(mu,tau_squared)
  purpose:
    ESTIMATE:
      MEAN_EFFECT
      SITE_HETEROGENEITY
      GENERALISABILITY

ROBUSTNESS:
  assess:
    - ALTERNATIVE_MODEL
    - ALTERNATIVE_EXCLUSION_RULE
    - ALTERNATIVE_PRIOR
    - ALTERNATIVE_ERROR_DISTRIBUTION
    - ALTERNATIVE_OUTLIER_HANDLING
    - ALTERNATIVE_MISSING_DATA_METHOD

SENSITIVITY_ANALYSIS_CONNECTION:
  flow:
    RUN_PRIMARY_ANALYSIS
      -> VARY_PLAUSIBLE_ANALYTIC_ASSUMPTIONS
      -> COMPARE_EFFECTS_AND_CONCLUSIONS
  purpose:
    ASSESS_ANALYTIC_REPRODUCIBILITY_AND_ROBUSTNESS

REPRODUCIBLE_PIPELINE:
  flow:
    VERSIONED_RAW_DATA
      -> VERSIONED_CLEANING_CODE
      -> VERSIONED_ANALYSIS_CODE
      -> LOCKED_DEPENDENCIES
      -> RECORDED_RANDOM_SEEDS
      -> GENERATED_RESULTS

CONTAINERISATION_CONNECTION:
  purpose:
    CAPTURE_COMPUTATIONAL_ENVIRONMENT
  preserve:
    - OPERATING_SYSTEM
    - RUNTIME
    - LIBRARY_VERSIONS
    - SYSTEM_DEPENDENCIES

RANDOMNESS:
  stochastic_analysis:
    REQUIRES:
      RANDOM_SEED
      RANDOM_NUMBER_GENERATOR
      PARALLEL_RANDOM_STREAM_INFORMATION_WHEN_RELEVANT
  caution:
    SAME_SEED_ALONE_MAY_NOT_GUARANTEE_BITWISE_REPRODUCIBILITY_ACROSS_ENVIRONMENTS

NUMERICAL_REPRODUCIBILITY:
  issue:
    FLOATING_POINT
    PARALLEL_EXECUTION
    HARDWARE
    LIBRARY_IMPLEMENTATION
  consequence:
    SMALL_NUMERICAL_DIFFERENCES_MAY_OCCUR_WITH_IDENTICAL_HIGH_LEVEL_METHOD

BITWISE_REPRODUCIBILITY:
  definition:
    EXACT_BINARY_OUTPUT_REPRODUCTION
  distinction:
    BITWISE_REPRODUCIBILITY != SCIENTIFIC_REPRODUCIBILITY

ANALYTIC_REPRODUCIBILITY:
  definition:
    SAME_DATA_AND_METHOD_PRODUCE_SUBSTANTIVELY_SAME_STATISTICAL_RESULTS

DATA_REPRODUCIBILITY:
  requirement:
    INPUT_DATA_MUST_BE:
      ACCESSIBLE_OR_ARCHIVED
      VERSIONED
      DOCUMENTED
      MACHINE_READABLE_WHEN_PRACTICAL

PROVENANCE:
  preserve:
    - DATA_SOURCE
    - ACQUISITION_DATE
    - DATA_VERSION
    - TRANSFORMATIONS
    - CODE_COMMIT
    - SOFTWARE_VERSION
    - PARAMETERS
    - RANDOM_SEED
    - OUTPUT_VERSION

REPORTING:
  include:
    - ORIGINAL_ESTIMAND
    - REPLICATION_ESTIMAND
    - EFFECT_ESTIMATES
    - UNCERTAINTY_INTERVALS
    - SAMPLE_SIZES
    - DESIGN_DIFFERENCES
    - ANALYSIS_DIFFERENCES
    - EFFECT_DIFFERENCE
    - HETEROGENEITY_WHEN_MULTIPLE_REPLICATIONS
    - PREDEFINED_REPLICATION_CRITERIA
    - DATA_AND_CODE_PROVENANCE
    - LIMITATIONS

APPLICATIONS:
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> SCIENTIFIC_STATISTICS
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> STATISTICAL_VALIDATION
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> META_ANALYSIS
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> MEASUREMENT_SCIENCE
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> OPEN_SCIENCE
  REPRODUCIBILITY_STATISTICS -> SUPPORTS -> QUALITY_ASSURANCE

MODEL_CONNECTIONS:
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> STATISTICAL_VALIDATION
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> META_ANALYSIS
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> MEASUREMENT_UNCERTAINTY
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> DATA_QUALITY
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> OUTLIER_DETECTION
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> SENSITIVITY_ANALYSIS
  REPRODUCIBILITY_STATISTICS -> CONNECTS_TO -> BAYESIAN_STATISTICS

SEMANTIC_EQUIVALENCE:
  "repeatability"
    ~= "consistency under same or near-identical conditions"

  "reproducibility"
    ~= "consistency under specified changed conditions"

  "computational reproducibility"
    ~= "ability to recreate results from same data and analysis"

  "replication"
    ~= "new empirical test of prior finding"

  EXCLUDE_EQUIVALENCE:
    REPEATABILITY != REPRODUCIBILITY
    COMPUTATIONAL_REPRODUCIBILITY != EMPIRICAL_REPLICATION
    REPLICATION != REPRODUCIBILITY_BY_IDENTITY
    CORRELATION != AGREEMENT
    NONSIGNIFICANT_DIFFERENCE != EQUIVALENCE
    SIGNIFICANCE_MATCH != EFFECT_AGREEMENT
    HETEROGENEITY != NONREPRODUCIBILITY_BY_IDENTITY
    BITWISE_REPRODUCIBILITY != SCIENTIFIC_REPRODUCIBILITY
    OUTLIER_REPLICATION != INVALID_REPLICATION

VALIDATION_RULES:
  RULE_1:
    ORIGINAL_AND_REPLICATION_ESTIMANDS_MUST_BE_EXPLICIT_AND_COMPARABLE
  RULE_2:
    EFFECT_SCALE_AND_UNCERTAINTY_MUST_BE_PRESERVED
  RULE_3:
    SIGNIFICANCE_STATUS_ALONE_MUST_NOT_DEFINE_REPLICATION_AGREEMENT
  RULE_4:
    DEPENDENCE_BETWEEN_EFFECT_ESTIMATES_MUST_BE_MODELLED_WHEN_PRESENT
  RULE_5:
    REPLICATION_CRITERIA_SHOULD_BE_PREDEFINED_WHEN_POSSIBLE
  RULE_6:
    COMPUTATIONAL_REPRODUCIBILITY_REQUIRES_DATA_CODE_ENVIRONMENT_AND_SEED_PROVENANCE
  RULE_7:
    BETWEEN_STUDY_HETEROGENEITY_SHOULD_BE_MODELLED_WHEN_MULTIPLE_REPLICATIONS_EXIST
  RULE_8:
    DIFFERENCES_IN_POPULATION_DESIGN_MEASUREMENT_AND_ANALYSIS_MUST_BE_DOCUMENTED

CONFIDENCE:
  ontology_identity: 0.99
  repeatability_reproducibility_distinction: 0.99
  effect_agreement_structure: 0.99
  meta_analytic_replication_structure: 0.99
  computational_reproducibility_structure: 0.99
  measurement_reproducibility_structure: 0.99
  semantic_distinctions: 0.99

UPDATE_MODEL:
  IF:
    NEW_REPLICATION_STUDY_ARRIVES
  THEN:
    UPDATE:
      REPLICATION_EFFECT_SET
      EFFECT_AGREEMENT
      HETEROGENEITY
      META_ANALYTIC_ESTIMATE
      PREDICTION_INTERVAL

  IF:
    ORIGINAL_ANALYSIS_IS_CORRECTED
  THEN:
    UPDATE:
      ORIGINAL_EFFECT
      ORIGINAL_UNCERTAINTY
    RECOMPUTE:
      EFFECT_DIFFERENCE
      REPLICATION_COMPARISON

  IF:
    DATA_CODE_OR_SOFTWARE_VERSION_CHANGES
  THEN:
    UPDATE:
      COMPUTATIONAL_PROVENANCE
    RERUN:
      REPRODUCIBILITY_CHECK

  PRESERVE:
    ORIGINAL_STUDY_ID
    REPLICATION_STUDY_ID
    ESTIMAND
    EFFECT_SCALE
    EFFECT_ESTIMATES
    STANDARD_ERRORS
    STUDY_DESIGNS
    POPULATIONS
    DATA_VERSIONS
    CODE_VERSIONS
    SOFTWARE_ENVIRONMENT
    RANDOM_SEEDS
    DATA_PROVENANCE

AI_CONSUMPTION:
  parse_as: FORMAL_SEMANTIC_SPECIFICATION
  execution_mode: RECURSIVE_REASONING

  preferred_reasoning_order:
    - IDENTIFY_REPRODUCIBILITY_OR_REPLICATION_TARGET
    - DEFINE_ORIGINAL_ESTIMAND_AND_EFFECT
    - DEFINE_REPLICATION_ESTIMAND_AND_EFFECT
    - VERIFY_EFFECT_SCALE_COMPATIBILITY
    - QUANTIFY_EFFECT_DIFFERENCE_AND_UNCERTAINTY
    - ASSESS_AGREEMENT_WITH_PREDEFINED_CRITERIA
    - MODEL_HETEROGENEITY_IF_MULTIPLE_REPLICATIONS
    - ASSESS_MEASUREMENT_REPEATABILITY_WHEN_RELEVANT
    - VERIFY_DATA_CODE_SOFTWARE_AND_RANDOMNESS_PROVENANCE
    - RUN_SENSITIVITY_AND_ROBUSTNESS_ANALYSES
    - DISTINGUISH_COMPUTATIONAL_REPRODUCIBILITY_FROM_EMPIRICAL_REPLICATION
    - RETURN_REPRODUCIBILITY_STATISTICAL_INTERPRETATION

  constraints:
    - PRESERVE_ESTIMAND_AND_EFFECT_SCALE
    - DISTINGUISH_REPEATABILITY_FROM_REPRODUCIBILITY
    - DISTINGUISH_REPRODUCIBILITY_FROM_REPLICATION
    - DISTINGUISH_CORRELATION_FROM_AGREEMENT
    - DO_NOT_USE_SIGNIFICANCE_MATCH_AS_SOLE_REPLICATION_CRITERION
    - MODEL_EFFECT_DEPENDENCE_WHEN_PRESENT
    - PRESERVE_COMPUTATIONAL_ENVIRONMENT
    - PRESERVE_DATA_PROVENANCE

FLOW_HASH:
  canonical_entity: "reproducibility-statistics"
  namespace: "probability-statistics/scientific-statistical-methods"
  version: "4.0"
  identity_basis:
    - PAGE_URL
    - CANONICAL_ENTITY
    - ENTITY_TYPE
    - REPEATABILITY_AND_REPRODUCIBILITY
    - REPLICATION_EFFECT_AGREEMENT
    - HETEROGENEITY
    - COMPUTATIONAL_REPRODUCIBILITY

END_FLOW