From af83fdcd13b5a414f42841ffb5822deee3ee1fdc Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 18 Nov 2025 12:20:03 -0500 Subject: [PATCH 01/58] intermediate update --- .gitignore | 2 + CMakeLists.txt | 109 --------- Makefile | 174 ++++++++++++++ include/HaplotypeVcfParser.h | 137 ----------- include/{Matrix.h => matrix.h} | 0 include/parse_hts.h | 142 ++++++++++++ include/utils.h | 102 --------- src/HaplotypeDataRecord.cpp | 139 ------------ src/HaplotypeVcfParser.cpp | 220 ------------------ src/main.cpp | 284 +++++++++++++---------- src/{Matrix.cpp => matrix.cpp} | 6 +- src/parse_hts.cpp | 235 +++++++++++++++++++ src/utils.cpp | 301 ------------------------- tests/geno_test_data.bcf | Bin 0 -> 2257 bytes tests/{test.vcf => geno_test_data.vcf} | 0 tests/geno_test_data.vcf.gz | Bin 0 -> 1984 bytes tests/main.cpp | 6 + tests/test_haplotype_data_record.cpp | 117 ---------- tests/test_haplotype_vcf_parser.cpp | 116 ---------- tests/test_matrix.cpp | 161 ++++++------- tests/test_parse_hts.cpp | 113 ++++++++++ tests/test_utils.cpp | 97 -------- 22 files changed, 916 insertions(+), 1545 deletions(-) delete mode 100644 CMakeLists.txt create mode 100644 Makefile delete mode 100644 include/HaplotypeVcfParser.h rename include/{Matrix.h => matrix.h} (100%) create mode 100644 include/parse_hts.h delete mode 100644 include/utils.h delete mode 100644 src/HaplotypeDataRecord.cpp delete mode 100644 src/HaplotypeVcfParser.cpp rename src/{Matrix.cpp => matrix.cpp} (95%) create mode 100644 src/parse_hts.cpp delete mode 100644 src/utils.cpp create mode 100644 tests/geno_test_data.bcf rename tests/{test.vcf => geno_test_data.vcf} (100%) create mode 100644 tests/geno_test_data.vcf.gz create mode 100644 tests/main.cpp delete mode 100644 tests/test_haplotype_data_record.cpp delete mode 100644 tests/test_haplotype_vcf_parser.cpp create mode 100644 tests/test_parse_hts.cpp delete mode 100644 tests/test_utils.cpp diff --git a/.gitignore b/.gitignore index b881939..03e5e7c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ build/ *~ *.DS_Store *.vscode/ +scratch/ +data/ diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 47f55ec..0000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,109 +0,0 @@ - -# By: Robert Vogel -# Affiliation: Palmer Lab at UCSD -# 2025-01-11 -# -# Acknowledgment -# This file was originally written by Robert Vogel. Claude, -# the AI assistant by Anthropic reviewed this file and provided -# recommendations. - -cmake_minimum_required(VERSION 3.21) -project(hgrm VERSION 0.0.1) - -set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_STANDARD_REQUIRED True) -# set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexceptions") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") - - -find_package(GTest REQUIRED) - -add_library(matrix_lib src/Matrix.cpp) -target_include_directories(matrix_lib PUBLIC include) - -add_library(utils_lib src/utils.cpp) -target_include_directories(utils_lib PUBLIC include) - -add_library(parse_lib src/HaplotypeDataRecord.cpp src/HaplotypeVcfParser.cpp) -target_include_directories(parse_lib PUBLIC include) - - - -# Testing configuration -enable_testing() - -add_executable( - test_matrix - tests/test_matrix.cpp -) - -target_link_libraries( - test_matrix - PRIVATE - matrix_lib - GTest::gtest_main -) - -add_executable( - test_utils - tests/test_utils.cpp -) -target_link_libraries( - test_utils - PRIVATE - utils_lib - GTest::gtest_main -) - - -add_executable( - test_haplotype_data_record - tests/test_haplotype_data_record.cpp -) -target_link_libraries( - test_haplotype_data_record - PRIVATE - parse_lib - matrix_lib - utils_lib - GTest::gtest_main -) - -add_executable( - test_haplotype_vcf_parser - tests/test_haplotype_vcf_parser.cpp -) - - - -target_link_libraries( - test_haplotype_vcf_parser - PRIVATE - parse_lib - matrix_lib - utils_lib - GTest::gtest_main -) - - -add_executable( - hgrm - src/main.cpp -) - -target_link_libraries( - hgrm - PRIVATE - parse_lib - matrix_lib - utils_lib -) - - -include(GoogleTest) -gtest_discover_tests(test_matrix) -gtest_discover_tests(test_utils) -gtest_discover_tests(test_haplotype_data_record) -gtest_discover_tests(test_haplotype_vcf_parser) - diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9040d07 --- /dev/null +++ b/Makefile @@ -0,0 +1,174 @@ +# +# 2025 Palmer Lab +# +###################################################################### +# machine dependent options +###################################################################### +# CXXFLAGS note: Remember that -g flag is for generating source-level +# debug info. + +# OBJ_OUTPUT_OPTIONS: compiler options. Clang, and I presume +# also gcc, support the creation of dependency files (-MMD) and (-MP) phony +# targets required for constructing an object file. The (-o) and $@ +# are the standard output file designation submitted to the compiler, +# and $@ is an automatic variable storing the rules target. +# library archive program +# + +ifneq ($(shell which clang++),) +CXX = clang++ +else ifneq ($(shell which g++),) +CXX = g++ +else +$(error "Couldn't establish either clang or gcc compiler availability") +endif + + +CXXFLAGS = -g -std=c++17 -Wall -Werror + +ifndef VIM +CXXFLAGS += -fdiagnostics-color=always +endif + +OBJ_OUTPUT_OPTIONS = -c -MMD -MP -o $@ +AR = ar +AR_FLAGS = crs + +LOCAL_LIB = $(HOME)/.local/lib +LOCAL_LD = $(HOME)/.local/include + +###################################################################### +# define src and obj variables +###################################################################### + +SRC_DIR = src +HEADER_DIR = include +BUILD_DIR = build + +CXXLD += $(PWD)/include +CXXLD += $(LOCAL_LD) + +CXXLDFLAGS = $(addprefix -I, $(CXXLD)) + +CXXLIB += $(LOCAL_LIB) +CXXLIBFLAGS = $(addprefix -L, $(CXXLIB)) + +APP_FILES = matrix.cpp parse_hts.cpp +APP_SRC = $(addprefix $(SRC_DIR)/, $(APP_FILES)) +APP_OBJS = $(addprefix $(BUILD_DIR)/, $(APP_FILES:.cpp=.o)) +APP_DEPS = $(APP_OBJS:.o=.d) + + +TEST_DIR = tests +TEST_SRC = $(wildcard $(TEST_DIR)/test_*.cpp) +TEST_OBJS = $(subst $(TEST_DIR), $(BUILD_DIR), $(TEST_SRC:.cpp=.o)) +TEST_DEPS = $(TEST_OBJS:.o=.d) +TEST_DATA = $(wildcard $(TEST_DIR)/geno_test_data.*) +TEST_TARGET_PRG = $(BUILD_DIR)/runtests + + +###################################################################### +# Executable Build Rules +###################################################################### + +TARGET = $(BUILD_DIR)/hgrm + +.PHONY: all +all: $(TARGET) $(TEST_TARGET_PRG) data + +$(TARGET): $(SRC_DIR)/main.cpp $(APP_OBJS) + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -largparse -lhts + + +# Recall that -c flag prevents the compiler linking object files +$(BUILD_DIR)/%.o: $(SRC_DIR)/%.cpp | $(BUILD_DIR) + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(OBJ_OUTPUT_OPTIONS) $< + +$(BUILD_DIR): + mkdir $@ + +###################################################################### +# Test Build Rules +###################################################################### + + +$(TEST_TARGET_PRG): $(TEST_DIR)/main.cpp $(TEST_OBJS) | $(TARGET) + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -lgtest + +$(BUILD_DIR)/test_%.o: $(TEST_DIR)/test_%.cpp + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(OBJ_OUTPUT_OPTIONS) $< + + +.PHONY: data +data: $(TEST_DATA) + rsync -avz $^ $(BUILD_DIR)/ + +# tests: $(BUILD_DIR)/test_log #$(BUILD_DIR)/test_argparse +# +# $(BUILD_DIR)/test_log: $(BUILD_DIR)/test_log.o $(BUILD_DIR)/logger.o ~/.local/lib/libgtest.a +# $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ +# +# $(BUILD_DIR)/test_argparse: $(BUILD_DIR)/test_argparse.o \ +# $(BUILD_DIR)/argparse.o \ +# ~/.local/lib/libgtest.a +# $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) -I$(LOCAL_INCLUDE) -L$(LOCAL_LIB) -o $@ $^ + +# $(TEST_OBJS): $(TEST_SRC) +# $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) -I$(LOCAL_INCLUDE) $(OBJ_OUTPUT_OPTIONS) $< +# +# $(BUILD_DIR)/test_log.o: $(TEST_DIR)/test_log.cpp +# $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(OBJ_OUTPUT_OPTIONS) $^ + + +# $(TEST_OBJS): $(TEST_SRC) +# $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) -I$(LOCAL_INCLUDE) -L$(LOCAL_LIB) -o $@ $^ + +# $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) $(OBJ_OUTPUT_OPTIONS) $^ + + +###################################################################### +# +###################################################################### + + +-include $(APP_DEPS) +-include $(TEST_DEPS) + +.PHONY: help +help: + -@echo "build hgrm" + -@echo "2025 Palmer Lab" + -@echo "" + -@echo "make hgrm executable" + -@echo "make libargparse" + + +###################################################################### +# install +###################################################################### + +# install: +# dir_header=$${prefix%/}/include/stitchr; \ +# if [ ! -d $${dir_header} ]; then \ +# mkdir -p $${dir_header}; \ +# fi; \ +# for hfile in $$(ls $(HEADER_DIR)); do \ +# cp $$hfile $${dir_header}/$${hfile}; \ +# done; \ +# \ +# dir_lib=$${prefix%/}/lib; \ +# if [ ! -d $${dir_lib} ]; then \ +# mkdir -p $${dir_lib}; \ +# fi; \ +# for libfile in $$(ls $(BUILD_DIR)/*.a); do +# cp $$libfile $${dir_lib}/$${libfile}; \ +# done; \ +# \ +# dir_bin = $${prefix%/}/bin; \ +# if [ ! -d $${dir_bin} ]; then \ +# mkdir -p $${dir_bin}; \ +# fi; \ +# cp $(TARGET) $${dir_bin}/$(notdir $(TARGET)) +# + + diff --git a/include/HaplotypeVcfParser.h b/include/HaplotypeVcfParser.h deleted file mode 100644 index 511be81..0000000 --- a/include/HaplotypeVcfParser.h +++ /dev/null @@ -1,137 +0,0 @@ -// Parse STITCH vcf file -// -// -// -// By: Robert Vogel -// Affiliation: Palmer Lab at UCSD -// Date: 2025-01-09 -// -// -// Acknowledgment -// -// Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. -// -#ifndef HEADER_HAPLOTYPEVCFPARSER_H -#define HEADER_HAPLOTYPEVCFPARSER_H - -#include -#include -#include -#include -#include -#include -#include "Matrix.h" -#include "utils.h" - - -// samples are separated by white space -const char HAP_CODE[] { "HD" }; -const char META_PREFIX { '#' }; -const char MEASUREMENT_DELIM { ':' }; -const char HAP_DELIM { ',' }; -const int NUM_VCF_FIELDS { 9 }; -const char SPACE_DELIM { '\t' }; - - -// NOTE: in the future it may be best to test for set membership -static const char* VCF_FIELD_NAMES[NUM_VCF_FIELDS] { - "#CHROM", - "POS", - "ID", - "REF", - "ALT", - "QUAL", - "FILTER", - "INFO", - "FORMAT" -}; - - -// Move semantics, I don't want to copy data -class HaplotypeDataRecord -{ -public: - - HaplotypeDataRecord()=delete; - HaplotypeDataRecord(size_t, size_t); - HaplotypeDataRecord(const HaplotypeDataRecord&)=delete; - HaplotypeDataRecord(HaplotypeDataRecord&&)=delete; - - - const std::string& chrom() const; - const long pos() const; - const std::string& id() const; - const char ref() const; - const char alt() const; - const std::string& qual() const; - const std::string& filter() const; - const std::string& info() const; - const std::string& format() const; - - void parse_vcf_line(const char*); - const double& operator()(size_t, size_t) const; - - std::array dims() const; - - -private: - size_t n_samples_; - size_t k_founders_; - - std::string chrom_ { "" }; - long pos_ { -1 }; - std::string id_ { "" }; - char ref_ { '\0' }; - char alt_ { '\0' }; - std::string qual_ { "" }; - std::string filter_ { "" }; - std::string info_ { "" }; - std::string format_ { "" }; - - std::unique_ptr samples_ { nullptr }; - - StringRecord line_parse_ { SPACE_DELIM }; - StringRecord field_parse_ { MEASUREMENT_DELIM }; - StringRecord hap_parse_ { HAP_DELIM }; -}; - - -class HaplotypeVcfParser -{ -public: - - HaplotypeVcfParser()=delete; // default constructor - HaplotypeVcfParser(char* filename); // constructor - HaplotypeVcfParser(char* filename, size_t buffer_size); // constructor - //HaplotypeVcfParser(std::string filename); // constructor - HaplotypeVcfParser(const HaplotypeVcfParser&)=delete; // copy constructor - HaplotypeVcfParser(const HaplotypeVcfParser&&)=delete; // move constructor - HaplotypeVcfParser& operator=(const HaplotypeVcfParser&)=delete; // copy assignment - // ~HaplotypeVcfParser(); // descructor - - size_t n_samples() const; - size_t k_founders() const; - - bool load_record(HaplotypeDataRecord&); - -private: - const std::string fname_; - BufferedRead file_io_; - - CharBuffer line_buffer_; - size_t line_buffer_size_ { 0 }; - - size_t n_cols_ { 0 }; - size_t n_samples_ { 0 }; - size_t k_founders_ { 0 }; - size_t fpos_record_one_ { 0 }; - - - void pos_(size_t); - size_t get_line_num_char_(); - void set_params_(); -}; - -#endif diff --git a/include/Matrix.h b/include/matrix.h similarity index 100% rename from include/Matrix.h rename to include/matrix.h diff --git a/include/parse_hts.h b/include/parse_hts.h new file mode 100644 index 0000000..9b21e75 --- /dev/null +++ b/include/parse_hts.h @@ -0,0 +1,142 @@ +// Parse STITCH vcf file +// +// +// +// By: Robert Vogel +// Affiliation: Palmer Lab at UCSD +// Date: 2025-01-09 +// +// +// Acknowledgment +// +// Code design and original version completed by Robert Vogel, +// reviewed by Claude Sonnet, the AI assistant from Anthropic +// (Jan 2025), with minor recommendations incorporated. +// +#ifndef HEADER_PARSE_HTS_H +#define HEADER_PARSE_HTS_H + +#include +#include +#include +#include + +namespace htslib { +extern "C" { +#include +#include +} +} + + + +// samples are separated by white space +const char HAP_CODE[] { "HD" }; +// const char META_PREFIX { '#' }; +// const char MEASUREMENT_DELIM { ':' }; +// const char HAP_DELIM { ',' }; +// const int NUM_VCF_FIELDS { 9 }; +// const char SPACE_DELIM { '\t' }; +// +// +// // NOTE: in the future it may be best to test for set membership +// static const char* VCF_FIELD_NAMES[NUM_VCF_FIELDS] { +// "#CHROM", +// "POS", +// "ID", +// "REF", +// "ALT", +// "QUAL", +// "FILTER", +// "INFO", +// "FORMAT" +// }; +// +// +// // Move semantics, I don't want to copy data +// class HaplotypeDataRecord +// { +// public: +// +// HaplotypeDataRecord()=delete; +// HaplotypeDataRecord(size_t, size_t); +// HaplotypeDataRecord(const HaplotypeDataRecord&)=delete; +// HaplotypeDataRecord(HaplotypeDataRecord&&)=delete; +// +// +// const std::string& chrom() const; +// const long pos() const; +// const std::string& id() const; +// const char ref() const; +// const char alt() const; +// const std::string& qual() const; +// const std::string& filter() const; +// const std::string& info() const; +// const std::string& format() const; +// +// void parse_vcf_line(const char*); +// const double& operator()(size_t, size_t) const; +// +// std::array dims() const; +// +// +// private: +// size_t n_samples_; +// size_t k_founders_; +// +// std::string chrom_ { "" }; +// long pos_ { -1 }; +// std::string id_ { "" }; +// char ref_ { '\0' }; +// char alt_ { '\0' }; +// std::string qual_ { "" }; +// std::string filter_ { "" }; +// std::string info_ { "" }; +// std::string format_ { "" }; +// +// std::unique_ptr samples_ { nullptr }; +// +// StringRecord line_parse_ { SPACE_DELIM }; +// StringRecord field_parse_ { MEASUREMENT_DELIM }; +// StringRecord hap_parse_ { HAP_DELIM }; +// }; + +// Interface with htslib bcf tools +class ParseHtsVariantFile +{ +public: + // HaplotypeVcfParser(const char* variant_fname); + ParseHtsVariantFile(const char *variant_fname, const char *sample_fname); + // HaplotypeVcfParser(const std::string& variant_fname); + // HaplotypeVcfParser(const std::string& variant_fname, + // const std::string& sample_fname); + + ParseHtsVariantFile()=delete; + ParseHtsVariantFile(const ParseHtsVariantFile&)=delete; + ParseHtsVariantFile(const ParseHtsVariantFile&&)=delete; + // HaplotypeVcfParser& operator=(const HaplotypeVcfParser&)=delete; + + ~ParseHtsVariantFile(); + + // size_t n_samples() const; + // size_t k_founders() const; + + // bool load_record(HaplotypeDataRecord&); + +private: + const std::string fname_; + htslib::htsFile *fid_; + htslib::bcf_hdr_t *hdr_; + + // size_t n_cols_ { 0 }; + // size_t n_samples_ { 0 }; + // size_t k_founders_ { 0 }; + // size_t fpos_record_one_ { 0 }; + + + // void pos_(size_t); + // size_t get_line_num_char_(); + // void set_params_(); +}; + +#endif diff --git a/include/utils.h b/include/utils.h deleted file mode 100644 index f9830b5..0000000 --- a/include/utils.h +++ /dev/null @@ -1,102 +0,0 @@ -#ifndef HEADER_UTILS_H -#define HEADER_UTILS_H - - -#include -#include -#include -#include -#include -#include -#include -#include - - - -class CharBuffer { -public: - CharBuffer(); - CharBuffer(size_t); - CharBuffer(const CharBuffer&)=delete; - CharBuffer(CharBuffer&&)=delete; - - const char& operator()(size_t) const; - const size_t& size() const; - const size_t& buffer_size() const; - - void append(char s); - void reset(); // set buffer_idx_ to zero - void reset(size_t buffer_size); // set buffer_idx_ to zero - - // remember that it is the caller's responsibility to not - // dereference raw pointer after the CharBuffer instance is - // destructed - const char* data() const; - -private: - size_t buffer_size_; - size_t buffer_idx_; - std::unique_ptr buffer_; -}; - - - -class StringRecord { -public: - // input only delimiters - StringRecord()=delete; - StringRecord(const char); - StringRecord(const char, const size_t); - StringRecord(const char, const char*); - - void update_str(const char* s); - - const char* data() const; - void reset(); - size_t size(); - bool next_field(); - - -private: - size_t size_ { 0 }; - size_t idx_ { 0 }; - const char* str_; - const char delim_; - - CharBuffer buf_; - std::function is_delim_; - bool char_is_delim_(char) const; - -}; - - -class BufferedRead { -public: - BufferedRead()=delete; - BufferedRead(const BufferedRead&)=delete; - BufferedRead(BufferedRead&&)=delete; - BufferedRead& operator=(const BufferedRead&)=delete; - - BufferedRead(char* filename, size_t buff_size); - ~BufferedRead(); - - size_t get_line(CharBuffer& line_buf); - // size_t get_line(std::unique_ptr line_buf); - char get_char(); - void seek(size_t n); - size_t tell(); - void reset(); - -private: - const char* filename_; - const size_t buff_size_; - - FILE* fid_; - std::unique_ptr buffer_; - - size_t buffer_pos_ { 0 }; - - size_t update_buffer_(); - -}; -#endif diff --git a/src/HaplotypeDataRecord.cpp b/src/HaplotypeDataRecord.cpp deleted file mode 100644 index 27ab865..0000000 --- a/src/HaplotypeDataRecord.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// -// -// By: Robert Vogel -// Affiliation: Palmer Lab at UCSD -// Date: 2025-01-09 - -// -// -// Acknowledgment -// -// Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. -// -#include "HaplotypeVcfParser.h" - - -// Default constructor -HaplotypeDataRecord::HaplotypeDataRecord(size_t n_samples, size_t k_founders) - : n_samples_(n_samples), - k_founders_(k_founders), - samples_(n_samples_ > 0 && k_founders_ > 0 - ? std::make_unique(n_samples_, k_founders_) : nullptr) { - - if (n_samples_ <= 0 || k_founders_ <= 0) - throw std::runtime_error("Data must have more than zero samples and founders"); - - }; - - -// access elements -const std::string& HaplotypeDataRecord::chrom() const { return chrom_; }; -const long HaplotypeDataRecord::pos() const { return pos_; }; -const std::string& HaplotypeDataRecord::id() const { return id_; }; -const char HaplotypeDataRecord::ref() const { return ref_; }; -const char HaplotypeDataRecord::alt() const { return alt_; }; -const std::string& HaplotypeDataRecord::qual() const { return qual_; }; -const std::string& HaplotypeDataRecord::filter() const { return filter_; }; -const std::string& HaplotypeDataRecord::info() const { return info_; }; -const std::string& HaplotypeDataRecord::format() const { return format_; }; - - -void HaplotypeDataRecord::parse_vcf_line(const char* vcf_line) { - - - if (std::isspace(vcf_line[0])) - throw std::runtime_error("No line can begin with spaces"); - - - // A field of a vcf line record is a single string separated from others - // by white space. - // A sample field is a field with the data for a single sample. Sample - // fields have numerous : delimited records - // The counts of the k founders in any one sample field is a comma delimited - // element of a sample field record. - size_t hap_idx { 0 }; // index with hap counts - bool hap_found { false }; // determine whether hap dose is in dataset - size_t sample_idx { 0 }; // sample index - size_t founder_idx { 0 }; - - line_parse_.update_str(vcf_line); - - for (int field_idx = 1; line_parse_.next_field(); field_idx++) { - - if (field_idx == 1) - chrom_ = line_parse_.data(); - else if (field_idx == 2) - pos_ = std::atoi(line_parse_.data()); - else if (field_idx == 3) - id_ = line_parse_.data(); - else if (field_idx == 4) - ref_ = line_parse_.data()[0]; - else if (field_idx == 5) - alt_ = line_parse_.data()[0]; - else if (field_idx == 6) - qual_ = line_parse_.data(); - else if (field_idx == 7) - filter_ = line_parse_.data(); - else if (field_idx == 8) - info_ = line_parse_.data(); - else if (field_idx == 9) { - format_ = line_parse_.data(); - - // verify in the format field that haplotype dose (HD) - // is included in the data. Find the index (hap_idx) - // for which haplotype count data is found in a sample field - // record - field_parse_.update_str(line_parse_.data()); - - hap_found = false; - for (size_t i = 0; field_parse_.next_field(); i++) { - if (std::strcmp(field_parse_.data(), HAP_CODE) == 0 ) { - hap_found = true; - hap_idx = i; - break; - } - hap_idx++; - } - - if (!hap_found) - throw std::runtime_error("Haplotype counts are not specified"); - - } else if (field_idx > 9 && samples_) { - - if (sample_idx < 0 || sample_idx >= n_samples_) - throw std::out_of_range("Index is out of matrix range."); - - field_parse_.update_str(line_parse_.data()); - for (size_t j = 0; field_parse_.next_field(); j++) - if (j == hap_idx) - break; - - hap_parse_.update_str(field_parse_.data()); - - // decompose haplotype counts to respective founders - for (founder_idx = 0; hap_parse_.next_field(); founder_idx++) - (*samples_)(sample_idx, founder_idx) = std::atof(hap_parse_.data()); - - - if (founder_idx != k_founders_) - throw std::runtime_error("Number of founders found for sample is incorrect"); - - sample_idx++; - } - } - - if (sample_idx != n_samples_) - throw std::runtime_error("Number of samples found is not equal to that expected."); - -} - - -const double& HaplotypeDataRecord::operator()(size_t i, size_t j) const { - return (*samples_)(i, j); -} - -std::array HaplotypeDataRecord::dims() const { - return (*samples_).dims(); -} diff --git a/src/HaplotypeVcfParser.cpp b/src/HaplotypeVcfParser.cpp deleted file mode 100644 index affda4a..0000000 --- a/src/HaplotypeVcfParser.cpp +++ /dev/null @@ -1,220 +0,0 @@ -// -// By: Robert Vogel -// Affiliation: Palmer Lab at UCSD -// Date: 2025-01-09 -// -// Input argument -// filename: vcf with haplotpye -// -// -// Acknowledgment -// -// Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. - -#include "HaplotypeVcfParser.h" - -const static size_t DEFAULT_BUFFER_SIZE { 100000 }; - - - - -HaplotypeVcfParser::HaplotypeVcfParser(char* filename) - : fname_(filename), - file_io_(BufferedRead(filename, DEFAULT_BUFFER_SIZE)) { - - // get number of characters in data record for line buffer size - size_t nchar { get_line_num_char_() }; - - if (nchar == 0) - throw std::runtime_error("No data to read"); - - // make buffer 10% larger then the number of characters read. - line_buffer_size_ = static_cast(nchar * 1.1); - line_buffer_.reset(line_buffer_size_); - - set_params_(); - pos_(fpos_record_one_); -}; - - -HaplotypeVcfParser::HaplotypeVcfParser(char* filename, size_t buff_size) - : fname_(filename), - file_io_(BufferedRead(filename, buff_size)) { - - // get number of characters in data record for line buffer size - size_t nchar { get_line_num_char_() }; - - if (nchar == 0) - throw std::runtime_error("No data to read"); - - // make buffer 10% larger then the number of characters read. - line_buffer_size_ = static_cast(nchar * 1.1); - line_buffer_.reset(line_buffer_size_); - - set_params_(); - pos_(fpos_record_one_); -}; - - -// HaplotypeVcfParser::HaplotypeVcfParser(std::string filename) -// : fname_(filename), -// fid_(filename) { -// -// if (fid_.bad()) -// throw std::runtime_error("File Access error"); -// else if (fid_.eof()) -// throw std::runtime_error("File is empty"); -// -// // get number of characters in data record for line buffer size -// size_t nchar { get_line_num_char_() }; -// -// -// // make buffer 10% larger then the number of characters read. -// line_buffer_size_ = static_cast(nchar * 1.1); -// line_buffer_ = new char[line_buffer_size_]; -// line_buffer_[0] = '\0'; -// -// pos_(std::ios_base::beg); -// set_params_(); -// }; - - -//HaplotypeVcfParser::~HaplotypeVcfParser() { -// if(fid_) -// fclose(fid_); -// // free(line_buffer_); -//} - - -size_t HaplotypeVcfParser::get_line_num_char_() { - - size_t char_count { 0 }; - size_t max_char_count { 0 }; - char c; - - while ((c = file_io_.get_char()) != '\0') { - char_count++; - - if (c == '\n' && char_count > max_char_count) { - max_char_count = char_count; - char_count = 0; - } - } - - return max_char_count; -} - - -size_t HaplotypeVcfParser::n_samples() const { return n_samples_; } - - -size_t HaplotypeVcfParser::k_founders() const { return k_founders_; } - - -void HaplotypeVcfParser::pos_(size_t n) { - file_io_.reset(); - if (n != 0) - file_io_.seek(n); -} - - -void HaplotypeVcfParser::set_params_() { - pos_(0); - - // skip meta data lines - - size_t n { 0 }; - size_t num_bytes { 0 }; - while ((n = file_io_.get_line(line_buffer_)) > 0) { - - // the +1 is because I don't write newline characters to buffer - num_bytes += line_buffer_.size()+1; - - if (line_buffer_(0) == META_PREFIX && line_buffer_(1) == META_PREFIX) - continue; - - break; - } - - // Store file position of first record - fpos_record_one_ = sizeof(line_buffer_(0)) * num_bytes; - - // if there is no header - if (line_buffer_(0) != META_PREFIX) - return; - - - if (std::isspace(line_buffer_(0))) - throw std::runtime_error("First element of VCF line must not be blank."); - - - // Get column number and sample number - StringRecord line_parser_ { SPACE_DELIM, line_buffer_.data() }; - StringRecord field_parser_ { MEASUREMENT_DELIM }; - StringRecord hap_parser_ { HAP_DELIM }; - - n_cols_ = 0; - n_samples_ = 0; - for (; line_parser_.next_field(); n_cols_++) { - - if (n_cols_ < NUM_VCF_FIELDS - && std::strcmp(line_parser_.data(), VCF_FIELD_NAMES[n_cols_]) != 0) - throw std::runtime_error("File doesn't follow vcf header specification"); - - if (n_cols_ >= NUM_VCF_FIELDS) - n_samples_++; - - } - - - // get k founders from record - if ((n = file_io_.get_line(line_buffer_)) == 0) - throw std::runtime_error("End of file"); - - line_parser_.update_str(line_buffer_.data()); - size_t hap_idx { 0 }; - for (int i = 0; line_parser_.next_field(); i++) { - - if (i == NUM_VCF_FIELDS-1) { - field_parser_.update_str(line_parser_.data()); - - for (;field_parser_.next_field(); hap_idx++) - if (std::strcmp(field_parser_.data(), HAP_CODE) == 0) - break; - - } else if(i == NUM_VCF_FIELDS) { - field_parser_.update_str(line_parser_.data()); - - for(int i = 0; field_parser_.next_field() && i < hap_idx; hap_idx++) - ; - - hap_parser_.update_str(field_parser_.data()); - for (;hap_parser_.next_field(); k_founders_++) - ; - - break; - } - - } - - if (k_founders_ == 0) - throw std::runtime_error("Parse error"); - -} - - -bool HaplotypeVcfParser::load_record(HaplotypeDataRecord& record) { - - size_t n { 0 }; - - if ((n = file_io_.get_line(line_buffer_)) == 0) - return false; - - record.parse_vcf_line(line_buffer_.data()); - - return true; -} - - diff --git a/src/main.cpp b/src/main.cpp index 3b81cb4..801385f 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,10 +21,12 @@ // reviewed by Claude Sonnet, the AI assistant from Anthropic // (Jan 2025), with minor recommendations incorporated. // +#include #include -#include -#include "HaplotypeVcfParser.h" - +// #include +#include +#include +#include size_t MARKER_PRINT_INTERVAL { 1000 }; @@ -33,139 +35,173 @@ char HELP_SHORT_FLAG[] { "-h" }; int main(int argc, char* argv[]) { - - if (argc != 2 && argc != 3) - throw std::runtime_error("Must specify vcf"); - - if (argc == 2 - && (strcmp(argv[1], HELP_SHORT_FLAG) == 0 - || strcmp(argv[1], HELP_LONG_FLAG) == 0)) { - printf("hgrm - Compute GRM from expected haplotype counts.\n" - "Usage\n" - "\n" - " hgrm []\n" - "\n" - "Options\n" - " output_matrix_filename Filename to print covariance matrix\n" - "\n" - "Description\n" - " A program to compute a genetic relationship matrix from a vcf\n" - " with expected haplotype counts record per sample per locus.\n"); - - - return 0; + if (argc != 2 && argc != 4) { + fprintf(stderr, "Incorrect input, see --help for correct usage.\n"); + exit(EXIT_FAILURE); } - char* filename_input { argv[1] }; - char* filename_output { nullptr }; - - if(argc == 3) - filename_output = argv[2]; - - - const std::chrono::time_point timer - { std::chrono::steady_clock::now() }; - - - fprintf(stdout, "Allocating memory\n"); - - // open VCF file and parse meta data and header - HaplotypeVcfParser vcf_data { filename_input, 100000 }; - - - // instantiate matrices to hold calculations - Matrix covariance { vcf_data.n_samples(), vcf_data.n_samples() }; - - // instantiate record object - HaplotypeDataRecord record { vcf_data.n_samples(), vcf_data.k_founders() }; - - // analyze each line, i.e. position, in the VCF - size_t m_markers { 1 }; - - double sum { 0 }; - const double* rowi { nullptr }; - const double* rowj { nullptr }; - double* rowi_cov { nullptr }; - const size_t k_founders { vcf_data.k_founders() }; - const size_t n_samples { vcf_data.n_samples() }; - - std::chrono::steady_clock::duration delta_t - { std::chrono::steady_clock::now() - timer }; - - fprintf(stdout, "Computing matrix, elapsed time %lld second(s)\n", - std::chrono::duration_cast(delta_t).count()); - - while(vcf_data.load_record(record)) { - - // for each founder, compute first and second moments - for (size_t i = 0; i < n_samples; i++) { - - rowi = &record(i, 0); - rowi_cov = &covariance(i, 0); - - for (size_t j = i; j < n_samples; j++) { - - rowj = &record(j,0); - sum = 0; - - for (int k = 0; k < k_founders; k++) - sum += rowi[k] * rowj[k]; - - rowi_cov[j] += sum; - } - } - - if (m_markers % MARKER_PRINT_INTERVAL == 0) { - delta_t = std::chrono::steady_clock::now() - timer; - - fprintf(stdout, "Completed %zu marker loci, elapsed time %lld second(s)\n", - m_markers, - std::chrono::duration_cast(delta_t).count()); - } - - m_markers++; - + argparse::ArgParser parser { + "hgrm: Haplotype Genetic Relationship Matrix", + "This program computes the haplotype genetic relationship matrix" + "from the expected haplotype counts per locus per sample and stored" + "as a text file in the variant call format (VCF)." + }; + + parser.add_arg("--sample_names", + argparse::ArgType::STRING, + "The path and name of the file containing sample names to be" + " included in computing the relationship matrix. The file must" + " include a single sample filename, and if necessary file system" + " path, per line."); + parser.add_arg("-o", + argparse::ArgType::STRING, + "the path and filename that the resulting haplotype genetic" + "relationship matrix is printed."); + parser.add_arg("vcf", + argparse::ArgType::STRING, + "the path and filename of the vcf in which the hgrm is computed."); + + if (parser.parse_args(argc, argv) != argparse::ArgStatus::SUCCESS) { + fprintf(stderr, "Error: couldn't parse command line args, exiting\n"); + exit(EXIT_FAILURE); } + std::optional tmp {}; + if((tmp = parser.get("vcf")) == std::nullopt) { + fprintf(stderr, "Error retrieving vcf name"); + exit(EXIT_FAILURE); + } + std::string vcf_fname { tmp.value() }; - FILE* fout = stdout; - - if (argc == 3 && filename_output != nullptr) { - - if ((fout = fopen(filename_output, "w")) == nullptr) - throw std::runtime_error("Error in opening file for writing."); - - delta_t = std::chrono::steady_clock::now() - timer; - fprintf(stdout, "Writing results to file %s, elapsed time %lld second(s)\n", - filename_output, - std::chrono::duration_cast(delta_t).count()); - - } else if (argc == 3 && filename_output == nullptr) - throw std::runtime_error("Output filename is not specified"); - - - size_t i { 0 }; - size_t j { 0 }; - for (i = 0; i < n_samples; i++) { - - for (j = 0; j < n_samples-1; j++) { - if (j < i) - fprintf(fout, "%0.5f,", covariance(j,i)); - else - fprintf(fout, "%0.5f,", covariance(i,j)); + if ((tmp = parser.get("o")) == std::nullopt) { + fprintf(stderr, "Error retrieving output name"); + exit(EXIT_FAILURE); + } + std::string out_fname { tmp.value() }; - } + if (out_fname.size() == 0) + out_fname = vcf_fname + ".mat"; - fprintf(fout,"%0.5f\n", covariance(i, j)); + std::string samp_fname {}; + if ((tmp = parser.get("sample_names")) == std::nullopt) { + fprintf(stderr, "Error retrieving sample_names file.\n"); + exit(EXIT_FAILURE); } + samp_fname = tmp.value(); - fclose(fout); + fprintf(stdout, "BCF/VCF file name: %s\n", vcf_fname.c_str()); + if (samp_fname.size() == 0) + fprintf(stdout, "Sample file: None, use all samples\n"); + else + fprintf(stdout, "Sample file: %s\n", samp_fname.c_str()); + fprintf(stdout, "Output matrix file: %s\n", out_fname.c_str()); - delta_t = std::chrono::steady_clock::now() - timer; + // const std::chrono::time_point timer; + // { std::chrono::steady_clock::now() }; + +// HaplotypeVcfParser vcf_data { filename_input, 100000 }; - fprintf(stdout, "Done, elapsed time %lld second(s)\n", - std::chrono::duration_cast(delta_t).count()); + fprintf(stdout, "Allocating memory\n"); + // instantiate matrices to hold calculations +// Matrix covariance { vcf_data.n_samples(), vcf_data.n_samples() }; + +// +// // open VCF file and parse meta data and header +// HaplotypeVcfParser vcf_data { filename_input, 100000 }; +// +// +// // instantiate record object +// HaplotypeDataRecord record { vcf_data.n_samples(), vcf_data.k_founders() }; +// +// // analyze each line, i.e. position, in the VCF +// size_t m_markers { 1 }; +// +// double sum { 0 }; +// const double* rowi { nullptr }; +// const double* rowj { nullptr }; +// double* rowi_cov { nullptr }; +// const size_t k_founders { vcf_data.k_founders() }; +// const size_t n_samples { vcf_data.n_samples() }; +// +// std::chrono::steady_clock::duration delta_t +// { std::chrono::steady_clock::now() - timer }; +// +// fprintf(stdout, "Computing matrix, elapsed time %lld second(s)\n", +// std::chrono::duration_cast(delta_t).count()); +// +// while(vcf_data.load_record(record)) { +// +// // for each founder, compute first and second moments +// for (size_t i = 0; i < n_samples; i++) { +// +// rowi = &record(i, 0); +// rowi_cov = &covariance(i, 0); +// +// for (size_t j = i; j < n_samples; j++) { +// +// rowj = &record(j,0); +// sum = 0; +// +// for (int k = 0; k < k_founders; k++) +// sum += rowi[k] * rowj[k]; +// +// rowi_cov[j] += sum; +// } +// } +// +// if (m_markers % MARKER_PRINT_INTERVAL == 0) { +// delta_t = std::chrono::steady_clock::now() - timer; +// +// fprintf(stdout, "Completed %zu marker loci, elapsed time %lld second(s)\n", +// m_markers, +// std::chrono::duration_cast(delta_t).count()); +// } +// +// m_markers++; +// +// } +// +// +// FILE* fout = stdout; +// +// if (argc == 3 && filename_output != nullptr) { +// +// if ((fout = fopen(filename_output, "w")) == nullptr) +// throw std::runtime_error("Error in opening file for writing."); +// +// delta_t = std::chrono::steady_clock::now() - timer; +// fprintf(stdout, "Writing results to file %s, elapsed time %lld second(s)\n", +// filename_output, +// std::chrono::duration_cast(delta_t).count()); +// +// } else if (argc == 3 && filename_output == nullptr) +// throw std::runtime_error("Output filename is not specified"); +// +// +// size_t i { 0 }; +// size_t j { 0 }; +// for (i = 0; i < n_samples; i++) { +// +// for (j = 0; j < n_samples-1; j++) { +// if (j < i) +// fprintf(fout, "%0.5f,", covariance(j,i)); +// else +// fprintf(fout, "%0.5f,", covariance(i,j)); +// +// } +// +// fprintf(fout,"%0.5f\n", covariance(i, j)); +// } +// +// fclose(fout); +// +// +// delta_t = std::chrono::steady_clock::now() - timer; +// +// fprintf(stdout, "Done, elapsed time %lld second(s)\n", +// std::chrono::duration_cast(delta_t).count()); +// return 0; } diff --git a/src/Matrix.cpp b/src/matrix.cpp similarity index 95% rename from src/Matrix.cpp rename to src/matrix.cpp index 32d7df0..3c072a7 100644 --- a/src/Matrix.cpp +++ b/src/matrix.cpp @@ -13,14 +13,14 @@ // // -#include "Matrix.h" +#include // default constructor Matrix::Matrix(size_t nrow, size_t mcol) : nrow_(nrow), mcol_(mcol), data_(nrow_ > 0 && mcol_ > 0 ? std::make_unique(size()) : nullptr) { - if (nrow_ <= 0 || mcol_ <= 0) + if (nrow_ == 0 || mcol_ == 0) throw std::runtime_error("Matrix must have minimum size of 1"); // set default values to zero @@ -40,7 +40,7 @@ Matrix::Matrix(const Matrix& other) data_[i] = other.data_[i]; } - +// TODO: check this. Matrix::Matrix(Matrix&& other) : nrow_(other.nrow_), mcol_(other.mcol_), data_(std::move(other.data_)) {}; diff --git a/src/parse_hts.cpp b/src/parse_hts.cpp new file mode 100644 index 0000000..1a3065e --- /dev/null +++ b/src/parse_hts.cpp @@ -0,0 +1,235 @@ +// +// By: Robert Vogel +// Affiliation: Palmer Lab at UCSD +// Date: 2025-01-09 +// +// Input argument +// filename: vcf with haplotpye +// +// +// Acknowledgment +// +// Code design and original version completed by Robert Vogel, +// reviewed by Claude Sonnnet, the AI assistant from Anthropic +// (Jan 2025), with minor recommendations incorporated. + +#include +#include + +// const static size_t DEFAULT_BUFFER_SIZE { 100000 }; + + +ParseHtsVariantFile::ParseHtsVariantFile(const char *variant_fname, + const char *sample_fname) + : fname_(variant_fname), + fid_(htslib::hts_open(variant_fname, "r")), + hdr_(htslib::bcf_hdr_read(fid_)) { + + int status { 0 }; + // Subset samples with those found in the file sample_fname + if (!sample_fname || *sample_fname == '\0') + fprintf(stdout, "No file with sample names detected, computing" + "hGRM over all samples.\n"); + else + status = htslib::bcf_hdr_set_samples(hdr_, sample_fname, 1); + + if (status < 0) { + fprintf(stderr, "Error: Couldn't read sample file\n"); + exit(EXIT_FAILURE); + } else if (status > 0) { + fprintf(stderr, "Error: A subset of samples in sample file are not" + " found in the VCF,BCF, or VCF.GZ file.\n"); + exit(EXIT_FAILURE); + } + + + // get number of characters in data record for line buffer size +}; + +ParseHtsVariantFile::~ParseHtsVariantFile() { + if (fid_) + htslib::hts_close(fid_); + if (hdr_) + htslib::bcf_hdr_destroy(hdr_); + +} + +// ParseHtsVariantFile::ParseHtsVariantFile(char* filename, size_t buff_size) +// : fname_(filename), +// file_io_(BufferedRead(filename, buff_size)) { +// +// // get number of characters in data record for line buffer size +// size_t nchar { get_line_num_char_() }; +// +// if (nchar == 0) +// throw std::runtime_error("No data to read"); +// +// // make buffer 10% larger then the number of characters read. +// line_buffer_size_ = static_cast(nchar * 1.1); +// line_buffer_.reset(line_buffer_size_); +// +// set_params_(); +// pos_(fpos_record_one_); +// }; +// +// +// // ParseHtsVariantFile::ParseHtsVariantFile(std::string filename) +// // : fname_(filename), +// // fid_(filename) { +// // +// // if (fid_.bad()) +// // throw std::runtime_error("File Access error"); +// // else if (fid_.eof()) +// // throw std::runtime_error("File is empty"); +// // +// // // get number of characters in data record for line buffer size +// // size_t nchar { get_line_num_char_() }; +// // +// // +// // // make buffer 10% larger then the number of characters read. +// // line_buffer_size_ = static_cast(nchar * 1.1); +// // line_buffer_ = new char[line_buffer_size_]; +// // line_buffer_[0] = '\0'; +// // +// // pos_(std::ios_base::beg); +// // set_params_(); +// // }; +// +// +// //ParseHtsVariantFile::~ParseHtsVariantFile() { +// // if(fid_) +// // fclose(fid_); +// // // free(line_buffer_); +// //} +// +// +// size_t ParseHtsVariantFile::get_line_num_char_() { +// +// size_t char_count { 0 }; +// size_t max_char_count { 0 }; +// char c; +// +// while ((c = file_io_.get_char()) != '\0') { +// char_count++; +// +// if (c == '\n' && char_count > max_char_count) { +// max_char_count = char_count; +// char_count = 0; +// } +// } +// +// return max_char_count; +// } +// +// +// size_t ParseHtsVariantFile::n_samples() const { return n_samples_; } +// +// +// size_t ParseHtsVariantFile::k_founders() const { return k_founders_; } +// +// +// void ParseHtsVariantFile::pos_(size_t n) { +// file_io_.reset(); +// if (n != 0) +// file_io_.seek(n); +// } +// +// +// void ParseHtsVariantFile::set_params_() { +// pos_(0); +// +// // skip meta data lines +// +// size_t n { 0 }; +// size_t num_bytes { 0 }; +// while ((n = file_io_.get_line(line_buffer_)) > 0) { +// +// // the +1 is because I don't write newline characters to buffer +// num_bytes += line_buffer_.size()+1; +// +// if (line_buffer_(0) == META_PREFIX && line_buffer_(1) == META_PREFIX) +// continue; +// +// break; +// } +// +// // Store file position of first record +// fpos_record_one_ = sizeof(line_buffer_(0)) * num_bytes; +// +// // if there is no header +// if (line_buffer_(0) != META_PREFIX) +// return; +// +// +// if (std::isspace(line_buffer_(0))) +// throw std::runtime_error("First element of VCF line must not be blank."); +// +// +// // Get column number and sample number +// StringRecord line_parser_ { SPACE_DELIM, line_buffer_.data() }; +// StringRecord field_parser_ { MEASUREMENT_DELIM }; +// StringRecord hap_parser_ { HAP_DELIM }; +// +// n_cols_ = 0; +// n_samples_ = 0; +// for (; line_parser_.next_field(); n_cols_++) { +// +// if (n_cols_ < NUM_VCF_FIELDS +// && std::strcmp(line_parser_.data(), VCF_FIELD_NAMES[n_cols_]) != 0) +// throw std::runtime_error("File doesn't follow vcf header specification"); +// +// if (n_cols_ >= NUM_VCF_FIELDS) +// n_samples_++; +// +// } +// +// +// // get k founders from record +// if ((n = file_io_.get_line(line_buffer_)) == 0) +// throw std::runtime_error("End of file"); +// +// line_parser_.update_str(line_buffer_.data()); +// size_t hap_idx { 0 }; +// for (int i = 0; line_parser_.next_field(); i++) { +// +// if (i == NUM_VCF_FIELDS-1) { +// field_parser_.update_str(line_parser_.data()); +// +// for (;field_parser_.next_field(); hap_idx++) +// if (std::strcmp(field_parser_.data(), HAP_CODE) == 0) +// break; +// +// } else if(i == NUM_VCF_FIELDS) { +// field_parser_.update_str(line_parser_.data()); +// +// for(int i = 0; field_parser_.next_field() && i < hap_idx; hap_idx++) +// ; +// +// hap_parser_.update_str(field_parser_.data()); +// for (;hap_parser_.next_field(); k_founders_++) +// ; +// +// break; +// } +// +// } +// +// if (k_founders_ == 0) +// throw std::runtime_error("Parse error"); +// +// } +// +// +// bool ParseHtsVariantFile::load_record(HaplotypeDataRecord& record) { +// +// size_t n { 0 }; +// +// if ((n = file_io_.get_line(line_buffer_)) == 0) +// return false; +// +// record.parse_vcf_line(line_buffer_.data()); +// +// return true; +// } + + diff --git a/src/utils.cpp b/src/utils.cpp deleted file mode 100644 index 46aebc2..0000000 --- a/src/utils.cpp +++ /dev/null @@ -1,301 +0,0 @@ -#include "../include/utils.h" - -const static size_t DEFAULT_BUFFER_SIZE { 1000 }; - - - - -CharBuffer::CharBuffer() - : buffer_size_(0), - buffer_idx_(0), - buffer_(nullptr) {} - - -CharBuffer::CharBuffer(size_t buffer_size) - : buffer_size_(buffer_size), - buffer_idx_(0), - buffer_(buffer_size > 0 ? std::make_unique(buffer_size+1) : nullptr) { - - if (buffer_ == nullptr) - throw std::runtime_error("CharBuffer needs to have length > 0"); - - buffer_[buffer_idx_] = '\0'; - buffer_[buffer_size_] = '\0'; -} - - -const char& CharBuffer::operator()(size_t idx) const { - if (idx >= size() || idx >= buffer_idx_) - throw std::out_of_range("index too large for buffer."); - - if (!buffer_) - throw std::runtime_error("error"); - return buffer_[idx]; -} - - -const size_t& CharBuffer::buffer_size() const { return buffer_size_; } -const size_t& CharBuffer::size() const { return buffer_idx_; } - - -void CharBuffer::append(char s) { - if (buffer_ == nullptr) - throw std::runtime_error("No data stored in buffer."); - - if (buffer_idx_ >= buffer_size_) - throw std::out_of_range("CharBuffer full"); - - buffer_[buffer_idx_++] = s; - buffer_[buffer_idx_] = '\0'; -} - - -void CharBuffer::reset(size_t buffer_size) { - buffer_size_ = buffer_size; - buffer_ = std::make_unique(buffer_size_); - buffer_[0] = '\0'; - buffer_[buffer_size_] = '\0'; - buffer_idx_ = 0; -} - - -void CharBuffer::reset() { - if (buffer_ == nullptr) - throw std::runtime_error("No data stored in buffer."); - - buffer_idx_ = 0; - buffer_[buffer_idx_] = '\0'; -} - - -const char* CharBuffer::data() const { - if (buffer_ == nullptr) - throw std::runtime_error("No data stored in buffer."); - - return buffer_.get(); -} - - - -StringRecord::StringRecord(const char delim) - : str_(nullptr), - delim_(delim), - buf_(DEFAULT_BUFFER_SIZE) { - - if (std::isspace(delim)) - is_delim_ = [](char c){ return std::isspace(c); }; - else - is_delim_ = [this](char c){ return char_is_delim_(c); }; -}; - -StringRecord::StringRecord(const char delim, const size_t buf_size) - : str_(nullptr), - delim_(delim), - buf_(buf_size) { - - if (std::isspace(delim)) - is_delim_ = [](char c){ return std::isspace(c); }; - else - is_delim_ = [this](char c){ return char_is_delim_(c); }; -}; - -StringRecord::StringRecord(const char delim, const char* str) - : str_(str), - delim_(delim), - buf_(DEFAULT_BUFFER_SIZE) { - - if (std::isspace(delim)) - is_delim_ = [](char c){ return std::isspace(c); }; - else - is_delim_ = [this](char c){ return char_is_delim_(c); }; - - for (;str_[size_] != '\0'; size_++) - ; -}; - - -bool StringRecord::char_is_delim_(char c) const { - if (c == delim_) - return true; - return false; -} - -// size including null character -void StringRecord::update_str(const char* str) { - reset(); - str_ = str; - - for (;str_[size_] != '\0'; size_++) - ; - - return ; -} - - -void StringRecord::reset() { - idx_ = 0; - size_ = 0; - str_ = nullptr; - buf_.reset(); -}; - - -size_t StringRecord::size() { - if (!str_) - return 0; - - return size_; -} - - -const char* StringRecord::data() const { - return buf_.data(); -} - - -bool StringRecord::next_field() { - if (!str_ || idx_ == size()) - return false; - - // eliminate preceeding white space - for(; idx_ < size(); idx_++) - if (!is_delim_(str_[idx_])) - break; - - buf_.reset(); - for (; idx_ < size(); idx_++) { - if (!is_delim_(str_[idx_])) - buf_.append(str_[idx_]); - - if (is_delim_(str_[idx_]) && !is_delim_(str_[idx_-1])) { - idx_++; - break; - } - } - - return true; -} - - -// bool StringRecord::is_delim_(char s) { -// return s == delim_; -// } - - -// buff_size is the count of characters to load -BufferedRead::BufferedRead(char* filename, size_t buff_size) - : filename_(filename), - buff_size_(buff_size), - fid_(std::fopen(filename, "r")), - buffer_(buff_size > 0 ? std::make_unique(buff_size+1) : nullptr) { - - if (!fid_) - throw std::runtime_error("File Access error"); - - if (std::feof(fid_)) - throw std::runtime_error("File is empty"); - - if (buffer_ == nullptr) - throw std::runtime_error("Buffer wasn't properly set"); - - buffer_pos_ = 0; - buffer_[0] = '\0'; -} - - -BufferedRead::~BufferedRead() { - if (fid_) - fclose(fid_); -} - -// Note: -// Return count == 0 indicates that we have reached the end of file. -// -size_t BufferedRead::get_line(CharBuffer& line_buffer) { - - line_buffer.reset(); - - size_t count { 0 }; - size_t buff_length { 1 }; - - if (buffer_[buffer_pos_] == '\0' - || buffer_pos_ == buff_size_) - buff_length = update_buffer_(); - - - if (buff_length == 0) - return count; - - while (buffer_[buffer_pos_] != '\n') { - - line_buffer.append(buffer_[buffer_pos_++]); - count++; - - if (buffer_[buffer_pos_] == '\0' || buffer_pos_ == buff_size_) - buff_length = update_buffer_(); - - if (buff_length == 0) - break; - } - - if (buffer_[buffer_pos_] == '\n') - buffer_pos_++; - - return count; -} - - -char BufferedRead::get_char() { - size_t buff_length { 1 }; - - - if (buffer_[buffer_pos_] == '\0') - buff_length = update_buffer_(); - - // when all entries of a file are read, detected by - // buff_length == 0, return the null character - if (buff_length == 0) - return '\0'; - - return buffer_[buffer_pos_++]; -} - - -size_t BufferedRead::update_buffer_() { - - if (std::ferror(fid_)) - throw std::runtime_error("File read error"); - - // note, if end of file, then we set n = 0; - size_t n { 0 }; - if (!std::feof(fid_)) - n = fread(buffer_.get(), sizeof(buffer_[0]), buff_size_, fid_); - - buffer_pos_ = 0; - buffer_[n] = '\0'; - - return n; -} - - -void BufferedRead::seek(size_t n) { - - int c = 0; - if ((c = std::fseek(fid_, n, SEEK_SET)) != 0) - throw std::runtime_error("Failed to relocate file stream to position."); - -} - - -size_t BufferedRead::tell() { - return std::ftell(fid_); -} - - -void BufferedRead::reset() { - buffer_pos_ = 0; - buffer_[buffer_pos_] = '\0'; - seek(0); -} - - diff --git a/tests/geno_test_data.bcf b/tests/geno_test_data.bcf new file mode 100644 index 0000000000000000000000000000000000000000..6c77e17c9eb369433d9ea22d1847bed1f7bb843e GIT binary patch literal 2257 zcmV;?2rl;@iwFb&00000{{{d;LjnMm0vhO0`uNPmS@Zd#4fq@`cdGARWD zXcQP>?d~d*oDfUK&TOX@ziWSR$4OZmw(djQ#X*)he&6SO#Mhm0clYa?ot;J_WgJqe zR@C^no$#UEa`zgIa5$RxX8yZj*PjMa)a-(e70V5i!ru=#CwR+%(j=!^L$co-c0c>> zd%QUuhm-Q2J={l~$*k8L7pny*ztxNs~9prIm^MB^!Wq?#OfheWFKD5N0=?jcEK zf#ytX4JD@nI7td+vDqpw47H*a(}iUUC$pPiUQ+ev8yv5~iHyBhr@01gNWTF44Us;e zKW36gW-^jXjkPl*%avTv1>?-D5lDf~Bu{Lw+}2g)eYx#D-FTnAI+y8p{pH5@rID3C zcANO)HOoamcXF95x9I+Ai(}>qbMgC&3T2Jl%g7^ zh5IbP_C^-VjC^>`dD<7Rd{1I23|p3#ai+YJ=Qj)M2S1^P5?w4b7~D*pht)OcYNs8` z1Tb+Cr$$Py@2S8$xRoxBAA03VA81ONa4 z009360763o03HVImupN^R}g@&3$7Gf&0@BYS|1c2wU(l#h_d&bk+&M5C{ztaMG6KL z1bJI|N(dozLk&`!S|x}NY+7T9F}50_P>l~%tQcap#1e{>XaP|Wq>`SQ%gWWRrhlaV z;3Q}7?A$qznfqnVnE*+WKGaCmCRj<*C<%LZ))+}6jg=(pG4`JJ{yIsg(Z3O+H|R#` zB`F(pR?5R#pLw`c){WL%I}R+=Y4sm`In|&WBdl#4H5QxhHMdAZ798B|W55NSD>L9! z(gYdh{qY%K+u%g*RFTz#%?y|Hed&1uIc@oT=8;v@S;lcUMYny$E1e>)=&fW^UnU>& zEgRa?;fX(}4>H<=+Fx}GjHu%QDh%HpZ5cyv!Z4sLQ5G0_ES$rKFzh_^9S95`4YYX) z46huRFF(Ohne$&5s?lhv^8&x)iR3_Uus1j!O&~e8)qsKH$jS3Cd>(pg{qt-DhA0+Q zCv#l>a_tw~jG-Ape46E#h~e3(uLg$rd{r=$|1hJHJHTD2CGkJiX0g+`J*CGQRkD~cd_!Ljpf zYukDX;El8mEHQcuWk0{;_t_Eub)^6M@I95hg*BANQ>Ja4rsd9+s|i^V06tK^ifqnY zEMrshHi5t}wW--qPlUI>in1iRc$@hy7cLP|R{AGiZW)!i_H;jm(u zuClU4ewmg%r;1ZwAsqoqKu-7oE^^uH3^a&TXP#CPwdI(K}*EzqjKMdz876&_g7r zYuAxJwW@T`UXc+!4C-|4C~oRogBWH&H>_WO8#)i&7U%yTgxl==9(eHh0i2!MDZY0% zbyHva2!_Lp;g1$EG#|=yItjz=726C$7=F_ff*2MhIQR$*<86|k&W08+LX=t} z6ycNq=1o@Y5DA0Ma&TKiapGUyB%C59k6xE@eII743xJKHnbD=5#|?Q5R@tKZM3?BGKs zL)WhXytRKZA|t!Hy7IOe%gDjj{#oMU7A)i_78-eyUwt?M?O9EC1$QrSq(BeUr*WI_ zWtix)nwk2lxZ_bf*xNCatr+)%$a<48tBOOrj5ki30eGwBWN%~z}OPe9c z*a`#ne~N2qwK#+?LB%-}e7(6Ewy=u=(JRnUetHDM;l*&4MGQj*?}jH&7_mh8>6W7^ z8y?(!*Hd7)f6i~14f`Qqwpb}Er3zt)&WjU}0whWj2yY2u<~r@}viS`4qHe%UN%~Vs zii~<~PBLMJ^0D(Ngc-^?C7;0&|0c4(c_kH*CcRn~=!27~k73{hhPVasomkdJnJqZB zi27Wc_nC;7NYpK8Mjg-b5e$bFLqFRWl_;?*GMK>7=zCq24V`PEutd3*;Ex!(6c#WG z*-*g{C4pg(aV0E2F_|1))D#M{!p2jXS{UOmE@mMdgSXUuF^D@N+Q=<+5zrgljEET! zH!NvzEFQFY;G&b5K{1nJK$g015Vc1o;btC-_Fmz>aP6H>xhRiis+660Rq0z?R5ko) zLNv##<&Cs9hS$)X5^wId=4XaFC)3vdf}z^omyO|9-i0p>LlKJq1;c*;b;^OQI1~T? fABzYC000000RIL6LPG)o8vp|U0000000000p#eAc literal 0 HcmV?d00001 diff --git a/tests/test.vcf b/tests/geno_test_data.vcf similarity index 100% rename from tests/test.vcf rename to tests/geno_test_data.vcf diff --git a/tests/geno_test_data.vcf.gz b/tests/geno_test_data.vcf.gz new file mode 100644 index 0000000000000000000000000000000000000000..43958ea07891aa72c219d8318500039e2cb706ec GIT binary patch literal 1984 zcmV;x2S4~9iwFb&00000{{{d;LjnL_0XGaK9vqKAuK|De-} zCe!6;5qz2qgE&l*?hs9?d1bkh!C@!_P?SY&pkl^gcKDfMCbMYXM7YC!GMFz$-C4a} zp$>Z8<)*?Q5{lXVToaiqFsahGpJNn8e-0iQ%ZVE^U_zik$Tj|`QKp;yWaGOxkc^qm zHYaz;CHWOlojfp6x2YC`zoC+iE2tb&RdF=#WJ?5%IcmbhCvLTFzEm`fx@x!Lzlbt7 zFi9?N5{>_@|4#*#Raz;LeXeSB{J8ae(&mqYP?fDe{~A+ zugt|x@wtRceA@5Q!|yhY_fP${%rI#c_Zyj^GOWOMq&6F?nShd2VoR7c z*$fSZR#l>FgsO`YzCQVE7a~>C@}kj7OYNVzcT^SC4QgGKlll$9c%Mq1pojaiwFb&00000 z{{{d;LjnLp1=W~KZ`()=$Isr+5-3B?i*%5KktXQ^i#A9K^w>i$d*1JV*x`R>B-ybo z$$MG^P^<^3;o~A-Md*O#Hb95TJM*9@o4oQmC`^v*s)&&& z60QP4_XHV-?N*(=OR?FWg#nNErA7;_1~W_pZw8{fim_zHlHn;?%fykhNhS@t7K>wh z6!w@oURRUR7h%`Qn<7ccR+0Q>I=U{~m?*%YkWE*hMM92D$uI@Mz5h6+j0P?D%K60=;c8-1m)THp<`xGANLwWIov#E#F9+Pdb97h*EWB80$SCC`3Gu)ow_OKS0qg=1R71oTn zw3?S=r^w_K(#^b_(>1EPj!QQsV5vxhOu#lT0h1}5>$Ehcd>jBYF{qYl`suvgY2gH&aNJ;}JmEhAIqbhb6FD==sB{B~wMbVcY2 zH!(9$FxiR>svNwIZA#a-UPopm>>?hE@bn*n!BQCokNrksJc#j#rY;b}?)+pf9Ql-ejUVcTx6tP-a4M(lyS^rdtJ`UQ+jumg@7`; zF+$ieY%BwVbVz`dA%_j^EHF(3vy6vGsl7!cGTqvd*+p;9CT}}LR9cHmQ(%`%Xw7Bb zmY-?$oYKv~8o+Esl119wkp+Ss7K4Ne6Xn*1%)`5At5dKfNVJ+4bjcp|HYO}%acp6y zC9tI-JDkq)g@ls=V-QTZuoi!%I?0#8x@5sP$)`w5lRRv1-?BD;%0j#9V!H>+mq(T+ zi=evD#=4NMLegchrZ9araN!~(U4**G`1nmTJO%~i7xeI2JeqidQr|uS_>F7`+(?t3 zOBcR%j8E{IZ{OBD-#W&h(X)n$(k^0+Rhvl>%KKF&%4T$Kly`|Fo + +int main(int argc, char *argv[]) { + testing::InitGoogleTest(&argc, argv) ; + return RUN_ALL_TESTS(); +} diff --git a/tests/test_haplotype_data_record.cpp b/tests/test_haplotype_data_record.cpp deleted file mode 100644 index 4f0ce29..0000000 --- a/tests/test_haplotype_data_record.cpp +++ /dev/null @@ -1,117 +0,0 @@ - - -#include "../include/HaplotypeVcfParser.h" -#include - - - - -TEST(TestConstructorAssignment, Constructor) { - - size_t num_vcf_columns { 13 }; - size_t num_samps { 4 }; - size_t num_founders { 3 }; - static constexpr int num_records { 3 }; - - - char vcf_record[] { "chr12 1 . A T Q1 F1 INFO1 GT:AB:HD 0/0:2:1,0,1 0/0:1:0,2,0 1/0:0:0,0,2 1/1:1:2,0,0\n" }; - - - // Test first record - HaplotypeDataRecord hap_record { num_samps, num_founders }; - - std::array dims { hap_record.dims() }; - - EXPECT_EQ(dims[0], num_samps); - EXPECT_EQ(dims[1], num_founders); - - hap_record.parse_vcf_line(vcf_record); - - EXPECT_EQ(hap_record(0,0), 1); - EXPECT_EQ(hap_record(1,0), 0); - EXPECT_EQ(hap_record(2,0), 0); - EXPECT_EQ(hap_record(3,0), 2); - - EXPECT_EQ(hap_record(0,1), 0); - EXPECT_EQ(hap_record(1,1), 2); - EXPECT_EQ(hap_record(2,1), 0); - EXPECT_EQ(hap_record(3,1), 0); - - EXPECT_EQ(hap_record(0,2), 1); - EXPECT_EQ(hap_record(1,2), 0); - EXPECT_EQ(hap_record(2,2), 2); - EXPECT_EQ(hap_record(3,2), 0); - - // Test second record - - char vcf_record2[] { "chr12 2 . G T Q2 F2 INFO2 GT:AB:HD 0/0:2:1,0,1 0/1:1:0,2,0 1/0:0:0,0,2 0/1:1:0,0,2\n" }; - HaplotypeDataRecord hap_record2 { num_samps, num_founders }; - - hap_record2.parse_vcf_line(vcf_record2); - - EXPECT_EQ(hap_record2(0,0), 1); - EXPECT_EQ(hap_record2(1,0), 0); - EXPECT_EQ(hap_record2(2,0), 0); - EXPECT_EQ(hap_record2(3,0), 0); - - EXPECT_EQ(hap_record2(0,1), 0); - EXPECT_EQ(hap_record2(1,1), 2); - EXPECT_EQ(hap_record2(2,1), 0); - EXPECT_EQ(hap_record2(3,1), 0); - - EXPECT_EQ(hap_record2(0,2), 1); - EXPECT_EQ(hap_record2(1,2), 0); - EXPECT_EQ(hap_record2(2,2), 2); - EXPECT_EQ(hap_record2(3,2), 2); - -} - - - -// TEST(TestConstructorAssignment, CopyConstructor) { -// -// size_t num_vcf_columns { 13 }; -// size_t num_founders { 3 }; -// size_t num_samps { 4 }; -// -// -// char vcf_record[] { "chr12 1 . A T Q1 F1 INFO1 GT:AB:HD 0/0:2:1,0,1 0/0:1:0,2,0 1/0:0:0,0,2 1/1:1:2,0,0\n" }; -// -// HaplotypeDataRecord origin { num_founders, num_samps }; -// HaplotypeDataRecord replicate { origin }; -// -// // test that the objects are indeed different -// EXPECT_NE(&origin, &replicate); -// -// std::array o_dims { origin.dims() }; -// std::array rep_dims { replicate.dims() }; -// -// EXPECT_EQ(o_dims[0], rep_dims[0]); -// EXPECT_EQ(o_dims[1], rep_dims[1]); -// -// -// // make sure that copy constructor correct -// EXPECT_EQ(origin(0,0), replicate(0,0)); -// EXPECT_EQ(origin(0,1), replicate(0,1)); -// EXPECT_EQ(origin(0,2), replicate(0,2)); -// EXPECT_EQ(origin(0,3), replicate(0,3)); -// -// -// EXPECT_EQ(origin(1,0), replicate(1,0)); -// EXPECT_EQ(origin(1,1), replicate(1,1)); -// EXPECT_EQ(origin(1,2), replicate(1,2)); -// EXPECT_EQ(origin(1,3), replicate(1,3)); -// -// -// EXPECT_EQ(origin(2,0), replicate(2,0)); -// EXPECT_EQ(origin(2,1), replicate(2,1)); -// EXPECT_EQ(origin(2,2), replicate(2,2)); -// EXPECT_EQ(origin(2,3), replicate(2,3)); -// -// } - -// std::array meta { "##version=4.2\n", -// "##ID=\n", -// "##FORMAT=\n" }; -// -// std::string h { "#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT S1 S2 S3 S4\n" }; diff --git a/tests/test_haplotype_vcf_parser.cpp b/tests/test_haplotype_vcf_parser.cpp deleted file mode 100644 index 0f8e5ce..0000000 --- a/tests/test_haplotype_vcf_parser.cpp +++ /dev/null @@ -1,116 +0,0 @@ - -#include "../include/HaplotypeVcfParser.h" -#include "../include/utils.h" -#include - - - -char VCF_NAME[] { "../tests/test.vcf" }; - - -TEST(TestHaplotypeVCFParser, Constructor) { - - HaplotypeVcfParser vcf { VCF_NAME }; - - EXPECT_EQ(vcf.n_samples(), 11); - EXPECT_EQ(vcf.k_founders(), 8); -} - - -TEST(TestHaplotypeVCFParser, LoadRecord) { - - HaplotypeVcfParser vcf { VCF_NAME }; - - HaplotypeDataRecord record { vcf.n_samples(), vcf.k_founders() }; - - bool record_loaded { false }; - record_loaded = vcf.load_record(record); - - EXPECT_TRUE(record_loaded); - - EXPECT_EQ(record.chrom(), "chr12"); - EXPECT_EQ(record.pos(), 788); - EXPECT_EQ(record.id(), "."); - EXPECT_EQ(record.ref(), 'A'); - EXPECT_EQ(record.alt(), 'G'); - EXPECT_EQ(record.qual(), "."); - EXPECT_EQ(record.filter(), "PASS"); - EXPECT_EQ(record.info(), "EAF=0.00228;INFO_SCORE=1;HWE=1;ERC=0.01949;EAC=7.94153;PAF=0.00245;REF_PANEL=0"); - EXPECT_EQ(record.format(), "GT:GP:DS:HD"); - - EXPECT_EQ(record(0,0), 1.004); - EXPECT_EQ(record(0,1), 0); - EXPECT_EQ(record(0,2),0.002); - EXPECT_EQ(record(0,3),0.001); - EXPECT_EQ(record(0,4),0); - EXPECT_EQ(record(0,5),0.991); - EXPECT_EQ(record(0,6), 0.001); - EXPECT_EQ(record(0,7), 0.002); - - EXPECT_EQ(record(1,0), 0.001); - EXPECT_EQ(record(1,1), 0); - EXPECT_EQ(record(1,2),0); - EXPECT_EQ(record(1,3),0.989); - EXPECT_EQ(record(1,4),0.005); - EXPECT_EQ(record(1,5),0.005); - EXPECT_EQ(record(1,6), 1); - EXPECT_EQ(record(1,7), 0); - - EXPECT_EQ(record(2,0), 0.998); - EXPECT_EQ(record(2,1), 0); - EXPECT_EQ(record(2,2),0.001); - EXPECT_EQ(record(2,3),0); - EXPECT_EQ(record(2,4),0); - EXPECT_EQ(record(2,5),0); - EXPECT_EQ(record(2,6), 1); - EXPECT_EQ(record(2,7), 0); - - EXPECT_EQ(record(3,0), 0.84); - EXPECT_EQ(record(3,1), 0); - EXPECT_EQ(record(3,2),0); - EXPECT_EQ(record(3,3),0); - EXPECT_EQ(record(3,4),0); - EXPECT_EQ(record(3,5),0); - EXPECT_EQ(record(3,6), 1.159); - EXPECT_EQ(record(3,7), 0); - - EXPECT_EQ(record(10,0), 0.592); - EXPECT_EQ(record(10,1), 0); - EXPECT_EQ(record(10,2),1); - EXPECT_EQ(record(10,3),0); - EXPECT_EQ(record(10,4),0); - EXPECT_EQ(record(10,5),0); - EXPECT_EQ(record(10,6),0); - EXPECT_EQ(record(10,7),0.407); - - // load second record - record_loaded = vcf.load_record(record); - EXPECT_EQ(record.chrom(), "chr12"); - EXPECT_EQ(record.pos(), 1321); - EXPECT_EQ(record.id(), "."); - EXPECT_EQ(record.ref(), 'A'); - EXPECT_EQ(record.alt(), 'C'); - EXPECT_EQ(record.qual(), "."); - EXPECT_EQ(record.filter(), "PASS"); - EXPECT_EQ(record.info(), "EAF=0.01487;INFO_SCORE=0.17212;HWE=1;ERC=1.33325;EAC=116.998;PAF=0.01127;REF_PANEL=0"); - EXPECT_EQ(record.format(), "GT:GP:DS:HD"); - - EXPECT_EQ(record(0,0),1.004); - EXPECT_EQ(record(0,1), 0); - EXPECT_EQ(record(0,2),0.002); - EXPECT_EQ(record(0,3),0.001); - EXPECT_EQ(record(0,4),0); - EXPECT_EQ(record(0,5),0.991); - EXPECT_EQ(record(0,6), 0.001); - EXPECT_EQ(record(0,7), 0.002); - - EXPECT_EQ(record(10,0), 0.592); - EXPECT_EQ(record(10,1), 0); - EXPECT_EQ(record(10,2),1); - EXPECT_EQ(record(10,3),0); - EXPECT_EQ(record(10,4),0); - EXPECT_EQ(record(10,5),0); - EXPECT_EQ(record(10,6),0); - EXPECT_EQ(record(10,7),0.407); - -} diff --git a/tests/test_matrix.cpp b/tests/test_matrix.cpp index bbc5e9f..e17c62f 100644 --- a/tests/test_matrix.cpp +++ b/tests/test_matrix.cpp @@ -1,87 +1,88 @@ -#include "../include/Matrix.h" #include +#include +#include -TEST(TestMatrix, initialize) { +TEST(TestMatrix, Init) { size_t n_row { 3 }; size_t m_col { 2 }; - - Matrix a { n_row, m_col }; - - std::array dims { a.dims() }; - EXPECT_EQ(dims[0], n_row); - EXPECT_EQ(dims[1], m_col); - - for (size_t i = 0; i < n_row; i++) - for (size_t j = 0; j < m_col; j++) - EXPECT_EQ(a(i, j), 0); - - n_row = 0; - - EXPECT_THROW({ - size_t n_row = 0; - size_t m_col = 2; - Matrix b(n_row, m_col); - }, - std::runtime_error); - - EXPECT_ANY_THROW({ - size_t n_row = -1; - size_t m_col = 2; - Matrix b(n_row, m_col); - }); - - EXPECT_THROW({Matrix b(1, 0);}, std::runtime_error); - EXPECT_ANY_THROW({Matrix b(1, -1);}); -} - - - -TEST(TestMatrix, val) { - size_t n_row { 3 }; - size_t m_col { 5 }; - - Matrix a { n_row, m_col }; - - std::array dims { a.dims() }; - - double x = { 1 }; - for (size_t i = 0; i < dims[0]; i++) - for (size_t j = 0; j < dims[1]; j++) - a(i, j) = x++; - - x = 1; - for (size_t i = 0; i < dims[0]; i++) - for (size_t j = 0; j < dims[1]; j++) - EXPECT_FLOAT_EQ(a(i, j), x++); -} - - -TEST(TestMatrix, out_of_bounds) { - size_t n_row { 3 }; - size_t m_col { 5 }; - - Matrix a { n_row, m_col }; - - EXPECT_THROW({ double tmp = a(4, 3); }, std::runtime_error); - EXPECT_THROW({ double tmp = a(3, 5); }, std::runtime_error); - EXPECT_THROW({ double tmp = a(3, 2); }, std::runtime_error); - EXPECT_THROW({ double tmp = a(2, 5); }, std::runtime_error); - EXPECT_THROW({ double tmp = a(-2, 4); }, std::runtime_error); - -} - - -TEST(TestMatrix, dim_and_size) { - size_t n_row { 3 }; - size_t m_col { 5 }; - - Matrix a { n_row, m_col }; - - std::array dims { a.dims() }; - EXPECT_EQ(dims[0], n_row); - EXPECT_EQ(dims[1], m_col); - - EXPECT_EQ(a.size(), n_row * m_col); + printf("%zu\n", n_row * m_col); + //Matrix a { n_row, m_col }; } +// std::array dims { a.dims() }; +// EXPECT_EQ(dims[0], n_row); +// EXPECT_EQ(dims[1], m_col); +// +// for (size_t i = 0; i < n_row; i++) +// for (size_t j = 0; j < m_col; j++) +// EXPECT_EQ(a(i, j), 0); +// +// n_row = 0; +// +// EXPECT_THROW({ +// size_t n_row = 0; +// size_t m_col = 2; +// Matrix b(n_row, m_col); +// }, +// std::runtime_error); +// +// EXPECT_ANY_THROW({ +// size_t n_row = -1; +// size_t m_col = 2; +// Matrix b(n_row, m_col); +// }); +// +// EXPECT_THROW({Matrix b(1, 0);}, std::runtime_error); +// EXPECT_ANY_THROW({Matrix b(1, -1);}); +// } +// +// +// +// TEST(TestMatrix, val) { +// size_t n_row { 3 }; +// size_t m_col { 5 }; +// +// Matrix a { n_row, m_col }; +// +// std::array dims { a.dims() }; +// +// double x = { 1 }; +// for (size_t i = 0; i < dims[0]; i++) +// for (size_t j = 0; j < dims[1]; j++) +// a(i, j) = x++; +// +// x = 1; +// for (size_t i = 0; i < dims[0]; i++) +// for (size_t j = 0; j < dims[1]; j++) +// EXPECT_FLOAT_EQ(a(i, j), x++); +// } +// +// +// TEST(TestMatrix, out_of_bounds) { +// size_t n_row { 3 }; +// size_t m_col { 5 }; +// +// Matrix a { n_row, m_col }; +// +// EXPECT_THROW({ double tmp = a(4, 3); }, std::runtime_error); +// EXPECT_THROW({ double tmp = a(3, 5); }, std::runtime_error); +// EXPECT_THROW({ double tmp = a(3, 2); }, std::runtime_error); +// EXPECT_THROW({ double tmp = a(2, 5); }, std::runtime_error); +// EXPECT_THROW({ double tmp = a(-2, 4); }, std::runtime_error); +// +// } +// +// +// TEST(TestMatrix, dim_and_size) { +// size_t n_row { 3 }; +// size_t m_col { 5 }; +// +// Matrix a { n_row, m_col }; +// +// std::array dims { a.dims() }; +// EXPECT_EQ(dims[0], n_row); +// EXPECT_EQ(dims[1], m_col); +// +// EXPECT_EQ(a.size(), n_row * m_col); +// } diff --git a/tests/test_parse_hts.cpp b/tests/test_parse_hts.cpp new file mode 100644 index 0000000..6075c7f --- /dev/null +++ b/tests/test_parse_hts.cpp @@ -0,0 +1,113 @@ + +#include +#include + + +char VCF_NAME[] { "geno_test_data.vcf" }; + + +TEST(TestHaplotypeVCFParser, Constructor) { + ParseHtsVariantFile vcf { VCF_NAME, "" }; + + // EXPECT_EQ(vcf.n_samples(), 11); + // EXPECT_EQ(vcf.k_founders(), 8); +} + + +// TEST(TestHaplotypeVCFParser, LoadRecord) { +// +// HaplotypeVcfParser vcf { VCF_NAME }; +// +// HaplotypeDataRecord record { vcf.n_samples(), vcf.k_founders() }; +// +// bool record_loaded { false }; +// record_loaded = vcf.load_record(record); +// +// EXPECT_TRUE(record_loaded); +// +// EXPECT_EQ(record.chrom(), "chr12"); +// EXPECT_EQ(record.pos(), 788); +// EXPECT_EQ(record.id(), "."); +// EXPECT_EQ(record.ref(), 'A'); +// EXPECT_EQ(record.alt(), 'G'); +// EXPECT_EQ(record.qual(), "."); +// EXPECT_EQ(record.filter(), "PASS"); +// EXPECT_EQ(record.info(), "EAF=0.00228;INFO_SCORE=1;HWE=1;ERC=0.01949;EAC=7.94153;PAF=0.00245;REF_PANEL=0"); +// EXPECT_EQ(record.format(), "GT:GP:DS:HD"); +// +// EXPECT_EQ(record(0,0), 1.004); +// EXPECT_EQ(record(0,1), 0); +// EXPECT_EQ(record(0,2),0.002); +// EXPECT_EQ(record(0,3),0.001); +// EXPECT_EQ(record(0,4),0); +// EXPECT_EQ(record(0,5),0.991); +// EXPECT_EQ(record(0,6), 0.001); +// EXPECT_EQ(record(0,7), 0.002); +// +// EXPECT_EQ(record(1,0), 0.001); +// EXPECT_EQ(record(1,1), 0); +// EXPECT_EQ(record(1,2),0); +// EXPECT_EQ(record(1,3),0.989); +// EXPECT_EQ(record(1,4),0.005); +// EXPECT_EQ(record(1,5),0.005); +// EXPECT_EQ(record(1,6), 1); +// EXPECT_EQ(record(1,7), 0); +// +// EXPECT_EQ(record(2,0), 0.998); +// EXPECT_EQ(record(2,1), 0); +// EXPECT_EQ(record(2,2),0.001); +// EXPECT_EQ(record(2,3),0); +// EXPECT_EQ(record(2,4),0); +// EXPECT_EQ(record(2,5),0); +// EXPECT_EQ(record(2,6), 1); +// EXPECT_EQ(record(2,7), 0); +// +// EXPECT_EQ(record(3,0), 0.84); +// EXPECT_EQ(record(3,1), 0); +// EXPECT_EQ(record(3,2),0); +// EXPECT_EQ(record(3,3),0); +// EXPECT_EQ(record(3,4),0); +// EXPECT_EQ(record(3,5),0); +// EXPECT_EQ(record(3,6), 1.159); +// EXPECT_EQ(record(3,7), 0); +// +// EXPECT_EQ(record(10,0), 0.592); +// EXPECT_EQ(record(10,1), 0); +// EXPECT_EQ(record(10,2),1); +// EXPECT_EQ(record(10,3),0); +// EXPECT_EQ(record(10,4),0); +// EXPECT_EQ(record(10,5),0); +// EXPECT_EQ(record(10,6),0); +// EXPECT_EQ(record(10,7),0.407); +// +// // load second record +// record_loaded = vcf.load_record(record); +// EXPECT_EQ(record.chrom(), "chr12"); +// EXPECT_EQ(record.pos(), 1321); +// EXPECT_EQ(record.id(), "."); +// EXPECT_EQ(record.ref(), 'A'); +// EXPECT_EQ(record.alt(), 'C'); +// EXPECT_EQ(record.qual(), "."); +// EXPECT_EQ(record.filter(), "PASS"); +// EXPECT_EQ(record.info(), "EAF=0.01487;INFO_SCORE=0.17212;HWE=1;ERC=1.33325;EAC=116.998;PAF=0.01127;REF_PANEL=0"); +// EXPECT_EQ(record.format(), "GT:GP:DS:HD"); +// +// EXPECT_EQ(record(0,0),1.004); +// EXPECT_EQ(record(0,1), 0); +// EXPECT_EQ(record(0,2),0.002); +// EXPECT_EQ(record(0,3),0.001); +// EXPECT_EQ(record(0,4),0); +// EXPECT_EQ(record(0,5),0.991); +// EXPECT_EQ(record(0,6), 0.001); +// EXPECT_EQ(record(0,7), 0.002); +// +// EXPECT_EQ(record(10,0), 0.592); +// EXPECT_EQ(record(10,1), 0); +// EXPECT_EQ(record(10,2),1); +// EXPECT_EQ(record(10,3),0); +// EXPECT_EQ(record(10,4),0); +// EXPECT_EQ(record(10,5),0); +// EXPECT_EQ(record(10,6),0); +// EXPECT_EQ(record(10,7),0.407); +// +// } diff --git a/tests/test_utils.cpp b/tests/test_utils.cpp deleted file mode 100644 index 50043fe..0000000 --- a/tests/test_utils.cpp +++ /dev/null @@ -1,97 +0,0 @@ - -#include "../include/utils.h" -#include - - - -TEST(TestCharBuffer, Constructor) { - size_t buff_size { 100 }; - CharBuffer buff { buff_size }; - - EXPECT_EQ(buff.buffer_size(), buff_size); -} - - -TEST(TestCharBuffer, Append) { - size_t buff_size { 5 }; - CharBuffer buff { buff_size }; - - buff.append('t'); - EXPECT_EQ(buff(0), 't'); - - buff.append('h'); - buff.append('e'); - EXPECT_EQ(buff(0), 't'); - EXPECT_EQ(buff(1), 'h'); - EXPECT_EQ(buff(2), 'e'); - - EXPECT_THROW(buff(6), std::out_of_range); - EXPECT_THROW(buff(5), std::out_of_range); - EXPECT_THROW(buff(-1), std::out_of_range); - - buff.append('r'); - buff.append('e'); - EXPECT_THROW(buff.append('e'), std::out_of_range); -} - - -TEST(TestCharBuffer, Reset) { - size_t buff_size { 5 }; - CharBuffer buff { buff_size }; - - buff.append('t'); - EXPECT_EQ(buff(0), 't'); - - buff.reset(); - EXPECT_THROW(buff(0), std::out_of_range); -} - - -TEST(TestCharBuffer, Data) { - size_t buff_size { 5 }; - CharBuffer buff { buff_size }; - buff.append('t'); - buff.append('h'); - buff.append('e'); - - EXPECT_EQ(buff.data()[0], 't'); - - std::string s { buff.data() }; - EXPECT_EQ(s, "the"); - - buff.append('e'); - s = buff.data(); - EXPECT_EQ(s, "thee"); - -} - - -TEST(TestStringRecord, ConstructorDelim) { - CharBuffer buf { 10 }; - char s[] { "the\tcat\n" }; - StringRecord record { '\t' }; - - record.update_str(s); - - EXPECT_TRUE(record.next_field()); - EXPECT_EQ(static_cast(record.data()), "the"); - - EXPECT_TRUE(record.next_field()); - EXPECT_EQ(static_cast(record.data()), "cat"); - - EXPECT_FALSE(record.next_field()); - - EXPECT_EQ(record.size(), 8); - - record.reset(); - - EXPECT_EQ(record.size(), 0); -} - -TEST(TestStringRecord, SingleArgConstructor) { - StringRecord record { '\t' }; - - EXPECT_EQ(record.size(), 0); - EXPECT_FALSE(record.next_field()); - -} From 907d0ee5a3104297a99a26de44c705a1fd736ef9 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 19 Nov 2025 08:46:31 -0800 Subject: [PATCH 02/58] intermediate progress on using htslib for reading vcf, vcf.gz, bcf files. --- Makefile | 22 ++- include/{parse_hts.h => bcfio.h} | 42 +++-- src/{parse_hts.cpp => bcfio.cpp} | 41 ++++- tests/{test_parse_hts.cpp => test_bcfio.cpp} | 8 +- tests/test_matrix.cpp | 154 +++++++++---------- 5 files changed, 161 insertions(+), 106 deletions(-) rename include/{parse_hts.h => bcfio.h} (80%) rename src/{parse_hts.cpp => bcfio.cpp} (83%) rename tests/{test_parse_hts.cpp => test_bcfio.cpp} (95%) diff --git a/Makefile b/Makefile index 9040d07..262a426 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ CXXLDFLAGS = $(addprefix -I, $(CXXLD)) CXXLIB += $(LOCAL_LIB) CXXLIBFLAGS = $(addprefix -L, $(CXXLIB)) -APP_FILES = matrix.cpp parse_hts.cpp +APP_FILES = matrix.cpp bcfio.cpp APP_SRC = $(addprefix $(SRC_DIR)/, $(APP_FILES)) APP_OBJS = $(addprefix $(BUILD_DIR)/, $(APP_FILES:.cpp=.o)) APP_DEPS = $(APP_OBJS:.o=.d) @@ -63,7 +63,8 @@ TEST_DIR = tests TEST_SRC = $(wildcard $(TEST_DIR)/test_*.cpp) TEST_OBJS = $(subst $(TEST_DIR), $(BUILD_DIR), $(TEST_SRC:.cpp=.o)) TEST_DEPS = $(TEST_OBJS:.o=.d) -TEST_DATA = $(wildcard $(TEST_DIR)/geno_test_data.*) +TEST_DATA_SRC = $(wildcard $(TEST_DIR)/geno_test_data.*) +TEST_DATA_DST = $(subst $(TEST_DIR), $(BUILD_DIR), $(TEST_DATA_SRC)) TEST_TARGET_PRG = $(BUILD_DIR)/runtests @@ -92,16 +93,16 @@ $(BUILD_DIR): ###################################################################### -$(TEST_TARGET_PRG): $(TEST_DIR)/main.cpp $(TEST_OBJS) | $(TARGET) - $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -lgtest +$(TEST_TARGET_PRG): $(TEST_DIR)/main.cpp $(TEST_OBJS) $(APP_OBJS) | $(TARGET) + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -lgtest -lhts $(BUILD_DIR)/test_%.o: $(TEST_DIR)/test_%.cpp $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(OBJ_OUTPUT_OPTIONS) $< +data: | $(TEST_DATA_DST) -.PHONY: data -data: $(TEST_DATA) - rsync -avz $^ $(BUILD_DIR)/ +$(BUILD_DIR)/geno_test_data%: $(TEST_DIR)/geno_test_data% + rsync -avz $< $(BUILD_DIR)/ # tests: $(BUILD_DIR)/test_log #$(BUILD_DIR)/test_argparse # @@ -130,6 +131,13 @@ data: $(TEST_DATA) # ###################################################################### +check: + ./$(TEST_TARGET_PRG) + +###################################################################### +# +###################################################################### + -include $(APP_DEPS) -include $(TEST_DEPS) diff --git a/include/parse_hts.h b/include/bcfio.h similarity index 80% rename from include/parse_hts.h rename to include/bcfio.h index 9b21e75..ca8686e 100644 --- a/include/parse_hts.h +++ b/include/bcfio.h @@ -101,32 +101,56 @@ const char HAP_CODE[] { "HD" }; // StringRecord hap_parse_ { HAP_DELIM }; // }; + +struct BcfHeaderFmt { + uint64_t number : 20; + uint64_t v : 4; + uint64_t type : 4; + uint64_t coltype : 4; +}; + + +struct BcfHeader { + htslib::bcf_hdr_t *hdr; + + BcfHeader(htslib::htsFile *fid): + hdr(fid ? htslib::bcf_hdr_read(fid) : nullptr) {}; + ~BcfHeader() { if (hdr) htslib::bcf_hdr_destroy(hdr); }; + + const bool isnull() const { return hdr == nullptr; }; + + // sample_names() + int get_format(const std::string& name, BcfHeaderFmt *b) const; +}; + + + // Interface with htslib bcf tools -class ParseHtsVariantFile +class ReadBcf { public: // HaplotypeVcfParser(const char* variant_fname); - ParseHtsVariantFile(const char *variant_fname, const char *sample_fname); + ReadBcf(const char *variant_fname, const char *sample_fname); // HaplotypeVcfParser(const std::string& variant_fname); // HaplotypeVcfParser(const std::string& variant_fname, // const std::string& sample_fname); - ParseHtsVariantFile()=delete; - ParseHtsVariantFile(const ParseHtsVariantFile&)=delete; - ParseHtsVariantFile(const ParseHtsVariantFile&&)=delete; + ReadBcf()=delete; + ReadBcf(const ReadBcf&)=delete; + ReadBcf(const ReadBcf&&)=delete; // HaplotypeVcfParser& operator=(const HaplotypeVcfParser&)=delete; - ~ParseHtsVariantFile(); + ~ReadBcf(); - // size_t n_samples() const; - // size_t k_founders() const; + const size_t n_samples() const; + const size_t k_founders() const; // bool load_record(HaplotypeDataRecord&); private: const std::string fname_; htslib::htsFile *fid_; - htslib::bcf_hdr_t *hdr_; + BcfHeader hdr_; // size_t n_cols_ { 0 }; // size_t n_samples_ { 0 }; diff --git a/src/parse_hts.cpp b/src/bcfio.cpp similarity index 83% rename from src/parse_hts.cpp rename to src/bcfio.cpp index 1a3065e..d954245 100644 --- a/src/parse_hts.cpp +++ b/src/bcfio.cpp @@ -13,17 +13,30 @@ // reviewed by Claude Sonnnet, the AI assistant from Anthropic // (Jan 2025), with minor recommendations incorporated. -#include +#include #include // const static size_t DEFAULT_BUFFER_SIZE { 100000 }; +int BcfHeader::get_format(const std::string& name, BcfHeaderFmt *b) const { + int idx = htslib::bcf_hdr_id2int(hdr, BCF_DT_ID, name.c_str()); + if (idx < 0) + return -1; -ParseHtsVariantFile::ParseHtsVariantFile(const char *variant_fname, - const char *sample_fname) + b->number = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT]>>12; + b->v = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT]>>8 & 0b1111; + b->type = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT]>>4 & 0b1111; + b->coltype = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT] & 0b1111; + + return 0; +} + + + +ReadBcf::ReadBcf(const char *variant_fname, const char *sample_fname) : fname_(variant_fname), fid_(htslib::hts_open(variant_fname, "r")), - hdr_(htslib::bcf_hdr_read(fid_)) { + hdr_(fid_) { int status { 0 }; // Subset samples with those found in the file sample_fname @@ -31,7 +44,7 @@ ParseHtsVariantFile::ParseHtsVariantFile(const char *variant_fname, fprintf(stdout, "No file with sample names detected, computing" "hGRM over all samples.\n"); else - status = htslib::bcf_hdr_set_samples(hdr_, sample_fname, 1); + status = htslib::bcf_hdr_set_samples(hdr_.hdr, sample_fname, 1); if (status < 0) { fprintf(stderr, "Error: Couldn't read sample file\n"); @@ -46,12 +59,24 @@ ParseHtsVariantFile::ParseHtsVariantFile(const char *variant_fname, // get number of characters in data record for line buffer size }; -ParseHtsVariantFile::~ParseHtsVariantFile() { +ReadBcf::~ReadBcf() { if (fid_) htslib::hts_close(fid_); - if (hdr_) - htslib::bcf_hdr_destroy(hdr_); +} + +const size_t ReadBcf::n_samples() const { + // See htslib/vcf.h line 649 + // Remember that n is the number of entries in the triplet of + // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n + // that correspondes to the number of samples. + return hdr_.hdr->n[BCF_DT_SAMPLE]; +}; +const size_t ReadBcf::k_founders() const { + std::unique_ptr hapvals = std::make_unique(); + if (hdr_.get_format("HD", hapvals.get()) < 0) + printf("errror\n"); + return static_cast(hapvals->number); } // ParseHtsVariantFile::ParseHtsVariantFile(char* filename, size_t buff_size) diff --git a/tests/test_parse_hts.cpp b/tests/test_bcfio.cpp similarity index 95% rename from tests/test_parse_hts.cpp rename to tests/test_bcfio.cpp index 6075c7f..9c942f3 100644 --- a/tests/test_parse_hts.cpp +++ b/tests/test_bcfio.cpp @@ -1,14 +1,14 @@ #include -#include +#include -char VCF_NAME[] { "geno_test_data.vcf" }; +char VCF_NAME[] { "build/geno_test_data.vcf" }; TEST(TestHaplotypeVCFParser, Constructor) { - ParseHtsVariantFile vcf { VCF_NAME, "" }; - + ReadBcf vcf { VCF_NAME, "" }; + printf("K founders %lu\n", vcf.k_founders()); // EXPECT_EQ(vcf.n_samples(), 11); // EXPECT_EQ(vcf.k_founders(), 8); } diff --git a/tests/test_matrix.cpp b/tests/test_matrix.cpp index e17c62f..7fda809 100644 --- a/tests/test_matrix.cpp +++ b/tests/test_matrix.cpp @@ -7,82 +7,80 @@ TEST(TestMatrix, Init) { size_t n_row { 3 }; size_t m_col { 2 }; - printf("%zu\n", n_row * m_col); - //Matrix a { n_row, m_col }; + Matrix a { n_row, m_col }; + std::array dims { a.dims() }; + EXPECT_EQ(dims[0], n_row); + EXPECT_EQ(dims[1], m_col); + + for (size_t i = 0; i < n_row; i++) + for (size_t j = 0; j < m_col; j++) + EXPECT_EQ(a(i, j), 0); + + n_row = 0; + + EXPECT_THROW({ + size_t n_row = 0; + size_t m_col = 2; + Matrix b(n_row, m_col); + }, + std::runtime_error); + + EXPECT_ANY_THROW({ + size_t n_row = -1; + size_t m_col = 2; + Matrix b(n_row, m_col); + }); + + EXPECT_THROW({Matrix b(1, 0);}, std::runtime_error); + EXPECT_ANY_THROW({Matrix b(1, -1);}); +} + + + +TEST(TestMatrix, Vals) { + size_t n_row { 3 }; + size_t m_col { 5 }; + + Matrix a { n_row, m_col }; + + std::array dims { a.dims() }; + + double x = { 1 }; + for (size_t i = 0; i < dims[0]; i++) + for (size_t j = 0; j < dims[1]; j++) + a(i, j) = x++; + + x = 1; + for (size_t i = 0; i < dims[0]; i++) + for (size_t j = 0; j < dims[1]; j++) + EXPECT_FLOAT_EQ(a(i, j), x++); +} + + +TEST(TestMatrix, OutOfBounds) { + size_t n_row { 3 }; + size_t m_col { 5 }; + + Matrix a { n_row, m_col }; + + EXPECT_THROW({ a(4, 3); }, std::runtime_error); + EXPECT_THROW({ a(3, 5); }, std::runtime_error); + EXPECT_THROW({ a(3, 2); }, std::runtime_error); + EXPECT_THROW({ a(2, 5); }, std::runtime_error); + EXPECT_THROW({ a(-2, 4); }, std::runtime_error); + +} + + +TEST(TestMatrix, DimAndSize) { + size_t n_row { 3 }; + size_t m_col { 5 }; + + Matrix a { n_row, m_col }; + + std::array dims { a.dims() }; + EXPECT_EQ(dims[0], n_row); + EXPECT_EQ(dims[1], m_col); + + EXPECT_EQ(a.size(), n_row * m_col); } -// std::array dims { a.dims() }; -// EXPECT_EQ(dims[0], n_row); -// EXPECT_EQ(dims[1], m_col); -// -// for (size_t i = 0; i < n_row; i++) -// for (size_t j = 0; j < m_col; j++) -// EXPECT_EQ(a(i, j), 0); -// -// n_row = 0; -// -// EXPECT_THROW({ -// size_t n_row = 0; -// size_t m_col = 2; -// Matrix b(n_row, m_col); -// }, -// std::runtime_error); -// -// EXPECT_ANY_THROW({ -// size_t n_row = -1; -// size_t m_col = 2; -// Matrix b(n_row, m_col); -// }); -// -// EXPECT_THROW({Matrix b(1, 0);}, std::runtime_error); -// EXPECT_ANY_THROW({Matrix b(1, -1);}); -// } -// -// -// -// TEST(TestMatrix, val) { -// size_t n_row { 3 }; -// size_t m_col { 5 }; -// -// Matrix a { n_row, m_col }; -// -// std::array dims { a.dims() }; -// -// double x = { 1 }; -// for (size_t i = 0; i < dims[0]; i++) -// for (size_t j = 0; j < dims[1]; j++) -// a(i, j) = x++; -// -// x = 1; -// for (size_t i = 0; i < dims[0]; i++) -// for (size_t j = 0; j < dims[1]; j++) -// EXPECT_FLOAT_EQ(a(i, j), x++); -// } -// -// -// TEST(TestMatrix, out_of_bounds) { -// size_t n_row { 3 }; -// size_t m_col { 5 }; -// -// Matrix a { n_row, m_col }; -// -// EXPECT_THROW({ double tmp = a(4, 3); }, std::runtime_error); -// EXPECT_THROW({ double tmp = a(3, 5); }, std::runtime_error); -// EXPECT_THROW({ double tmp = a(3, 2); }, std::runtime_error); -// EXPECT_THROW({ double tmp = a(2, 5); }, std::runtime_error); -// EXPECT_THROW({ double tmp = a(-2, 4); }, std::runtime_error); -// -// } -// -// -// TEST(TestMatrix, dim_and_size) { -// size_t n_row { 3 }; -// size_t m_col { 5 }; -// -// Matrix a { n_row, m_col }; -// -// std::array dims { a.dims() }; -// EXPECT_EQ(dims[0], n_row); -// EXPECT_EQ(dims[1], m_col); -// -// EXPECT_EQ(a.size(), n_row * m_col); -// } From 24a0f80423511d64c34918c0cf5bc507240aacbb Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 19 Nov 2025 11:43:34 -0800 Subject: [PATCH 03/58] update for reading bcf header --- include/bcfio.h | 101 +++++------------- src/bcfio.cpp | 236 +++++++------------------------------------ tests/test_bcfio.cpp | 217 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 276 insertions(+), 278 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index ca8686e..23c354f 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -28,89 +28,29 @@ extern "C" { } } - - // samples are separated by white space const char HAP_CODE[] { "HD" }; -// const char META_PREFIX { '#' }; -// const char MEASUREMENT_DELIM { ':' }; -// const char HAP_DELIM { ',' }; -// const int NUM_VCF_FIELDS { 9 }; -// const char SPACE_DELIM { '\t' }; -// -// -// // NOTE: in the future it may be best to test for set membership -// static const char* VCF_FIELD_NAMES[NUM_VCF_FIELDS] { -// "#CHROM", -// "POS", -// "ID", -// "REF", -// "ALT", -// "QUAL", -// "FILTER", -// "INFO", -// "FORMAT" -// }; -// -// -// // Move semantics, I don't want to copy data -// class HaplotypeDataRecord -// { -// public: -// -// HaplotypeDataRecord()=delete; -// HaplotypeDataRecord(size_t, size_t); -// HaplotypeDataRecord(const HaplotypeDataRecord&)=delete; -// HaplotypeDataRecord(HaplotypeDataRecord&&)=delete; -// -// -// const std::string& chrom() const; -// const long pos() const; -// const std::string& id() const; -// const char ref() const; -// const char alt() const; -// const std::string& qual() const; -// const std::string& filter() const; -// const std::string& info() const; -// const std::string& format() const; -// -// void parse_vcf_line(const char*); -// const double& operator()(size_t, size_t) const; -// -// std::array dims() const; -// -// -// private: -// size_t n_samples_; -// size_t k_founders_; -// -// std::string chrom_ { "" }; -// long pos_ { -1 }; -// std::string id_ { "" }; -// char ref_ { '\0' }; -// char alt_ { '\0' }; -// std::string qual_ { "" }; -// std::string filter_ { "" }; -// std::string info_ { "" }; -// std::string format_ { "" }; -// -// std::unique_ptr samples_ { nullptr }; -// -// StringRecord line_parse_ { SPACE_DELIM }; -// StringRecord field_parse_ { MEASUREMENT_DELIM }; -// StringRecord hap_parse_ { HAP_DELIM }; -// }; - - -struct BcfHeaderFmt { + +namespace bcfio { +// @title The meta data on a BCF attribute +// @description BCF, VCF, and VCF.GZ files hold metadata in the header that +// specify the type and format of data in records. I call each unique +// piece of data in a record a record attribute, e.g. an INFO column or +// FORMAT column of a record are attributes of that record. HTSLIB encodes +// attribute information an unsigned 64 bit integer, and to access any value +// one needs to correctly implement bit shifting and masking. This struct +// contains bit-fields representing each value stored in the uint64_t. +// +struct BcfHdrAttr { uint64_t number : 20; - uint64_t v : 4; + uint64_t vl_type : 4; uint64_t type : 4; uint64_t coltype : 4; }; -struct BcfHeader { +class BcfHeader { +public: htslib::bcf_hdr_t *hdr; BcfHeader(htslib::htsFile *fid): @@ -120,7 +60,14 @@ struct BcfHeader { const bool isnull() const { return hdr == nullptr; }; // sample_names() - int get_format(const std::string& name, BcfHeaderFmt *b) const; + const int get_format(const char *name, BcfHdrAttr *ptr) const; + const int get_info(const char *name, BcfHdrAttr *ptr) const; + const int get_filter(const char *name, BcfHdrAttr *ptr) const; + +private: + const int decode_hts_idinfo_(const char *name, + const int bcf_dt_type, + BcfHdrAttr *ptr) const; }; @@ -130,6 +77,7 @@ class ReadBcf { public: // HaplotypeVcfParser(const char* variant_fname); + ReadBcf(const char *variant_fname); ReadBcf(const char *variant_fname, const char *sample_fname); // HaplotypeVcfParser(const std::string& variant_fname); // HaplotypeVcfParser(const std::string& variant_fname, @@ -162,5 +110,6 @@ class ReadBcf // size_t get_line_num_char_(); // void set_params_(); }; +} #endif diff --git a/src/bcfio.cpp b/src/bcfio.cpp index d954245..92a1938 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -7,33 +7,52 @@ // filename: vcf with haplotpye // // -// Acknowledgment // -// Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. #include #include -// const static size_t DEFAULT_BUFFER_SIZE { 100000 }; +// decoder based upon htslib/vcf.h line 100 in the typedef struct bcf_idinfo_t. +const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, + const int bcf_dt_type, + bcfio::BcfHdrAttr *ptr) const { + + // BCF_DT_ID is the ID dictionary index defined by htslib + int idx = htslib::bcf_hdr_id2int(hdr, BCF_DT_ID, name); -int BcfHeader::get_format(const std::string& name, BcfHeaderFmt *b) const { - int idx = htslib::bcf_hdr_id2int(hdr, BCF_DT_ID, name.c_str()); if (idx < 0) return -1; - b->number = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT]>>12; - b->v = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT]>>8 & 0b1111; - b->type = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT]>>4 & 0b1111; - b->coltype = hdr->id[BCF_DT_ID][idx].val->info[BCF_HL_FMT] & 0b1111; + uint64_t val = hdr->id[BCF_DT_ID][idx].val->info[bcf_dt_type]; + + ptr->number = val >> 12 & 0xfffff; + ptr->vl_type = val >> 8 & 0xf; + ptr->type = val >> 4 & 0xf; + ptr->coltype = val & 0xf; return 0; } +const int bcfio::BcfHeader::get_format(const char *name, BcfHdrAttr *ptr) const { + return decode_hts_idinfo_(name, BCF_HL_FMT, ptr); +} + +const int bcfio::BcfHeader::get_info(const char *name, BcfHdrAttr *ptr) const { + return decode_hts_idinfo_(name, BCF_HL_INFO, ptr); +} + +const int bcfio::BcfHeader::get_filter(const char *name, BcfHdrAttr *ptr) const { + return decode_hts_idinfo_(name, BCF_HL_FLT, ptr); +} -ReadBcf::ReadBcf(const char *variant_fname, const char *sample_fname) +bcfio::ReadBcf::ReadBcf(const char *variant_fname) + : fname_(variant_fname), + fid_(htslib::hts_open(variant_fname, "r")), + hdr_(fid_) {}; + + +bcfio::ReadBcf::ReadBcf(const char *variant_fname, const char *sample_fname) : fname_(variant_fname), fid_(htslib::hts_open(variant_fname, "r")), hdr_(fid_) { @@ -59,12 +78,12 @@ ReadBcf::ReadBcf(const char *variant_fname, const char *sample_fname) // get number of characters in data record for line buffer size }; -ReadBcf::~ReadBcf() { +bcfio::ReadBcf::~ReadBcf() { if (fid_) htslib::hts_close(fid_); } -const size_t ReadBcf::n_samples() const { +const size_t bcfio::ReadBcf::n_samples() const { // See htslib/vcf.h line 649 // Remember that n is the number of entries in the triplet of // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n @@ -72,189 +91,12 @@ const size_t ReadBcf::n_samples() const { return hdr_.hdr->n[BCF_DT_SAMPLE]; }; -const size_t ReadBcf::k_founders() const { - std::unique_ptr hapvals = std::make_unique(); - if (hdr_.get_format("HD", hapvals.get()) < 0) - printf("errror\n"); - return static_cast(hapvals->number); -} +const size_t bcfio::ReadBcf::k_founders() const { + BcfHdrAttr fmt {}; -// ParseHtsVariantFile::ParseHtsVariantFile(char* filename, size_t buff_size) -// : fname_(filename), -// file_io_(BufferedRead(filename, buff_size)) { -// -// // get number of characters in data record for line buffer size -// size_t nchar { get_line_num_char_() }; -// -// if (nchar == 0) -// throw std::runtime_error("No data to read"); -// -// // make buffer 10% larger then the number of characters read. -// line_buffer_size_ = static_cast(nchar * 1.1); -// line_buffer_.reset(line_buffer_size_); -// -// set_params_(); -// pos_(fpos_record_one_); -// }; -// -// -// // ParseHtsVariantFile::ParseHtsVariantFile(std::string filename) -// // : fname_(filename), -// // fid_(filename) { -// // -// // if (fid_.bad()) -// // throw std::runtime_error("File Access error"); -// // else if (fid_.eof()) -// // throw std::runtime_error("File is empty"); -// // -// // // get number of characters in data record for line buffer size -// // size_t nchar { get_line_num_char_() }; -// // -// // -// // // make buffer 10% larger then the number of characters read. -// // line_buffer_size_ = static_cast(nchar * 1.1); -// // line_buffer_ = new char[line_buffer_size_]; -// // line_buffer_[0] = '\0'; -// // -// // pos_(std::ios_base::beg); -// // set_params_(); -// // }; -// -// -// //ParseHtsVariantFile::~ParseHtsVariantFile() { -// // if(fid_) -// // fclose(fid_); -// // // free(line_buffer_); -// //} -// -// -// size_t ParseHtsVariantFile::get_line_num_char_() { -// -// size_t char_count { 0 }; -// size_t max_char_count { 0 }; -// char c; -// -// while ((c = file_io_.get_char()) != '\0') { -// char_count++; -// -// if (c == '\n' && char_count > max_char_count) { -// max_char_count = char_count; -// char_count = 0; -// } -// } -// -// return max_char_count; -// } -// -// -// size_t ParseHtsVariantFile::n_samples() const { return n_samples_; } -// -// -// size_t ParseHtsVariantFile::k_founders() const { return k_founders_; } -// -// -// void ParseHtsVariantFile::pos_(size_t n) { -// file_io_.reset(); -// if (n != 0) -// file_io_.seek(n); -// } -// -// -// void ParseHtsVariantFile::set_params_() { -// pos_(0); -// -// // skip meta data lines -// -// size_t n { 0 }; -// size_t num_bytes { 0 }; -// while ((n = file_io_.get_line(line_buffer_)) > 0) { -// -// // the +1 is because I don't write newline characters to buffer -// num_bytes += line_buffer_.size()+1; -// -// if (line_buffer_(0) == META_PREFIX && line_buffer_(1) == META_PREFIX) -// continue; -// -// break; -// } -// -// // Store file position of first record -// fpos_record_one_ = sizeof(line_buffer_(0)) * num_bytes; -// -// // if there is no header -// if (line_buffer_(0) != META_PREFIX) -// return; -// -// -// if (std::isspace(line_buffer_(0))) -// throw std::runtime_error("First element of VCF line must not be blank."); -// -// -// // Get column number and sample number -// StringRecord line_parser_ { SPACE_DELIM, line_buffer_.data() }; -// StringRecord field_parser_ { MEASUREMENT_DELIM }; -// StringRecord hap_parser_ { HAP_DELIM }; -// -// n_cols_ = 0; -// n_samples_ = 0; -// for (; line_parser_.next_field(); n_cols_++) { -// -// if (n_cols_ < NUM_VCF_FIELDS -// && std::strcmp(line_parser_.data(), VCF_FIELD_NAMES[n_cols_]) != 0) -// throw std::runtime_error("File doesn't follow vcf header specification"); -// -// if (n_cols_ >= NUM_VCF_FIELDS) -// n_samples_++; -// -// } -// -// -// // get k founders from record -// if ((n = file_io_.get_line(line_buffer_)) == 0) -// throw std::runtime_error("End of file"); -// -// line_parser_.update_str(line_buffer_.data()); -// size_t hap_idx { 0 }; -// for (int i = 0; line_parser_.next_field(); i++) { -// -// if (i == NUM_VCF_FIELDS-1) { -// field_parser_.update_str(line_parser_.data()); -// -// for (;field_parser_.next_field(); hap_idx++) -// if (std::strcmp(field_parser_.data(), HAP_CODE) == 0) -// break; -// -// } else if(i == NUM_VCF_FIELDS) { -// field_parser_.update_str(line_parser_.data()); -// -// for(int i = 0; field_parser_.next_field() && i < hap_idx; hap_idx++) -// ; -// -// hap_parser_.update_str(field_parser_.data()); -// for (;hap_parser_.next_field(); k_founders_++) -// ; -// -// break; -// } -// -// } -// -// if (k_founders_ == 0) -// throw std::runtime_error("Parse error"); -// -// } -// -// -// bool ParseHtsVariantFile::load_record(HaplotypeDataRecord& record) { -// -// size_t n { 0 }; -// -// if ((n = file_io_.get_line(line_buffer_)) == 0) -// return false; -// -// record.parse_vcf_line(line_buffer_.data()); -// -// return true; -// } + if (hdr_.get_format("HD", &fmt) < 0) + printf("errror\n"); + return static_cast(fmt.number); +} diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index 9c942f3..ac7b8ef 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -1,16 +1,223 @@ #include + +namespace htslib { +extern "C" { +#include +#include +} +} + #include char VCF_NAME[] { "build/geno_test_data.vcf" }; +char VCFGZ_NAME[] { "build/geno_test_data.vcf.gz" }; +char BCF_NAME[] { "build/geno_test_data.bcf" }; +size_t K_FOUNDERS = 8; +size_t N_SAMPS = 11; + + +TEST(TestBcfHeader, ConstructorVcfHdr) { + htslib::htsFile *fid = htslib::hts_open(VCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("HD", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.number, K_FOUNDERS); + EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); + EXPECT_EQ(attr.type, BCF_HT_REAL); + + if (fid) htslib::hts_close(fid); +} + +TEST(TestBcfHeader, ConstructorVcfGzHdr) { + htslib::htsFile *fid = htslib::hts_open(VCFGZ_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("HD", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.number, K_FOUNDERS); + EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); + EXPECT_EQ(attr.type, BCF_HT_REAL); + + if (fid) htslib::hts_close(fid); +} + +TEST(TestBcfHeader, ConstructorBcfHdr) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("HD", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.number, K_FOUNDERS); + EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); + EXPECT_EQ(attr.type, BCF_HT_REAL); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrFmtGt) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("GT", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.number, 1); + EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); + EXPECT_EQ(attr.type, BCF_HT_STR); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrFmtGp) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("GP", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.number, 3); + EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); + EXPECT_EQ(attr.type, BCF_HT_REAL); + + if (fid) htslib::hts_close(fid); +} + +TEST(TestBcfHeader, BcfHdrFmtDs) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("DS", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.number, 1); + EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); + EXPECT_EQ(attr.type, BCF_HT_REAL); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrFmtErr) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_format("DOESNOTEXIST", &attr); + EXPECT_NE(status, 0); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrFilter) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_filter("PASS", &attr); + EXPECT_EQ(status, 0); + + status = hdr.get_filter("PASSING", &attr); + EXPECT_NE(status, 0); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrInfoEaf) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_info("EAF", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.vl_type, BCF_VL_VAR); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrInfoErc) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_info("ERC", &attr); + EXPECT_EQ(status, 0); + EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.vl_type, BCF_VL_VAR); + + if (fid) htslib::hts_close(fid); +} + + +TEST(TestBcfHeader, BcfHdrInfoErr) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_FALSE(hdr.isnull()); + + bcfio::BcfHdrAttr attr {}; + + int status = hdr.get_info("NOTAINFOMEMBER", &attr); + EXPECT_NE(status, 0); + + if (fid) htslib::hts_close(fid); +} + +TEST(TestBcfHeader, BcfHdrNull) { + htslib::htsFile *fid = htslib::hts_open("doesnotexist", "r"); + bcfio::BcfHeader hdr { fid }; + + EXPECT_TRUE(hdr.isnull()); + if (fid) htslib::hts_close(fid); +} + -TEST(TestHaplotypeVCFParser, Constructor) { - ReadBcf vcf { VCF_NAME, "" }; - printf("K founders %lu\n", vcf.k_founders()); - // EXPECT_EQ(vcf.n_samples(), 11); - // EXPECT_EQ(vcf.k_founders(), 8); +TEST(TestReadBcf, Constructor) { + bcfio::ReadBcf bcf { VCF_NAME }; + EXPECT_EQ(bcf.n_samples(), N_SAMPS); + EXPECT_EQ(bcf.k_founders(), K_FOUNDERS); } From c3c6730a09f0fe72d97f00ce0a743af129b75746 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 19 Nov 2025 12:16:52 -0800 Subject: [PATCH 04/58] added retrieval of sample names --- include/bcfio.h | 1 + src/bcfio.cpp | 11 +++++++++++ tests/test_bcfio.cpp | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/include/bcfio.h b/include/bcfio.h index 23c354f..dd89bed 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -92,6 +92,7 @@ class ReadBcf const size_t n_samples() const; const size_t k_founders() const; + std::unique_ptr sample_names() const; // bool load_record(HaplotypeDataRecord&); diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 92a1938..990beb7 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -52,6 +52,7 @@ bcfio::ReadBcf::ReadBcf(const char *variant_fname) hdr_(fid_) {}; +// TODO: subset samples by those in sample_fname file bcfio::ReadBcf::ReadBcf(const char *variant_fname, const char *sample_fname) : fname_(variant_fname), fid_(htslib::hts_open(variant_fname, "r")), @@ -100,3 +101,13 @@ const size_t bcfio::ReadBcf::k_founders() const { return static_cast(fmt.number); } +std::unique_ptr bcfio::ReadBcf::sample_names() const { + + std::unique_ptr samp_names = + std::make_unique(n_samples()); + + for (int i = 0; i < n_samples(); i++) + samp_names[i] = std::string(*(hdr_.hdr->samples + i)); + + return samp_names; +} diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index ac7b8ef..2ff570c 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -1,5 +1,8 @@ #include +#include +#include +#include namespace htslib { extern "C" { @@ -220,7 +223,46 @@ TEST(TestReadBcf, Constructor) { EXPECT_EQ(bcf.k_founders(), K_FOUNDERS); } +TEST(TestReadBcf, VcfSampNames) { + bcfio::ReadBcf bcf { VCF_NAME }; + + std::unique_ptr s = bcf.sample_names(); + + char samp_name[] = "S01"; + + for (int i = 0; i < bcf.n_samples(); i++) { + snprintf(samp_name, 4, "S%02d", i+1); + EXPECT_STREQ(s[i].c_str(), samp_name); + } +} + + +TEST(TestReadBcf, VcfGzSampNames) { + bcfio::ReadBcf bcf { VCFGZ_NAME }; + + std::unique_ptr s = bcf.sample_names(); + char samp_name[] = "S01"; + + for (int i = 0; i < bcf.n_samples(); i++) { + snprintf(samp_name, 4, "S%02d", i+1); + EXPECT_STREQ(s[i].c_str(), samp_name); + } +} + + +TEST(TestReadBcf, BcfSampNames) { + bcfio::ReadBcf bcf { BCF_NAME }; + + std::unique_ptr s = bcf.sample_names(); + + char samp_name[] = "S01"; + + for (int i = 0; i < bcf.n_samples(); i++) { + snprintf(samp_name, 4, "S%02d", i+1); + EXPECT_STREQ(s[i].c_str(), samp_name); + } +} // TEST(TestHaplotypeVCFParser, LoadRecord) { // // HaplotypeVcfParser vcf { VCF_NAME }; From 053d877c145c4b53798903b054ed199fc44bd0df Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 11 Dec 2025 11:48:51 -0500 Subject: [PATCH 05/58] updates --- include/bcfio.h | 45 ++++++++++++++++++++++++-------------------- src/bcfio.cpp | 22 ++++++++++++++++------ tests/test_bcfio.cpp | 2 ++ 3 files changed, 43 insertions(+), 26 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index dd89bed..65b989f 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -1,7 +1,5 @@ // Parse STITCH vcf file // -// -// // By: Robert Vogel // Affiliation: Palmer Lab at UCSD // Date: 2025-01-09 @@ -9,9 +7,6 @@ // // Acknowledgment // -// Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. // #ifndef HEADER_PARSE_HTS_H #define HEADER_PARSE_HTS_H @@ -29,7 +24,7 @@ extern "C" { } // samples are separated by white space -const char HAP_CODE[] { "HD" }; +// const char HAP_CODE[] { "HD" }; namespace bcfio { // @title The meta data on a BCF attribute @@ -41,12 +36,7 @@ namespace bcfio { // one needs to correctly implement bit shifting and masking. This struct // contains bit-fields representing each value stored in the uint64_t. // -struct BcfHdrAttr { - uint64_t number : 20; - uint64_t vl_type : 4; - uint64_t type : 4; - uint64_t coltype : 4; -}; +struct BcfHdrAttr { uint64_t number : 20, vl_type : 4, type : 4, coltype : 4; }; class BcfHeader { @@ -71,14 +61,32 @@ class BcfHeader { }; +// @title Manage bcf record +// @description Manage the lifetime of a htslib::bcf1_t type record using +// htslib functions with RAII. Provide some a simpler interface to +// quantities of interest +struct BcfRecord { + BcfRecord(): rec(htslib::bcf_init()) {}; + ~BcfRecord() { if (rec) htslib::bcf_destroy(rec); }; -// Interface with htslib bcf tools + bool is_snp() const { return htslib::bcf_is_snp(rec); } + htslib::bcf1_t *rec; +}; + + +// @title Interface with htslib bcf tools +// @description ReadBCF manages the lifetime of an open htslib file +// and organizes the bcf file header and any one record for easy +// and memory safe parsing. +// @param bcfname: the path and filename to the bcf file to be read. +// @param sample_fname: the path and filename of the text file listing the +// samples id's of records to be retreived. If this is not included +// all sample records are retrieved. class ReadBcf { public: - // HaplotypeVcfParser(const char* variant_fname); - ReadBcf(const char *variant_fname); - ReadBcf(const char *variant_fname, const char *sample_fname); + ReadBcf(const char *bcfname); + ReadBcf(const char *bcfname, const char *sample_fname); // HaplotypeVcfParser(const std::string& variant_fname); // HaplotypeVcfParser(const std::string& variant_fname, // const std::string& sample_fname); @@ -101,12 +109,9 @@ class ReadBcf htslib::htsFile *fid_; BcfHeader hdr_; - // size_t n_cols_ { 0 }; - // size_t n_samples_ { 0 }; - // size_t k_founders_ { 0 }; + int next_record(BcfRecord *rec); // size_t fpos_record_one_ { 0 }; - // void pos_(size_t); // size_t get_line_num_char_(); // void set_params_(); diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 990beb7..cdd1b2f 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -46,16 +46,16 @@ const int bcfio::BcfHeader::get_filter(const char *name, BcfHdrAttr *ptr) const } -bcfio::ReadBcf::ReadBcf(const char *variant_fname) - : fname_(variant_fname), - fid_(htslib::hts_open(variant_fname, "r")), +bcfio::ReadBcf::ReadBcf(const char *bcfname) + : fname_(bcfname), + fid_(htslib::hts_open(bcfname, "r")), hdr_(fid_) {}; // TODO: subset samples by those in sample_fname file -bcfio::ReadBcf::ReadBcf(const char *variant_fname, const char *sample_fname) - : fname_(variant_fname), - fid_(htslib::hts_open(variant_fname, "r")), +bcfio::ReadBcf::ReadBcf(const char *bcfname, const char *sample_fname) + : fname_(bcfname), + fid_(htslib::hts_open(bcfname, "r")), hdr_(fid_) { int status { 0 }; @@ -101,6 +101,7 @@ const size_t bcfio::ReadBcf::k_founders() const { return static_cast(fmt.number); } +// Note: May be better to just return a reference? std::unique_ptr bcfio::ReadBcf::sample_names() const { std::unique_ptr samp_names = @@ -111,3 +112,12 @@ std::unique_ptr bcfio::ReadBcf::sample_names() const { return samp_names; } + + +int bcfio::ReadBcf::next_record(bcfio::BcfRecord *ptr) { + int status = htslib::bcf_read(fid_, hdr_.hdr, ptr->rec); + if (status != 0) + return status; + + return htslib::bcf_unpack(ptr->rec, BCF_UN_ALL); +} diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index 2ff570c..c3d8198 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -263,6 +263,8 @@ TEST(TestReadBcf, BcfSampNames) { EXPECT_STREQ(s[i].c_str(), samp_name); } } + + // TEST(TestHaplotypeVCFParser, LoadRecord) { // // HaplotypeVcfParser vcf { VCF_NAME }; From d82567f987d3003d72a6ffd0c7774f0b99022275 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 11 Dec 2025 20:37:32 -0500 Subject: [PATCH 06/58] added options for different matrices, made place holders for computations that I need to complete. --- Makefile | 17 +++++---- include/calc.h | 6 ++++ src/main.cpp | 83 +++++++++++++++++++++++++++++++++++-------- src/use_both.cpp | 6 ++++ src/use_genotype.cpp | 6 ++++ src/use_haplotype.cpp | 6 ++++ tests/test_bcfio.cpp | 3 +- 7 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 include/calc.h create mode 100644 src/use_both.cpp create mode 100644 src/use_genotype.cpp create mode 100644 src/use_haplotype.cpp diff --git a/Makefile b/Makefile index 262a426..82f58e2 100644 --- a/Makefile +++ b/Makefile @@ -45,26 +45,29 @@ SRC_DIR = src HEADER_DIR = include BUILD_DIR = build -CXXLD += $(PWD)/include -CXXLD += $(LOCAL_LD) - +CXXLD := $(PWD)/include $(LOCAL_LD) $(CXXLD) CXXLDFLAGS = $(addprefix -I, $(CXXLD)) CXXLIB += $(LOCAL_LIB) CXXLIBFLAGS = $(addprefix -L, $(CXXLIB)) -APP_FILES = matrix.cpp bcfio.cpp -APP_SRC = $(addprefix $(SRC_DIR)/, $(APP_FILES)) -APP_OBJS = $(addprefix $(BUILD_DIR)/, $(APP_FILES:.cpp=.o)) +# APPLICATION FILES +# APP_FILES = matrix.cpp bcfio.cpp +APP_SRC = $(filter-out $(SRC_DIR)/main.cpp, $(wildcard $(SRC_DIR)/*.cpp)) +#APP_SRC = $(addprefix $(SRC_DIR)/, $(APP_FILES)) +APP_OBJS = $(subst $(SRC_DIR), $(BUILD_DIR), $(APP_SRC:.cpp=.o)) APP_DEPS = $(APP_OBJS:.o=.d) - +# TESTS TEST_DIR = tests TEST_SRC = $(wildcard $(TEST_DIR)/test_*.cpp) TEST_OBJS = $(subst $(TEST_DIR), $(BUILD_DIR), $(TEST_SRC:.cpp=.o)) TEST_DEPS = $(TEST_OBJS:.o=.d) + + TEST_DATA_SRC = $(wildcard $(TEST_DIR)/geno_test_data.*) TEST_DATA_DST = $(subst $(TEST_DIR), $(BUILD_DIR), $(TEST_DATA_SRC)) + TEST_TARGET_PRG = $(BUILD_DIR)/runtests diff --git a/include/calc.h b/include/calc.h new file mode 100644 index 0000000..7a54b33 --- /dev/null +++ b/include/calc.h @@ -0,0 +1,6 @@ + +int compute_genotype_matrix(); + +int compute_haplotype_matrix(); + +int compute_geno_and_haplo_matrix(); diff --git a/src/main.cpp b/src/main.cpp index 801385f..4ca4840 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -28,6 +28,13 @@ #include #include +#include + + +#define FAILED_CALC -1 +#define SUCCESS_CALC 0 + + size_t MARKER_PRINT_INTERVAL { 1000 }; char HELP_LONG_FLAG[] { "--help" }; @@ -47,16 +54,25 @@ int main(int argc, char* argv[]) "as a text file in the variant call format (VCF)." }; + parser.add_arg("-o", + argparse::ArgType::STRING, + "the path and filename that the resulting haplotype genetic" + "relationship matrix is printed."); + parser.add_arg("--sample_names", argparse::ArgType::STRING, "The path and name of the file containing sample names to be" " included in computing the relationship matrix. The file must" " include a single sample filename, and if necessary file system" " path, per line."); - parser.add_arg("-o", - argparse::ArgType::STRING, - "the path and filename that the resulting haplotype genetic" - "relationship matrix is printed."); + + parser.add_arg("--use_genotypes", + argparse::ArgType::BOOLEAN, + "Use sample genotypes to compute the relationship matrix"); + parser.add_arg("--use_both", + argparse::ArgType::BOOLEAN, + "Use both genotypes and haplotypes to compute relationship matrix"); + parser.add_arg("vcf", argparse::ArgType::STRING, "the path and filename of the vcf in which the hgrm is computed."); @@ -66,28 +82,48 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } - std::optional tmp {}; - if((tmp = parser.get("vcf")) == std::nullopt) { + std::optional tmp_str {}; + if((tmp_str = parser.get("vcf")) == std::nullopt) { fprintf(stderr, "Error retrieving vcf name"); exit(EXIT_FAILURE); } - std::string vcf_fname { tmp.value() }; + std::string vcf_fname { tmp_str.value() }; - if ((tmp = parser.get("o")) == std::nullopt) { + if ((tmp_str = parser.get("o")) == std::nullopt) { fprintf(stderr, "Error retrieving output name"); exit(EXIT_FAILURE); } - std::string out_fname { tmp.value() }; + std::string out_fname { tmp_str.value() }; if (out_fname.size() == 0) out_fname = vcf_fname + ".mat"; - std::string samp_fname {}; - if ((tmp = parser.get("sample_names")) == std::nullopt) { + if ((tmp_str = parser.get("sample_names")) == std::nullopt) { fprintf(stderr, "Error retrieving sample_names file.\n"); exit(EXIT_FAILURE); } - samp_fname = tmp.value(); + std::string samp_fname { tmp_str.value() }; + + + std::optional tmp_bool {}; + if ((tmp_bool = parser.get("use_genotypes")) == std::nullopt) { + fprintf(stderr, "Error retrieving relationship matrix type.\n"); + exit(EXIT_FAILURE); + } + bool use_genotypes { tmp_bool.value() }; + + if ((tmp_bool = parser.get("use_both")) == std::nullopt) { + fprintf(stderr, "Error retrieving relationship matrix type.\n"); + exit(EXIT_FAILURE); + } + bool use_both { tmp_bool.value() }; + + if (use_genotypes && use_both) { + fprintf(stderr, "user must specify either use_genotypes, use_both," + " or omit both options to compute the haplotype based" + " relationship matrix."); + exit(EXIT_FAILURE); + } fprintf(stdout, "BCF/VCF file name: %s\n", vcf_fname.c_str()); @@ -95,14 +131,33 @@ int main(int argc, char* argv[]) fprintf(stdout, "Sample file: None, use all samples\n"); else fprintf(stdout, "Sample file: %s\n", samp_fname.c_str()); + fprintf(stdout, "Output matrix file: %s\n", out_fname.c_str()); + + int status = FAILED_CALC; + + if (use_genotypes) { + fprintf(stdout, "Relationship matrix: genotype\n"); + status = compute_genotype_matrix(); + } else if (use_both) { + fprintf(stdout, "Relationship matrix: genotype and haplotype\n"); + status = compute_geno_and_haplo_matrix(); + } else { + fprintf(stdout, "Relationship matrix: haplotype\n"); + status = compute_haplotype_matrix(); + } + + if (status == FAILED_CALC) + fprintf(stderr, "%s\n", "Computation failed"); + + // const std::chrono::time_point timer; // { std::chrono::steady_clock::now() }; // HaplotypeVcfParser vcf_data { filename_input, 100000 }; - fprintf(stdout, "Allocating memory\n"); +// fprintf(stdout, "Allocating memory\n"); // instantiate matrices to hold calculations // Matrix covariance { vcf_data.n_samples(), vcf_data.n_samples() }; @@ -203,5 +258,5 @@ int main(int argc, char* argv[]) // fprintf(stdout, "Done, elapsed time %lld second(s)\n", // std::chrono::duration_cast(delta_t).count()); // - return 0; + return status; } diff --git a/src/use_both.cpp b/src/use_both.cpp new file mode 100644 index 0000000..f3011c0 --- /dev/null +++ b/src/use_both.cpp @@ -0,0 +1,6 @@ + +#include + +int compute_geno_and_haplo_matrix() { + return -1; +} diff --git a/src/use_genotype.cpp b/src/use_genotype.cpp new file mode 100644 index 0000000..99cb7f7 --- /dev/null +++ b/src/use_genotype.cpp @@ -0,0 +1,6 @@ + +#include + +int compute_genotype_matrix() { + return -1; +} diff --git a/src/use_haplotype.cpp b/src/use_haplotype.cpp new file mode 100644 index 0000000..e5fd431 --- /dev/null +++ b/src/use_haplotype.cpp @@ -0,0 +1,6 @@ + +#include + +int compute_haplotype_matrix() { + return -1; +} diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index c3d8198..68b42ab 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -208,7 +208,8 @@ TEST(TestBcfHeader, BcfHdrInfoErr) { } TEST(TestBcfHeader, BcfHdrNull) { - htslib::htsFile *fid = htslib::hts_open("doesnotexist", "r"); + // htslib::htsFile *fid = htslib::hts_open("doesnotexist", "r"); + htslib::htsFile *fid = nullptr; bcfio::BcfHeader hdr { fid }; EXPECT_TRUE(hdr.isnull()); From e238d212c64ba120e1fa174ce1f133df48bce8dd Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 12 Dec 2025 18:15:32 -0500 Subject: [PATCH 07/58] IMPORTANT: not working. Added logger tool and applied to the main.cpp. I can run the program and all inputs but --help are parsed correctly. --- include/logger.h | 40 +++++++++++ src/logger.cpp | 96 +++++++++++++++++++++++++++ src/main.cpp | 150 ++++++------------------------------------ src/use_haplotype.cpp | 107 ++++++++++++++++++++++++++++++ 4 files changed, 262 insertions(+), 131 deletions(-) create mode 100644 include/logger.h create mode 100644 src/logger.cpp diff --git a/include/logger.h b/include/logger.h new file mode 100644 index 0000000..e95a36d --- /dev/null +++ b/include/logger.h @@ -0,0 +1,40 @@ + +#include +#include +#include +#include + +class Logger { +public: + Logger(); + ~Logger(); + + int info(const char *format, const char *msg); + int warn(const char *format, const char *msg); + int error(const char *format, const char *msg); + + int info(const char *msg); + int warn(const char *msg); + int error(const char *msg); + +private: + time_t t_; + tm *time_point_; + char *time_buf_; + char *str_buf_; + + int msg_len_ { 0 }; + size_t time_len_ { 0 }; + + static const size_t time_buf_len_ { 30 }; + static const size_t str_buf_len_ { 500 }; + static const size_t max_str_ { 450 }; + + static constexpr char err_str_[] = { "ERROR" }; + static constexpr char warn_str_[] = { "WARN" }; + static constexpr char info_str_[] = { "INFO" }; + + int print_(FILE *stream, const char *log_type, + const char *format, const char *msg); + void empty_time_buf_(); +}; diff --git a/src/logger.cpp b/src/logger.cpp new file mode 100644 index 0000000..ee04646 --- /dev/null +++ b/src/logger.cpp @@ -0,0 +1,96 @@ + +#include + + +Logger::Logger(): + t_(time(nullptr)), + time_point_(localtime(&t_)), + time_buf_(new char[time_buf_len_]), + str_buf_(new char[str_buf_len_]) { + + if (!time_buf_ || !str_buf_) { + fprintf(stderr, "logger failure. please notify maintainer"); + exit(EXIT_FAILURE); + } + + for (int i = max_str_; i < str_buf_len_; i++) + str_buf_[i] = '\0'; + + for (int i = 0; i < time_buf_len_; i++) + time_buf_[i] = '\0'; +}; + +Logger::~Logger() { + if (time_buf_) delete[] time_buf_; + if (str_buf_) delete[] str_buf_; +} + +int Logger::print_(FILE *stream, + const char *log_type, + const char *format, + const char *msg) { + + // get time and format time string + t_ = time(nullptr); + time_point_ = localtime(&t_); + + + // strftime returns the number of characters written to buffer, + // a 0 returned indicates an error has occured. + time_len_ = strftime(time_buf_, time_buf_len_,"%FT%H:%M:%S", time_point_); + + if(time_len_ == 0) { + empty_time_buf_(); + + fprintf(stderr, "%s\t%s\t%s\n", time_buf_, + err_str_, "logger time buf failure, please notify maintainer."); + return -1; + } + + // construct logging message + // TODO: truncation of msg notification when msg exceeds buffer + // Recall that snprintf returns int less than 0 if an error occurs + msg_len_ = snprintf(str_buf_, max_str_, format, msg); + if (msg_len_ < 0) { + fprintf(stderr, "%s\t%s\t%s\n", time_buf_, + err_str_, "logger msg failure, please notify maintainer."); + return -1; + } + + fprintf(stdout, "%s\t%s\t%s\n", time_buf_, log_type, str_buf_); + + return 0; +} + +int Logger::info(const char *format, const char *msg) { + return print_(stdout, info_str_, format, msg); +} + +int Logger::warn(const char *format, const char *msg) { + return print_(stdout, warn_str_, format, msg); +} + +int Logger::error(const char *format, const char *msg) { + return print_(stderr, err_str_, format, msg); +} + +int Logger::info(const char *msg) { + return print_(stdout, info_str_, "%s", msg); +} + +int Logger::warn(const char *msg) { + return print_(stdout, warn_str_, "%s", msg); +} + +int Logger::error(const char *msg) { + return print_(stderr, err_str_, "%s", msg); +} + +void Logger::empty_time_buf_() { + int i = 0; + for (; i < time_buf_len_; i++) + time_buf_[i] = '0'; + + time_buf_[i] = '\0'; +} + diff --git a/src/main.cpp b/src/main.cpp index 4ca4840..5160077 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -13,21 +13,13 @@ // of the haplotype based covariance I had to derive the recursion // relations myself. // -// -// -// Acknowledgment -// -// Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. -// #include #include -// #include #include #include #include +#include #include @@ -35,17 +27,16 @@ #define SUCCESS_CALC 0 +const size_t STR_BUF_LEN { 500 }; +char STR_BUF[STR_BUF_LEN]; -size_t MARKER_PRINT_INTERVAL { 1000 }; -char HELP_LONG_FLAG[] { "--help" }; -char HELP_SHORT_FLAG[] { "-h" }; int main(int argc, char* argv[]) { - if (argc != 2 && argc != 4) { - fprintf(stderr, "Incorrect input, see --help for correct usage.\n"); - exit(EXIT_FAILURE); - } + // if (argc != 2 && argc != 4) { + // fprintf(stderr, "Incorrect input, see --help for correct usage.\n"); + // exit(EXIT_FAILURE); + // } argparse::ArgParser parser { "hgrm: Haplotype Genetic Relationship Matrix", @@ -82,6 +73,8 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } + // TODO: Update below to use logger + // std::optional tmp_str {}; if((tmp_str = parser.get("vcf")) == std::nullopt) { fprintf(stderr, "Error retrieving vcf name"); @@ -126,137 +119,32 @@ int main(int argc, char* argv[]) } - fprintf(stdout, "BCF/VCF file name: %s\n", vcf_fname.c_str()); + Logger log {}; + + log.info("BCF/VCF file name: %s", vcf_fname.c_str()); if (samp_fname.size() == 0) - fprintf(stdout, "Sample file: None, use all samples\n"); + log.info("Sample file: None, use all samples"); else - fprintf(stdout, "Sample file: %s\n", samp_fname.c_str()); - - fprintf(stdout, "Output matrix file: %s\n", out_fname.c_str()); + log.info("Sample file: %s", samp_fname.c_str()); + log.info("Output matrix file: %s", out_fname.c_str()); int status = FAILED_CALC; if (use_genotypes) { - fprintf(stdout, "Relationship matrix: genotype\n"); + log.info("Relationship matrix: genotype"); status = compute_genotype_matrix(); } else if (use_both) { - fprintf(stdout, "Relationship matrix: genotype and haplotype\n"); + log.info("Relationship matrix: genotype and haplotype"); status = compute_geno_and_haplo_matrix(); } else { - fprintf(stdout, "Relationship matrix: haplotype\n"); + log.info("Relationship matrix: haplotype"); status = compute_haplotype_matrix(); } if (status == FAILED_CALC) - fprintf(stderr, "%s\n", "Computation failed"); - + log.error("Computation failed"); - // const std::chrono::time_point timer; - // { std::chrono::steady_clock::now() }; - -// HaplotypeVcfParser vcf_data { filename_input, 100000 }; - -// fprintf(stdout, "Allocating memory\n"); - // instantiate matrices to hold calculations -// Matrix covariance { vcf_data.n_samples(), vcf_data.n_samples() }; -// -// // open VCF file and parse meta data and header -// HaplotypeVcfParser vcf_data { filename_input, 100000 }; -// -// - -// // instantiate record object -// HaplotypeDataRecord record { vcf_data.n_samples(), vcf_data.k_founders() }; -// -// // analyze each line, i.e. position, in the VCF -// size_t m_markers { 1 }; -// -// double sum { 0 }; -// const double* rowi { nullptr }; -// const double* rowj { nullptr }; -// double* rowi_cov { nullptr }; -// const size_t k_founders { vcf_data.k_founders() }; -// const size_t n_samples { vcf_data.n_samples() }; -// -// std::chrono::steady_clock::duration delta_t -// { std::chrono::steady_clock::now() - timer }; -// -// fprintf(stdout, "Computing matrix, elapsed time %lld second(s)\n", -// std::chrono::duration_cast(delta_t).count()); -// -// while(vcf_data.load_record(record)) { -// -// // for each founder, compute first and second moments -// for (size_t i = 0; i < n_samples; i++) { -// -// rowi = &record(i, 0); -// rowi_cov = &covariance(i, 0); -// -// for (size_t j = i; j < n_samples; j++) { -// -// rowj = &record(j,0); -// sum = 0; -// -// for (int k = 0; k < k_founders; k++) -// sum += rowi[k] * rowj[k]; -// -// rowi_cov[j] += sum; -// } -// } -// -// if (m_markers % MARKER_PRINT_INTERVAL == 0) { -// delta_t = std::chrono::steady_clock::now() - timer; -// -// fprintf(stdout, "Completed %zu marker loci, elapsed time %lld second(s)\n", -// m_markers, -// std::chrono::duration_cast(delta_t).count()); -// } -// -// m_markers++; -// -// } -// -// -// FILE* fout = stdout; -// -// if (argc == 3 && filename_output != nullptr) { -// -// if ((fout = fopen(filename_output, "w")) == nullptr) -// throw std::runtime_error("Error in opening file for writing."); -// -// delta_t = std::chrono::steady_clock::now() - timer; -// fprintf(stdout, "Writing results to file %s, elapsed time %lld second(s)\n", -// filename_output, -// std::chrono::duration_cast(delta_t).count()); -// -// } else if (argc == 3 && filename_output == nullptr) -// throw std::runtime_error("Output filename is not specified"); -// -// -// size_t i { 0 }; -// size_t j { 0 }; -// for (i = 0; i < n_samples; i++) { -// -// for (j = 0; j < n_samples-1; j++) { -// if (j < i) -// fprintf(fout, "%0.5f,", covariance(j,i)); -// else -// fprintf(fout, "%0.5f,", covariance(i,j)); -// -// } -// -// fprintf(fout,"%0.5f\n", covariance(i, j)); -// } -// -// fclose(fout); -// -// -// delta_t = std::chrono::steady_clock::now() - timer; -// -// fprintf(stdout, "Done, elapsed time %lld second(s)\n", -// std::chrono::duration_cast(delta_t).count()); -// return status; } diff --git a/src/use_haplotype.cpp b/src/use_haplotype.cpp index e5fd431..a8253e4 100644 --- a/src/use_haplotype.cpp +++ b/src/use_haplotype.cpp @@ -4,3 +4,110 @@ int compute_haplotype_matrix() { return -1; } + + // const std::chrono::time_point timer; + // { std::chrono::steady_clock::now() }; + +// HaplotypeVcfParser vcf_data { filename_input, 100000 }; + +// fprintf(stdout, "Allocating memory\n"); + // instantiate matrices to hold calculations +// Matrix covariance { vcf_data.n_samples(), vcf_data.n_samples() }; + +// +// // open VCF file and parse meta data and header +// HaplotypeVcfParser vcf_data { filename_input, 100000 }; +// +// + +// // instantiate record object +// HaplotypeDataRecord record { vcf_data.n_samples(), vcf_data.k_founders() }; +// +// // analyze each line, i.e. position, in the VCF +// size_t m_markers { 1 }; +// +// double sum { 0 }; +// const double* rowi { nullptr }; +// const double* rowj { nullptr }; +// double* rowi_cov { nullptr }; +// const size_t k_founders { vcf_data.k_founders() }; +// const size_t n_samples { vcf_data.n_samples() }; +// +// std::chrono::steady_clock::duration delta_t +// { std::chrono::steady_clock::now() - timer }; +// +// fprintf(stdout, "Computing matrix, elapsed time %lld second(s)\n", +// std::chrono::duration_cast(delta_t).count()); +// +// while(vcf_data.load_record(record)) { +// +// // for each founder, compute first and second moments +// for (size_t i = 0; i < n_samples; i++) { +// +// rowi = &record(i, 0); +// rowi_cov = &covariance(i, 0); +// +// for (size_t j = i; j < n_samples; j++) { +// +// rowj = &record(j,0); +// sum = 0; +// +// for (int k = 0; k < k_founders; k++) +// sum += rowi[k] * rowj[k]; +// +// rowi_cov[j] += sum; +// } +// } +// +// if (m_markers % MARKER_PRINT_INTERVAL == 0) { +// delta_t = std::chrono::steady_clock::now() - timer; +// +// fprintf(stdout, "Completed %zu marker loci, elapsed time %lld second(s)\n", +// m_markers, +// std::chrono::duration_cast(delta_t).count()); +// } +// +// m_markers++; +// +// } +// +// +// FILE* fout = stdout; +// +// if (argc == 3 && filename_output != nullptr) { +// +// if ((fout = fopen(filename_output, "w")) == nullptr) +// throw std::runtime_error("Error in opening file for writing."); +// +// delta_t = std::chrono::steady_clock::now() - timer; +// fprintf(stdout, "Writing results to file %s, elapsed time %lld second(s)\n", +// filename_output, +// std::chrono::duration_cast(delta_t).count()); +// +// } else if (argc == 3 && filename_output == nullptr) +// throw std::runtime_error("Output filename is not specified"); +// +// +// size_t i { 0 }; +// size_t j { 0 }; +// for (i = 0; i < n_samples; i++) { +// +// for (j = 0; j < n_samples-1; j++) { +// if (j < i) +// fprintf(fout, "%0.5f,", covariance(j,i)); +// else +// fprintf(fout, "%0.5f,", covariance(i,j)); +// +// } +// +// fprintf(fout,"%0.5f\n", covariance(i, j)); +// } +// +// fclose(fout); +// +// +// delta_t = std::chrono::steady_clock::now() - timer; +// +// fprintf(stdout, "Done, elapsed time %lld second(s)\n", +// std::chrono::duration_cast(delta_t).count()); +// From 97800d8ee86b69d3e93c2a16c6f1d223ca81fa15 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 12 Dec 2025 20:53:09 -0500 Subject: [PATCH 08/58] simplified logger --- include/logger.h | 15 +++++++-------- src/logger.cpp | 31 ++++--------------------------- 2 files changed, 11 insertions(+), 35 deletions(-) diff --git a/include/logger.h b/include/logger.h index e95a36d..4a28a1a 100644 --- a/include/logger.h +++ b/include/logger.h @@ -2,12 +2,11 @@ #include #include #include -#include +#include class Logger { public: Logger(); - ~Logger(); int info(const char *format, const char *msg); int warn(const char *format, const char *msg); @@ -20,15 +19,16 @@ class Logger { private: time_t t_; tm *time_point_; - char *time_buf_; - char *str_buf_; int msg_len_ { 0 }; size_t time_len_ { 0 }; - static const size_t time_buf_len_ { 30 }; - static const size_t str_buf_len_ { 500 }; - static const size_t max_str_ { 450 }; + static constexpr size_t time_buf_len_ { 30 }; + static constexpr size_t str_buf_len_ { 500 }; + static constexpr size_t max_str_ { 450 }; + + char time_buf_[time_buf_len_]; + char str_buf_[str_buf_len_]; static constexpr char err_str_[] = { "ERROR" }; static constexpr char warn_str_[] = { "WARN" }; @@ -36,5 +36,4 @@ class Logger { int print_(FILE *stream, const char *log_type, const char *format, const char *msg); - void empty_time_buf_(); }; diff --git a/src/logger.cpp b/src/logger.cpp index ee04646..d2a7904 100644 --- a/src/logger.cpp +++ b/src/logger.cpp @@ -4,27 +4,12 @@ Logger::Logger(): t_(time(nullptr)), - time_point_(localtime(&t_)), - time_buf_(new char[time_buf_len_]), - str_buf_(new char[str_buf_len_]) { + time_point_(localtime(&t_)) { - if (!time_buf_ || !str_buf_) { - fprintf(stderr, "logger failure. please notify maintainer"); - exit(EXIT_FAILURE); - } - - for (int i = max_str_; i < str_buf_len_; i++) - str_buf_[i] = '\0'; - - for (int i = 0; i < time_buf_len_; i++) - time_buf_[i] = '\0'; + std::memset(time_buf_, '\0', time_buf_len_); + std::memset(str_buf_, '\0', str_buf_len_); }; -Logger::~Logger() { - if (time_buf_) delete[] time_buf_; - if (str_buf_) delete[] str_buf_; -} - int Logger::print_(FILE *stream, const char *log_type, const char *format, @@ -40,7 +25,7 @@ int Logger::print_(FILE *stream, time_len_ = strftime(time_buf_, time_buf_len_,"%FT%H:%M:%S", time_point_); if(time_len_ == 0) { - empty_time_buf_(); + std::memset(time_buf_, '\0', time_buf_len_); fprintf(stderr, "%s\t%s\t%s\n", time_buf_, err_str_, "logger time buf failure, please notify maintainer."); @@ -86,11 +71,3 @@ int Logger::error(const char *msg) { return print_(stderr, err_str_, "%s", msg); } -void Logger::empty_time_buf_() { - int i = 0; - for (; i < time_buf_len_; i++) - time_buf_[i] = '0'; - - time_buf_[i] = '\0'; -} - From fd1d7fb914597c4b718020bdd83078641ac764c1 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 12 Dec 2025 20:59:59 -0500 Subject: [PATCH 09/58] added todo to make the logger accept more than one message string. Fixed bug where I don't use the specified data stream to print logger record. --- include/logger.h | 3 +++ src/logger.cpp | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/logger.h b/include/logger.h index 4a28a1a..2f78ddc 100644 --- a/include/logger.h +++ b/include/logger.h @@ -8,6 +8,9 @@ class Logger { public: Logger(); + // TODO: right now only accepts a single msg string, I should make + // this arbitrary message elements using va_list, this makes the + // interface match that of sprintf int info(const char *format, const char *msg); int warn(const char *format, const char *msg); int error(const char *format, const char *msg); diff --git a/src/logger.cpp b/src/logger.cpp index d2a7904..9f07ae3 100644 --- a/src/logger.cpp +++ b/src/logger.cpp @@ -42,7 +42,7 @@ int Logger::print_(FILE *stream, return -1; } - fprintf(stdout, "%s\t%s\t%s\n", time_buf_, log_type, str_buf_); + fprintf(stream, "%s\t%s\t%s\n", time_buf_, log_type, str_buf_); return 0; } From b320b886a177de78c13d7f336cad2d472aa73274 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 12 Dec 2025 21:24:58 -0500 Subject: [PATCH 10/58] workflow --- .github/workflows/c-cpp.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/c-cpp.yml diff --git a/.github/workflows/c-cpp.yml b/.github/workflows/c-cpp.yml new file mode 100644 index 0000000..0a8d354 --- /dev/null +++ b/.github/workflows/c-cpp.yml @@ -0,0 +1,28 @@ +name: C/C++ CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + +jobs: + build: + runs-on: ${{ matrix.config.os }} + name: ${{ matrix.config.os }} + strategy: + matrix: + os: [ ubuntu-latest, ubuntu-latest-arm, macos-15 ] + + steps: + - uses: actions/checkout@v4 +# - name: configure +# run: ./configure + - name: make + run: make + - name: make check + run: make check +# - name: make distcheck +# run: make distcheck + From 3ec9aefad78746d5e99a4b52beec3e601c0a5216 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 13 Dec 2025 12:40:01 -0500 Subject: [PATCH 11/58] added a new github action. --- .github/workflows/main.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..c5185ae --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: Main C/C++ CI + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - name: clone repo + uses: actions/checkout@v4 + - name: make + run: make + - name: make check + run: make check + +# - name: make distcheck +# run: make distcheck +# - name: configure +# run: ./configure + From 7c5ab04696ccc23e772c145bdb7bbf4263469b2b Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 16 Dec 2025 16:11:56 -0500 Subject: [PATCH 12/58] added record parsing, but no computations are complete, just trying to correctly extract data. --- include/bcfio.h | 62 ++++++++++++++++++++++++++++++++----------- include/calc.h | 14 +++++++++- include/logger.h | 5 ++++ include/matrix.h | 2 +- src/bcfio.cpp | 47 +++++++++++++++++++++++--------- src/main.cpp | 13 ++++++--- src/matrix.cpp | 2 +- src/use_haplotype.cpp | 51 ++++++++++++++++++++++------------- tests/test_bcfio.cpp | 4 +-- 9 files changed, 144 insertions(+), 56 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index 65b989f..2d1013a 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -27,29 +27,54 @@ extern "C" { // const char HAP_CODE[] { "HD" }; namespace bcfio { + // @title The meta data on a BCF attribute // @description BCF, VCF, and VCF.GZ files hold metadata in the header that // specify the type and format of data in records. I call each unique // piece of data in a record a record attribute, e.g. an INFO column or // FORMAT column of a record are attributes of that record. HTSLIB encodes -// attribute information an unsigned 64 bit integer, and to access any value -// one needs to correctly implement bit shifting and masking. This struct -// contains bit-fields representing each value stored in the uint64_t. -// +// attribute information in an unsigned 64 bit integer, and to access any +// value one needs to correctly implement bit shifting and masking. This +// struct contains bit-fields representing each value stored in the +// uint64_t. +// @bitfield number: the number of distinct values required to specify a sample +// record at loci i. For example, a SNP genotype is specified by a single +// string, e.g. 0/1, while the posterior genotype (0/0, 0/1, 1/1) +// probabilities requires three numbers. +// @bitfield vl_type: Specifies whether a variable is fixed length (BCF_VL_FIXED, +// in htslib/vcf.h line 68), variable length, etc. +// @bitfield type: the type of variable: binary flag (BCF_HT_FLAG), integer, +// real number, string, and 64 bit integers. Note that HT is header type. +// @bitfield coltype: struct BcfHdrAttr { uint64_t number : 20, vl_type : 4, type : 4, coltype : 4; }; +// @title: Manage bcf header +// @description: The bcf header C-struct requires manual allocation and release +// of memory. This class manages applies RAII, reducing the chance of a +// memory leak. class BcfHeader { public: htslib::bcf_hdr_t *hdr; BcfHeader(htslib::htsFile *fid): hdr(fid ? htslib::bcf_hdr_read(fid) : nullptr) {}; - ~BcfHeader() { if (hdr) htslib::bcf_hdr_destroy(hdr); }; + + ~BcfHeader() { + if (hdr) htslib::bcf_hdr_destroy(hdr); + hdr = nullptr; + }; const bool isnull() const { return hdr == nullptr; }; // sample_names() + + // @title: "get_*" member functions for info retrieval + // @description: + // @param name: the id of the formatted data field to retrieve + // @param ptr: the pointer to memory for which the BcfHdrAttr + // data will be copied into memory. + // @return const int get_format(const char *name, BcfHdrAttr *ptr) const; const int get_info(const char *name, BcfHdrAttr *ptr) const; const int get_filter(const char *name, BcfHdrAttr *ptr) const; @@ -63,18 +88,23 @@ class BcfHeader { // @title Manage bcf record // @description Manage the lifetime of a htslib::bcf1_t type record using -// htslib functions with RAII. Provide some a simpler interface to -// quantities of interest +// htslib functions with RAII. struct BcfRecord { BcfRecord(): rec(htslib::bcf_init()) {}; - ~BcfRecord() { if (rec) htslib::bcf_destroy(rec); }; + ~BcfRecord(); bool is_snp() const { return htslib::bcf_is_snp(rec); } + + int get_fmt(BcfHeader *hdr, const char *tag); + htslib::bcf1_t *rec; + + int ndst = 0; + float *dst = nullptr; }; -// @title Interface with htslib bcf tools +// @title Interface with htslib bcf // @description ReadBCF manages the lifetime of an open htslib file // and organizes the bcf file header and any one record for easy // and memory safe parsing. @@ -84,6 +114,10 @@ struct BcfRecord { // all sample records are retrieved. class ReadBcf { +private: + const std::string fname_; + htslib::htsFile *fid_; + public: ReadBcf(const char *bcfname); ReadBcf(const char *bcfname, const char *sample_fname); @@ -99,17 +133,13 @@ class ReadBcf ~ReadBcf(); const size_t n_samples() const; - const size_t k_founders() const; + const size_t k_haps() const; std::unique_ptr sample_names() const; - // bool load_record(HaplotypeDataRecord&); + int next_record(BcfRecord *rec); -private: - const std::string fname_; - htslib::htsFile *fid_; - BcfHeader hdr_; + BcfHeader hdr; - int next_record(BcfRecord *rec); // size_t fpos_record_one_ { 0 }; // void pos_(size_t); diff --git a/include/calc.h b/include/calc.h index 7a54b33..b9c3052 100644 --- a/include/calc.h +++ b/include/calc.h @@ -1,6 +1,18 @@ +#ifndef HEADER_COV_CALC_H +#define HEADER_COV_CALC_H + +#include + +#include +#include +#include + int compute_genotype_matrix(); -int compute_haplotype_matrix(); +// +int compute_haplotype_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov); int compute_geno_and_haplo_matrix(); + +#endif diff --git a/include/logger.h b/include/logger.h index 2f78ddc..78665c9 100644 --- a/include/logger.h +++ b/include/logger.h @@ -1,4 +1,7 @@ +#ifndef HEADER_LOGGER_H +#define HEADER_LOGGER_H + #include #include #include @@ -40,3 +43,5 @@ class Logger { int print_(FILE *stream, const char *log_type, const char *format, const char *msg); }; + +#endif diff --git a/include/matrix.h b/include/matrix.h index 821232e..dce983c 100644 --- a/include/matrix.h +++ b/include/matrix.h @@ -23,7 +23,7 @@ class Matrix { public: - Matrix(size_t, size_t); // constructorconstructor + Matrix(const size_t, const size_t); Matrix(const Matrix&); // copy constructor Matrix(Matrix&&); // move constructor Matrix& operator=(const Matrix&)=delete; // copy assignment diff --git a/src/bcfio.cpp b/src/bcfio.cpp index cdd1b2f..d72646b 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -12,12 +12,19 @@ #include #include -// decoder based upon htslib/vcf.h line 100 in the typedef struct bcf_idinfo_t. +// @title: +// @description decoder based upon htslib/vcf.h line 100 in the typedef +// struct bcf_idinfo_t. +// @param name: +// @param bcf_dt_type +// @param ptr +// @return -1 indicates an error has occured and 0 a success const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, const int bcf_dt_type, bcfio::BcfHdrAttr *ptr) const { - // BCF_DT_ID is the ID dictionary index defined by htslib + // BCF_DT_ID is the C macro for the ID dictionary index defined by htslib + // see htslib/vcf.h line 86 int idx = htslib::bcf_hdr_id2int(hdr, BCF_DT_ID, name); if (idx < 0) @@ -45,26 +52,39 @@ const int bcfio::BcfHeader::get_filter(const char *name, BcfHdrAttr *ptr) const return decode_hts_idinfo_(name, BCF_HL_FLT, ptr); } +bcfio::BcfRecord::~BcfRecord() { + if (rec) htslib::bcf_destroy(rec); + + // TODO: double check destructor of dst + if (dst) delete[] dst; +} + +int bcfio::BcfRecord::get_fmt(bcfio::BcfHeader *hdr, const char *tag) { + + return htslib::bcf_get_format_values(hdr->hdr, rec, tag, + (void**)(&dst), &ndst, BCF_HT_REAL); +} + bcfio::ReadBcf::ReadBcf(const char *bcfname) : fname_(bcfname), fid_(htslib::hts_open(bcfname, "r")), - hdr_(fid_) {}; + hdr(fid_) {}; // TODO: subset samples by those in sample_fname file bcfio::ReadBcf::ReadBcf(const char *bcfname, const char *sample_fname) : fname_(bcfname), fid_(htslib::hts_open(bcfname, "r")), - hdr_(fid_) { + hdr(fid_) { int status { 0 }; // Subset samples with those found in the file sample_fname if (!sample_fname || *sample_fname == '\0') - fprintf(stdout, "No file with sample names detected, computing" - "hGRM over all samples.\n"); + fprintf(stdout, "No file with sample names detected, retreiving" + " records for all samples.\n"); else - status = htslib::bcf_hdr_set_samples(hdr_.hdr, sample_fname, 1); + status = htslib::bcf_hdr_set_samples(hdr.hdr, sample_fname, 1); if (status < 0) { fprintf(stderr, "Error: Couldn't read sample file\n"); @@ -89,13 +109,13 @@ const size_t bcfio::ReadBcf::n_samples() const { // Remember that n is the number of entries in the triplet of // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n // that correspondes to the number of samples. - return hdr_.hdr->n[BCF_DT_SAMPLE]; + return hdr.hdr->n[BCF_DT_SAMPLE]; }; -const size_t bcfio::ReadBcf::k_founders() const { +const size_t bcfio::ReadBcf::k_haps() const { BcfHdrAttr fmt {}; - if (hdr_.get_format("HD", &fmt) < 0) + if (hdr.get_format("HD", &fmt) < 0) printf("errror\n"); return static_cast(fmt.number); @@ -108,16 +128,17 @@ std::unique_ptr bcfio::ReadBcf::sample_names() const { std::make_unique(n_samples()); for (int i = 0; i < n_samples(); i++) - samp_names[i] = std::string(*(hdr_.hdr->samples + i)); + samp_names[i] = std::string(*(hdr.hdr->samples + i)); return samp_names; } - +// title: load next record int bcfio::ReadBcf::next_record(bcfio::BcfRecord *ptr) { - int status = htslib::bcf_read(fid_, hdr_.hdr, ptr->rec); + int status = htslib::bcf_read(fid_, hdr.hdr, ptr->rec); if (status != 0) return status; + // Unpacking options defined in htslib/vcf.h line 419 return htslib::bcf_unpack(ptr->rec, BCF_UN_ALL); } diff --git a/src/main.cpp b/src/main.cpp index 5160077..4509283 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,6 +21,7 @@ #include #include +#include #define FAILED_CALC -1 @@ -41,14 +42,14 @@ int main(int argc, char* argv[]) argparse::ArgParser parser { "hgrm: Haplotype Genetic Relationship Matrix", "This program computes the haplotype genetic relationship matrix" - "from the expected haplotype counts per locus per sample and stored" - "as a text file in the variant call format (VCF)." + " from the expected haplotype counts per locus per sample and stored" + " as a text file in the variant call format (VCF)." }; parser.add_arg("-o", argparse::ArgType::STRING, "the path and filename that the resulting haplotype genetic" - "relationship matrix is printed."); + " relationship matrix is printed."); parser.add_arg("--sample_names", argparse::ArgType::STRING, @@ -60,6 +61,7 @@ int main(int argc, char* argv[]) parser.add_arg("--use_genotypes", argparse::ArgType::BOOLEAN, "Use sample genotypes to compute the relationship matrix"); + parser.add_arg("--use_both", argparse::ArgType::BOOLEAN, "Use both genotypes and haplotypes to compute relationship matrix"); @@ -131,6 +133,9 @@ int main(int argc, char* argv[]) int status = FAILED_CALC; + bcfio::ReadBcf bfid { vcf_fname.c_str() }; + Matrix cov { bfid.n_samples(), bfid.n_samples() }; + if (use_genotypes) { log.info("Relationship matrix: genotype"); status = compute_genotype_matrix(); @@ -139,7 +144,7 @@ int main(int argc, char* argv[]) status = compute_geno_and_haplo_matrix(); } else { log.info("Relationship matrix: haplotype"); - status = compute_haplotype_matrix(); + status = compute_haplotype_matrix(&log, &bfid, &cov); } if (status == FAILED_CALC) diff --git a/src/matrix.cpp b/src/matrix.cpp index 3c072a7..3449980 100644 --- a/src/matrix.cpp +++ b/src/matrix.cpp @@ -16,7 +16,7 @@ #include // default constructor -Matrix::Matrix(size_t nrow, size_t mcol) +Matrix::Matrix(const size_t nrow, const size_t mcol) : nrow_(nrow), mcol_(mcol), data_(nrow_ > 0 && mcol_ > 0 ? std::make_unique(size()) : nullptr) { diff --git a/src/use_haplotype.cpp b/src/use_haplotype.cpp index a8253e4..e6e4180 100644 --- a/src/use_haplotype.cpp +++ b/src/use_haplotype.cpp @@ -1,29 +1,44 @@ + #include -int compute_haplotype_matrix() { - return -1; -} +int compute_haplotype_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov) { - // const std::chrono::time_point timer; - // { std::chrono::steady_clock::now() }; - -// HaplotypeVcfParser vcf_data { filename_input, 100000 }; + int output_status = -1; -// fprintf(stdout, "Allocating memory\n"); // instantiate matrices to hold calculations -// Matrix covariance { vcf_data.n_samples(), vcf_data.n_samples() }; + const size_t n_samples { bfid->n_samples() }; + const size_t k_haps { bfid->k_haps() }; + + size_t idx_row { 0 }; + // size_t idx_col { 0 }; + size_t idx_hap { 0 }; + size_t idx_rec { 0 }; -// -// // open VCF file and parse meta data and header -// HaplotypeVcfParser vcf_data { filename_input, 100000 }; -// -// + bcfio::BcfRecord rec {}; + + int num = 0; + while (bfid->next_record(&rec) == 0) { + + // remember that -> has higher precedence thatn & + num = rec.get_fmt(&bfid->hdr, "HD"); + + printf("num: %d\n", num); + for (idx_row = 0; idx_row < n_samples; idx_row++) { + for (idx_hap = 0; idx_hap < k_haps; idx_hap++) + printf("%f\t ", rec.dst[idx_row*k_haps + idx_hap]); + printf("\n"); + } + + log->info("Processed %s records", + std::to_string(++idx_rec).c_str()); + + + } + + return output_status; +} -// // instantiate record object -// HaplotypeDataRecord record { vcf_data.n_samples(), vcf_data.k_founders() }; -// -// // analyze each line, i.e. position, in the VCF // size_t m_markers { 1 }; // // double sum { 0 }; diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index 68b42ab..0abce14 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -221,7 +221,7 @@ TEST(TestBcfHeader, BcfHdrNull) { TEST(TestReadBcf, Constructor) { bcfio::ReadBcf bcf { VCF_NAME }; EXPECT_EQ(bcf.n_samples(), N_SAMPS); - EXPECT_EQ(bcf.k_founders(), K_FOUNDERS); + EXPECT_EQ(bcf.k_haps(), K_FOUNDERS); } TEST(TestReadBcf, VcfSampNames) { @@ -270,7 +270,7 @@ TEST(TestReadBcf, BcfSampNames) { // // HaplotypeVcfParser vcf { VCF_NAME }; // -// HaplotypeDataRecord record { vcf.n_samples(), vcf.k_founders() }; +// HaplotypeDataRecord record { vcf.n_samples(), vcf.k_haps() }; // // bool record_loaded { false }; // record_loaded = vcf.load_record(record); From c26fb8670aea8d93b3a1443d069a24cfe37d7f51 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 16 Dec 2025 20:18:01 -0500 Subject: [PATCH 13/58] intermediate progress, still not operational --- include/bcfio.h | 83 +++++++++++++++++++++++++++++-------------------- src/bcfio.cpp | 70 +++++++++++++++++++++++------------------ 2 files changed, 88 insertions(+), 65 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index 2d1013a..fb0dda0 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -55,32 +55,40 @@ struct BcfHdrAttr { uint64_t number : 20, vl_type : 4, type : 4, coltype : 4; }; // memory leak. class BcfHeader { public: - htslib::bcf_hdr_t *hdr; BcfHeader(htslib::htsFile *fid): - hdr(fid ? htslib::bcf_hdr_read(fid) : nullptr) {}; + hdr_(fid ? htslib::bcf_hdr_read(fid) : nullptr) {}; - ~BcfHeader() { - if (hdr) htslib::bcf_hdr_destroy(hdr); - hdr = nullptr; - }; + ~BcfHeader() { if (hdr_) htslib::bcf_hdr_destroy(hdr_); }; - const bool isnull() const { return hdr == nullptr; }; + bool isnull() const { return hdr_ == nullptr; }; // sample_names() // @title: "get_*" member functions for info retrieval // @description: - // @param name: the id of the formatted data field to retrieve - // @param ptr: the pointer to memory for which the BcfHdrAttr - // data will be copied into memory. - // @return - const int get_format(const char *name, BcfHdrAttr *ptr) const; - const int get_info(const char *name, BcfHdrAttr *ptr) const; - const int get_filter(const char *name, BcfHdrAttr *ptr) const; + // @param id: the id of the formatted data field to retrieve + // @param ptr: the pointer to memory for which the BcfHdrAttr data will be + // copied into memory. + // @return 0 for success < 0 for fail + int get_format(const char *id, BcfHdrAttr *ptr) const; + int get_info(const char *id, BcfHdrAttr *ptr) const; + int get_filter(const char *id, BcfHdrAttr *ptr) const; + + const htslib::bcf_hdr_t *hts_hdr() const { return hdr_; }; private: - const int decode_hts_idinfo_(const char *name, + htslib::bcf_hdr_t *hdr_; + BcfHdrAttr attr_ {}; + + // @title: + // @description decoder based upon htslib/vcf.h line 100 in the typedef + // struct bcf_idinfo_t. + // @param name: + // @param bcf_dt_type + // @param ptr + // @return -1 indicates an error has occured and 0 a success + int decode_hts_idinfo_(const char *name, const int bcf_dt_type, BcfHdrAttr *ptr) const; }; @@ -89,18 +97,32 @@ class BcfHeader { // @title Manage bcf record // @description Manage the lifetime of a htslib::bcf1_t type record using // htslib functions with RAII. -struct BcfRecord { - BcfRecord(): rec(htslib::bcf_init()) {}; +class BcfRecord { +public: + BcfRecord(): rec_(htslib::bcf_init()) {}; ~BcfRecord(); + operator[](size_t idx); bool is_snp() const { return htslib::bcf_is_snp(rec); } - int get_fmt(BcfHeader *hdr, const char *tag); + // @title: Load sample data at the current locus + // @description: Sample data of the specified format at the current locus + // is not made available by reading a locus's record and storing in + // the bcf1_t type. Instead, we need to supply a pointer variable + // and format id to make that id's smaple data available. This + // function help simplify this process. + // @param hdr: instance of the bcf header to retreive meta data + // @param tag: the C-string id representing the data we want to query. + // @return 0 upon success and != 0 for failure + int load_data(BcfHeader *hdr, const char *tag); - htslib::bcf1_t *rec; + const htslib::bcf1_t *cur_rec() const { return rec_; }; - int ndst = 0; - float *dst = nullptr; +private: + htslib::bcf1_t *rec_; + int ndst_ = 0; + float *fdst_ = nullptr; + char **cdst_ = nullptr; }; @@ -117,34 +139,27 @@ class ReadBcf private: const std::string fname_; htslib::htsFile *fid_; + BcfHeader hdr_; public: ReadBcf(const char *bcfname); ReadBcf(const char *bcfname, const char *sample_fname); - // HaplotypeVcfParser(const std::string& variant_fname); - // HaplotypeVcfParser(const std::string& variant_fname, - // const std::string& sample_fname); ReadBcf()=delete; ReadBcf(const ReadBcf&)=delete; ReadBcf(const ReadBcf&&)=delete; - // HaplotypeVcfParser& operator=(const HaplotypeVcfParser&)=delete; ~ReadBcf(); - const size_t n_samples() const; + // See htslib/vcf.h line 649 + // Remember that n is the number of entries in the triplet of + // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n + // that correspondes to the number of samples. + const size_t n_samples() const { return hdr_.hdr_->n[BCF_DT_SAMPLE]; }; const size_t k_haps() const; std::unique_ptr sample_names() const; int next_record(BcfRecord *rec); - - BcfHeader hdr; - - // size_t fpos_record_one_ { 0 }; - - // void pos_(size_t); - // size_t get_line_num_char_(); - // void set_params_(); }; } diff --git a/src/bcfio.cpp b/src/bcfio.cpp index d72646b..b0505ba 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -12,25 +12,22 @@ #include #include -// @title: -// @description decoder based upon htslib/vcf.h line 100 in the typedef -// struct bcf_idinfo_t. -// @param name: -// @param bcf_dt_type -// @param ptr -// @return -1 indicates an error has occured and 0 a success +// ***************************************************************************** +// class BcfHeader +// ***************************************************************************** +// const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, const int bcf_dt_type, bcfio::BcfHdrAttr *ptr) const { // BCF_DT_ID is the C macro for the ID dictionary index defined by htslib // see htslib/vcf.h line 86 - int idx = htslib::bcf_hdr_id2int(hdr, BCF_DT_ID, name); + int idx = htslib::bcf_hdr_id2int(hdr_, BCF_DT_ID, name); if (idx < 0) return -1; - uint64_t val = hdr->id[BCF_DT_ID][idx].val->info[bcf_dt_type]; + uint64_t val = hdr_->id[BCF_DT_ID][idx].val->info[bcf_dt_type]; ptr->number = val >> 12 & 0xfffff; ptr->vl_type = val >> 8 & 0xf; @@ -40,32 +37,50 @@ const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, return 0; } -const int bcfio::BcfHeader::get_format(const char *name, BcfHdrAttr *ptr) const { - return decode_hts_idinfo_(name, BCF_HL_FMT, ptr); +int bcfio::BcfHeader::get_format(const char *id, BcfHdrAttr *ptr) const { + return decode_hts_idinfo_(id, BCF_HL_FMT, ptr); } -const int bcfio::BcfHeader::get_info(const char *name, BcfHdrAttr *ptr) const { - return decode_hts_idinfo_(name, BCF_HL_INFO, ptr); +int bcfio::BcfHeader::get_info(const char *id, BcfHdrAttr *ptr) const { + return decode_hts_idinfo_(id, BCF_HL_INFO, ptr); } -const int bcfio::BcfHeader::get_filter(const char *name, BcfHdrAttr *ptr) const { - return decode_hts_idinfo_(name, BCF_HL_FLT, ptr); +int bcfio::BcfHeader::get_filter(const char *id, BcfHdrAttr *ptr) const { + return decode_hts_idinfo_(id, BCF_HL_FLT, ptr); } +// ***************************************************************************** +// class BcfRecord +// ***************************************************************************** + bcfio::BcfRecord::~BcfRecord() { if (rec) htslib::bcf_destroy(rec); - - // TODO: double check destructor of dst - if (dst) delete[] dst; + if (dst_) free(dst_); } -int bcfio::BcfRecord::get_fmt(bcfio::BcfHeader *hdr, const char *tag) { - - return htslib::bcf_get_format_values(hdr->hdr, rec, tag, - (void**)(&dst), &ndst, BCF_HT_REAL); +int bcfio::BcfRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { + int status { 0 }; + + if ((status = hdr->get_format(id, &attr_)) != 0) + return status; + + if (attr_->type == BCF_HT_REAL) + return htslib::bcf_get_format_values(hdr->hts_hdr(), rec, id, + (void**)(&fdst_), &ndst_, BCF_HT_REAL); + else if (attr_->type == BCF_HT_STR) + return htslib::bcf_get_format_values(hdr->hts_hdr(), rec, id, + (void**)(&cdst_), &ndst_, BCF_HT_STR); + + printf("ERROR: Only types string and float are currently supported." + " contact the project maintainer if your type is not yet" + " supported.\n"); + return -1; } +// ***************************************************************************** +// class BcfRead +// ***************************************************************************** bcfio::ReadBcf::ReadBcf(const char *bcfname) : fname_(bcfname), fid_(htslib::hts_open(bcfname, "r")), @@ -104,21 +119,14 @@ bcfio::ReadBcf::~ReadBcf() { htslib::hts_close(fid_); } -const size_t bcfio::ReadBcf::n_samples() const { - // See htslib/vcf.h line 649 - // Remember that n is the number of entries in the triplet of - // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n - // that correspondes to the number of samples. - return hdr.hdr->n[BCF_DT_SAMPLE]; -}; const size_t bcfio::ReadBcf::k_haps() const { BcfHdrAttr fmt {}; if (hdr.get_format("HD", &fmt) < 0) - printf("errror\n"); + printf("error\n"); - return static_cast(fmt.number); + return static_cast(fmt.number); } // Note: May be better to just return a reference? From d7227e94c1f5cb94450ef62e3c06a2f759f21d42 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 17 Dec 2025 11:00:49 -0500 Subject: [PATCH 14/58] intermediate update --- include/bcfio.h | 68 +++++++++++++++++++++++++++++++++----------- src/bcfio.cpp | 67 ++++++++++++++++++++++++++----------------- tests/test_bcfio.cpp | 18 ++++++++++-- 3 files changed, 108 insertions(+), 45 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index fb0dda0..45f5b6d 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -49,7 +49,7 @@ namespace bcfio { struct BcfHdrAttr { uint64_t number : 20, vl_type : 4, type : 4, coltype : 4; }; -// @title: Manage bcf header +// @title: Interface and manager of htslib bcf_hdr_t // @description: The bcf header C-struct requires manual allocation and release // of memory. This class manages applies RAII, reducing the chance of a // memory leak. @@ -65,15 +65,26 @@ class BcfHeader { // sample_names() - // @title: "get_*" member functions for info retrieval + // @title: "get_*" member functions for attribute retrieval // @description: // @param id: the id of the formatted data field to retrieve // @param ptr: the pointer to memory for which the BcfHdrAttr data will be // copied into memory. // @return 0 for success < 0 for fail - int get_format(const char *id, BcfHdrAttr *ptr) const; - int get_info(const char *id, BcfHdrAttr *ptr) const; - int get_filter(const char *id, BcfHdrAttr *ptr) const; + int get_format_attr(const char *id, BcfHdrAttr *ptr) const; + int get_info_attr(const char *id, BcfHdrAttr *ptr) const; + int get_filter_attr(const char *id, BcfHdrAttr *ptr) const; + + // @title: The number of values stored in format id + // @description: Each bcf format field is able to hold unique number of + // values per sample. This function provides a simple interface to + // the bcf file to retrieve that number. + // @param id: the format field id + // @return if an error occured that value returned is < 0, otherwise the + // number of values of fmt field id recorded per sample is returned. + int32_t k_fmt(const char *id) const; + + size_t n_samples() const { return hdr_->n[BCF_DT_SAMPLE]; }; const htslib::bcf_hdr_t *hts_hdr() const { return hdr_; }; @@ -94,16 +105,21 @@ class BcfHeader { }; -// @title Manage bcf record -// @description Manage the lifetime of a htslib::bcf1_t type record using -// htslib functions with RAII. -class BcfRecord { +// @title: Interface and manage htslib bcf1_t +// @description: The htslib bcf1_t data structure requires manual memory +// management, knowledge of several bit-packed values, knowledge of +// several functions for querying data. This class simplifies +// memory management using C++ RAII idiom and provides a simplified, +// albeit non-comprehensive, interface for loading and querying data +// stored in the bcf1_t struct. +class BcfFloatRecord { public: + BcfRecord(): rec_(htslib::bcf_init()) {}; ~BcfRecord(); - operator[](size_t idx); - bool is_snp() const { return htslib::bcf_is_snp(rec); } + // access to loaded data + float operator[](const size_t idx) const; // @title: Load sample data at the current locus // @description: Sample data of the specified format at the current locus @@ -115,14 +131,21 @@ class BcfRecord { // @param tag: the C-string id representing the data we want to query. // @return 0 upon success and != 0 for failure int load_data(BcfHeader *hdr, const char *tag); - const htslib::bcf1_t *cur_rec() const { return rec_; }; + bool is_snp() const { return htslib::bcf_is_snp(rec); } + private: htslib::bcf1_t *rec_; + + // These attributes store htslib access points to record data int ndst_ = 0; - float *fdst_ = nullptr; - char **cdst_ = nullptr; + float *dst_ = nullptr; + + // data that dst_ point to are stored in row major order, with columns + // being k_fmt and rows being n_samples. + size_t col_num_ = 0; + size_t row_num_ = 0; }; @@ -142,6 +165,7 @@ class ReadBcf BcfHeader hdr_; public: + // TODO: Review C++ idioms the rule of three and five ReadBcf(const char *bcfname); ReadBcf(const char *bcfname, const char *sample_fname); @@ -151,15 +175,25 @@ class ReadBcf ~ReadBcf(); + // @title: The number of values stored in format id + // @description: Each bcf format field is able to hold unique number of + // values per sample. This function provides a simple interface to + // the bcf file to retrieve that number. + // @param id: the format field id + // @return if an error occured that value returned is < 0, otherwise the + // number of values of fmt field id recorded per sample is returned. + int32_t k_fmt(const char *id) const { return hdr_.k_fmt(id); }; + // See htslib/vcf.h line 649 // Remember that n is the number of entries in the triplet of // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n // that correspondes to the number of samples. - const size_t n_samples() const { return hdr_.hdr_->n[BCF_DT_SAMPLE]; }; - const size_t k_haps() const; + size_t n_samples() const { return hdr_.n_samples() }; + + // TODO: sample_names std::unique_ptr sample_names() const; - int next_record(BcfRecord *rec); + int next_record(BcfFloatRecord *rec); }; } diff --git a/src/bcfio.cpp b/src/bcfio.cpp index b0505ba..4eb7948 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -25,7 +25,7 @@ const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, int idx = htslib::bcf_hdr_id2int(hdr_, BCF_DT_ID, name); if (idx < 0) - return -1; + return idx; uint64_t val = hdr_->id[BCF_DT_ID][idx].val->info[bcf_dt_type]; @@ -37,44 +37,68 @@ const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, return 0; } -int bcfio::BcfHeader::get_format(const char *id, BcfHdrAttr *ptr) const { +int bcfio::BcfHeader::get_format_attr(const char *id, BcfHdrAttr *ptr) const { return decode_hts_idinfo_(id, BCF_HL_FMT, ptr); } -int bcfio::BcfHeader::get_info(const char *id, BcfHdrAttr *ptr) const { +int bcfio::BcfHeader::get_info_attr(const char *id, BcfHdrAttr *ptr) const { return decode_hts_idinfo_(id, BCF_HL_INFO, ptr); } -int bcfio::BcfHeader::get_filter(const char *id, BcfHdrAttr *ptr) const { +int bcfio::BcfHeader::get_filter_attr(const char *id, BcfHdrAttr *ptr) const { return decode_hts_idinfo_(id, BCF_HL_FLT, ptr); } +int32_t bcfio::BcfHeader::k_fmt(const char *id) const { + BcfHdrAttr fmt {}; + + int32_t status { 0 }; + + if ((status = hdr.get_format(id, &fmt)) < 0) { + printf("ERROR: invalid id: %s\n", id); + return status; + } + + return static_cast(fmt.number); +} + // ***************************************************************************** // class BcfRecord // ***************************************************************************** bcfio::BcfRecord::~BcfRecord() { - if (rec) htslib::bcf_destroy(rec); + if (rec_) htslib::bcf_destroy(rec_); if (dst_) free(dst_); + rec = nullptr; + dst_ = nullptr; +} + +float operator[](const size_t idx) const { + if (idx > col_num_ * row_num_) + return + return *(dst_ + idx); } + int bcfio::BcfRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { int status { 0 }; + col_num_ = row_num_ = 0; - if ((status = hdr->get_format(id, &attr_)) != 0) + if ((status = hdr->get_format_attr(id, &attr_)) < 0) return status; - if (attr_->type == BCF_HT_REAL) - return htslib::bcf_get_format_values(hdr->hts_hdr(), rec, id, - (void**)(&fdst_), &ndst_, BCF_HT_REAL); - else if (attr_->type == BCF_HT_STR) - return htslib::bcf_get_format_values(hdr->hts_hdr(), rec, id, - (void**)(&cdst_), &ndst_, BCF_HT_STR); - - printf("ERROR: Only types string and float are currently supported." - " contact the project maintainer if your type is not yet" - " supported.\n"); - return -1; + status = htslib::bcf_get_format_values(hdr->hts_hdr(), + rec, + id, + (void**)(&fdst_), + &fndst_, + BCF_HT_REAL); + + if (status < 0) + return status; + + col_num_ = hdr.k_fmt(id); + row_num_ = hdr.n_samples(); } @@ -120,15 +144,6 @@ bcfio::ReadBcf::~ReadBcf() { } -const size_t bcfio::ReadBcf::k_haps() const { - BcfHdrAttr fmt {}; - - if (hdr.get_format("HD", &fmt) < 0) - printf("error\n"); - - return static_cast(fmt.number); -} - // Note: May be better to just return a reference? std::unique_ptr bcfio::ReadBcf::sample_names() const { diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index 0abce14..be0c572 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -17,7 +17,7 @@ extern "C" { char VCF_NAME[] { "build/geno_test_data.vcf" }; char VCFGZ_NAME[] { "build/geno_test_data.vcf.gz" }; char BCF_NAME[] { "build/geno_test_data.bcf" }; -size_t K_FOUNDERS = 8; +int32_t K_FOUNDERS = 8; size_t N_SAMPS = 11; @@ -221,7 +221,21 @@ TEST(TestBcfHeader, BcfHdrNull) { TEST(TestReadBcf, Constructor) { bcfio::ReadBcf bcf { VCF_NAME }; EXPECT_EQ(bcf.n_samples(), N_SAMPS); - EXPECT_EQ(bcf.k_haps(), K_FOUNDERS); + EXPECT_EQ(bcf.k_fmt("HD"), K_FOUNDERS); +} + + +TEST(TestReadBcf, K_fmt) { + bcfio::ReadBcf bcf { VCF_NAME }; + + // DS is alt allele dosage, which is more clearly defined as the expected + // count of alt alleles under the trained HMM + EXPECT_EQ(bcf.k_fmt("DS"), 1); + + // error detection + EXPECT_TRUE(bcf.k_fmt("WRONG_ID") < 0); + EXPECT_TRUE(bcf.k_fmt("") < 0); + EXPECT_TRUE(bcf.k_fmt(nullptr) < 0); } TEST(TestReadBcf, VcfSampNames) { From 5328e99aa5efde60f10b88d5e7cdc8ba75d6aeee Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 17 Dec 2025 11:20:50 -0500 Subject: [PATCH 15/58] update BcfRecord to BcfFloatRecord --- include/bcfio.h | 4 ++-- src/bcfio.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index 45f5b6d..d63445d 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -115,8 +115,8 @@ class BcfHeader { class BcfFloatRecord { public: - BcfRecord(): rec_(htslib::bcf_init()) {}; - ~BcfRecord(); + BcfFloatRecord(): rec_(htslib::bcf_init()) {}; + ~BcfFloatRecord(); // access to loaded data float operator[](const size_t idx) const; diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 4eb7948..a246de6 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -63,10 +63,10 @@ int32_t bcfio::BcfHeader::k_fmt(const char *id) const { } // ***************************************************************************** -// class BcfRecord +// class BcfFloatRecord // ***************************************************************************** -bcfio::BcfRecord::~BcfRecord() { +bcfio::BcfFloatRecord::~BcfFloatRecord() { if (rec_) htslib::bcf_destroy(rec_); if (dst_) free(dst_); rec = nullptr; @@ -80,7 +80,7 @@ float operator[](const size_t idx) const { } -int bcfio::BcfRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { +int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { int status { 0 }; col_num_ = row_num_ = 0; @@ -157,7 +157,7 @@ std::unique_ptr bcfio::ReadBcf::sample_names() const { } // title: load next record -int bcfio::ReadBcf::next_record(bcfio::BcfRecord *ptr) { +int bcfio::ReadBcf::next_record(bcfio::BcfFloatRecord *ptr) { int status = htslib::bcf_read(fid_, hdr.hdr, ptr->rec); if (status != 0) return status; From b1bb96e31a29db4e9e47da686a2ec14b7c0cffe5 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 18 Dec 2025 13:11:04 -0500 Subject: [PATCH 16/58] intermediate upate --- include/bcfio.h | 46 ++++++++++------ src/bcfio.cpp | 88 ++++++++++++++++-------------- src/use_haplotype.cpp | 29 ++++++---- tests/test_bcfio.cpp | 123 ++++++++++++++++++++++++++++++------------ 4 files changed, 186 insertions(+), 100 deletions(-) diff --git a/include/bcfio.h b/include/bcfio.h index d63445d..f02b602 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -11,10 +11,12 @@ #ifndef HEADER_PARSE_HTS_H #define HEADER_PARSE_HTS_H -#include +#include +#include #include #include -#include + +#include namespace htslib { extern "C" { @@ -63,7 +65,9 @@ class BcfHeader { bool isnull() const { return hdr_ == nullptr; }; - // sample_names() + + // @title: Retreive the set of smaple names + const std::unique_ptr sample_names() const; // @title: "get_*" member functions for attribute retrieval // @description: @@ -75,6 +79,7 @@ class BcfHeader { int get_info_attr(const char *id, BcfHdrAttr *ptr) const; int get_filter_attr(const char *id, BcfHdrAttr *ptr) const; + int subset_samples(const char *filename); // @title: The number of values stored in format id // @description: Each bcf format field is able to hold unique number of // values per sample. This function provides a simple interface to @@ -86,11 +91,12 @@ class BcfHeader { size_t n_samples() const { return hdr_->n[BCF_DT_SAMPLE]; }; + // TODO: what unit test should I do for this? const htslib::bcf_hdr_t *hts_hdr() const { return hdr_; }; private: htslib::bcf_hdr_t *hdr_; - BcfHdrAttr attr_ {}; + // BcfHdrAttr attr_ {}; // @title: // @description decoder based upon htslib/vcf.h line 100 in the typedef @@ -118,8 +124,11 @@ class BcfFloatRecord { BcfFloatRecord(): rec_(htslib::bcf_init()) {}; ~BcfFloatRecord(); - // access to loaded data - float operator[](const size_t idx) const; + // provide check-free fast, but unsafe, access to loaded data + float operator[](const size_t idx) const { return *(dst_ + idx); }; + + // provide index checked access to data. + std::optional get(const size_t row_idx, const size_t col_idx) const; // @title: Load sample data at the current locus // @description: Sample data of the specified format at the current locus @@ -131,9 +140,11 @@ class BcfFloatRecord { // @param tag: the C-string id representing the data we want to query. // @return 0 upon success and != 0 for failure int load_data(BcfHeader *hdr, const char *tag); - const htslib::bcf1_t *cur_rec() const { return rec_; }; + size_t size() const { return static_cast(ndst_); }; + + htslib::bcf1_t *cur_rec() const { return rec_; }; - bool is_snp() const { return htslib::bcf_is_snp(rec); } + bool is_snp() const { return htslib::bcf_is_snp(rec_); } private: htslib::bcf1_t *rec_; @@ -159,11 +170,6 @@ class BcfFloatRecord { // all sample records are retrieved. class ReadBcf { -private: - const std::string fname_; - htslib::htsFile *fid_; - BcfHeader hdr_; - public: // TODO: Review C++ idioms the rule of three and five ReadBcf(const char *bcfname); @@ -188,12 +194,20 @@ class ReadBcf // Remember that n is the number of entries in the triplet of // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n // that correspondes to the number of samples. - size_t n_samples() const { return hdr_.n_samples() }; + size_t n_samples() const { return hdr_.n_samples(); }; // TODO: sample_names - std::unique_ptr sample_names() const; + const std::unique_ptr sample_names() const { + return hdr_.sample_names(); + } + + int next_record(BcfFloatRecord *rec, const char *id); + +private: + const std::string fname_; + htslib::htsFile *fid_; + BcfHeader hdr_; - int next_record(BcfFloatRecord *rec); }; } diff --git a/src/bcfio.cpp b/src/bcfio.cpp index a246de6..1b9c160 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -10,13 +10,12 @@ // #include -#include // ***************************************************************************** // class BcfHeader // ***************************************************************************** // -const int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, +int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, const int bcf_dt_type, bcfio::BcfHdrAttr *ptr) const { @@ -50,18 +49,34 @@ int bcfio::BcfHeader::get_filter_attr(const char *id, BcfHdrAttr *ptr) const { } int32_t bcfio::BcfHeader::k_fmt(const char *id) const { + if (!id) + return -1; + BcfHdrAttr fmt {}; int32_t status { 0 }; - if ((status = hdr.get_format(id, &fmt)) < 0) { - printf("ERROR: invalid id: %s\n", id); + if ((status = get_format_attr(id, &fmt)) < 0) return status; - } return static_cast(fmt.number); } +int bcfio::BcfHeader::subset_samples(const char *filename) { + return htslib::bcf_hdr_set_samples(hdr_, filename, 1); +} + +const std::unique_ptr bcfio::BcfHeader::sample_names() const { + + std::unique_ptr samp_names = + std::make_unique(n_samples()); + + for (int i = 0; i < n_samples(); i++) + samp_names[i] = std::string(*(hdr_->samples + i)); + + return samp_names; +} + // ***************************************************************************** // class BcfFloatRecord // ***************************************************************************** @@ -69,36 +84,41 @@ int32_t bcfio::BcfHeader::k_fmt(const char *id) const { bcfio::BcfFloatRecord::~BcfFloatRecord() { if (rec_) htslib::bcf_destroy(rec_); if (dst_) free(dst_); - rec = nullptr; + rec_ = nullptr; dst_ = nullptr; } -float operator[](const size_t idx) const { - if (idx > col_num_ * row_num_) - return - return *(dst_ + idx); -} +std::optional bcfio::BcfFloatRecord::get(const size_t row_idx, + const size_t col_idx) const { + if ((row_idx * col_idx + col_idx) >= ndst_) return std::nullopt; + return *(dst_ + row_idx * col_idx + col_idx); +} int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { int status { 0 }; col_num_ = row_num_ = 0; - if ((status = hdr->get_format_attr(id, &attr_)) < 0) - return status; - status = htslib::bcf_get_format_values(hdr->hts_hdr(), - rec, + rec_, id, - (void**)(&fdst_), - &fndst_, + (void**)(&dst_), + &ndst_, BCF_HT_REAL); if (status < 0) return status; - col_num_ = hdr.k_fmt(id); - row_num_ = hdr.n_samples(); + int32_t k { 0 }; + if ((k = hdr->k_fmt(id)) < 0) { + col_num_ = row_num_ = 0; + return k; + } + + col_num_ = static_cast(k); + row_num_ = hdr->n_samples(); + + return 0; } @@ -108,14 +128,14 @@ int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { bcfio::ReadBcf::ReadBcf(const char *bcfname) : fname_(bcfname), fid_(htslib::hts_open(bcfname, "r")), - hdr(fid_) {}; + hdr_(fid_) {}; // TODO: subset samples by those in sample_fname file bcfio::ReadBcf::ReadBcf(const char *bcfname, const char *sample_fname) : fname_(bcfname), fid_(htslib::hts_open(bcfname, "r")), - hdr(fid_) { + hdr_(fid_) { int status { 0 }; // Subset samples with those found in the file sample_fname @@ -123,7 +143,7 @@ bcfio::ReadBcf::ReadBcf(const char *bcfname, const char *sample_fname) fprintf(stdout, "No file with sample names detected, retreiving" " records for all samples.\n"); else - status = htslib::bcf_hdr_set_samples(hdr.hdr, sample_fname, 1); + status = hdr_.subset_samples(sample_fname); if (status < 0) { fprintf(stderr, "Error: Couldn't read sample file\n"); @@ -139,29 +159,19 @@ bcfio::ReadBcf::ReadBcf(const char *bcfname, const char *sample_fname) }; bcfio::ReadBcf::~ReadBcf() { - if (fid_) - htslib::hts_close(fid_); + if (fid_) htslib::hts_close(fid_); } -// Note: May be better to just return a reference? -std::unique_ptr bcfio::ReadBcf::sample_names() const { - - std::unique_ptr samp_names = - std::make_unique(n_samples()); - - for (int i = 0; i < n_samples(); i++) - samp_names[i] = std::string(*(hdr.hdr->samples + i)); - - return samp_names; -} - // title: load next record -int bcfio::ReadBcf::next_record(bcfio::BcfFloatRecord *ptr) { - int status = htslib::bcf_read(fid_, hdr.hdr, ptr->rec); +int bcfio::ReadBcf::next_record(bcfio::BcfFloatRecord *ptr, const char *id) { + int status = htslib::bcf_read(fid_, hdr_.hts_hdr(), ptr->cur_rec()); if (status != 0) return status; // Unpacking options defined in htslib/vcf.h line 419 - return htslib::bcf_unpack(ptr->rec, BCF_UN_ALL); + if (htslib::bcf_unpack(ptr->cur_rec(), BCF_UN_ALL) < 0) + return -1; + + return ptr->load_data(&hdr_, id); } diff --git a/src/use_haplotype.cpp b/src/use_haplotype.cpp index e6e4180..1157371 100644 --- a/src/use_haplotype.cpp +++ b/src/use_haplotype.cpp @@ -4,29 +4,38 @@ int compute_haplotype_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov) { - int output_status = -1; + int output_status = 0; // instantiate matrices to hold calculations const size_t n_samples { bfid->n_samples() }; - const size_t k_haps { bfid->k_haps() }; + int32_t k { 0 }; + if ((k = bfid->k_fmt("HD")) < 0) { + log->error("%s\n", "Wrong format id tag"); + exit(EXIT_FAILURE); + } + const size_t k_haps { static_cast(k) }; size_t idx_row { 0 }; // size_t idx_col { 0 }; size_t idx_hap { 0 }; size_t idx_rec { 0 }; - bcfio::BcfRecord rec {}; + bcfio::BcfFloatRecord rec {}; - int num = 0; - while (bfid->next_record(&rec) == 0) { + std::optional val { 0 }; - // remember that -> has higher precedence thatn & - num = rec.get_fmt(&bfid->hdr, "HD"); + while (bfid->next_record(&rec, "HD") == 0) { - printf("num: %d\n", num); for (idx_row = 0; idx_row < n_samples; idx_row++) { - for (idx_hap = 0; idx_hap < k_haps; idx_hap++) - printf("%f\t ", rec.dst[idx_row*k_haps + idx_hap]); + + for (idx_hap = 0; idx_hap < k_haps; idx_hap++) { + if ((val = rec.get(idx_row, idx_hap)) == std::nullopt) { + printf("IDX: (%zu, %zu) = null\n", idx_row, idx_hap); + return -1; + } + printf("%f\t ", val.value()); + } + printf("\n"); } diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index be0c572..6adf7a4 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -21,6 +21,10 @@ int32_t K_FOUNDERS = 8; size_t N_SAMPS = 11; +// ************************************************************************ +// Test bcfio::BcfHeader +// ************************************************************************ + TEST(TestBcfHeader, ConstructorVcfHdr) { htslib::htsFile *fid = htslib::hts_open(VCF_NAME, "r"); bcfio::BcfHeader hdr { fid }; @@ -29,7 +33,7 @@ TEST(TestBcfHeader, ConstructorVcfHdr) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("HD", &attr); + int status = hdr.get_format_attr("HD", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.number, K_FOUNDERS); EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); @@ -46,7 +50,7 @@ TEST(TestBcfHeader, ConstructorVcfGzHdr) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("HD", &attr); + int status = hdr.get_format_attr("HD", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.number, K_FOUNDERS); EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); @@ -63,7 +67,7 @@ TEST(TestBcfHeader, ConstructorBcfHdr) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("HD", &attr); + int status = hdr.get_format_attr("HD", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.number, K_FOUNDERS); EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); @@ -81,7 +85,7 @@ TEST(TestBcfHeader, BcfHdrFmtGt) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("GT", &attr); + int status = hdr.get_format_attr("GT", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.number, 1); EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); @@ -99,7 +103,7 @@ TEST(TestBcfHeader, BcfHdrFmtGp) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("GP", &attr); + int status = hdr.get_format_attr("GP", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.number, 3); EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); @@ -116,7 +120,7 @@ TEST(TestBcfHeader, BcfHdrFmtDs) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("DS", &attr); + int status = hdr.get_format_attr("DS", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.number, 1); EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); @@ -134,7 +138,7 @@ TEST(TestBcfHeader, BcfHdrFmtErr) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_format("DOESNOTEXIST", &attr); + int status = hdr.get_format_attr("DOESNOTEXIST", &attr); EXPECT_NE(status, 0); if (fid) htslib::hts_close(fid); @@ -149,10 +153,10 @@ TEST(TestBcfHeader, BcfHdrFilter) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_filter("PASS", &attr); + int status = hdr.get_filter_attr("PASS", &attr); EXPECT_EQ(status, 0); - status = hdr.get_filter("PASSING", &attr); + status = hdr.get_filter_attr("PASSING", &attr); EXPECT_NE(status, 0); if (fid) htslib::hts_close(fid); @@ -167,7 +171,7 @@ TEST(TestBcfHeader, BcfHdrInfoEaf) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_info("EAF", &attr); + int status = hdr.get_info_attr("EAF", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.type, BCF_HT_REAL); EXPECT_EQ(attr.vl_type, BCF_VL_VAR); @@ -184,7 +188,7 @@ TEST(TestBcfHeader, BcfHdrInfoErc) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_info("ERC", &attr); + int status = hdr.get_info_attr("ERC", &attr); EXPECT_EQ(status, 0); EXPECT_EQ(attr.type, BCF_HT_REAL); EXPECT_EQ(attr.vl_type, BCF_VL_VAR); @@ -201,7 +205,7 @@ TEST(TestBcfHeader, BcfHdrInfoErr) { bcfio::BcfHdrAttr attr {}; - int status = hdr.get_info("NOTAINFOMEMBER", &attr); + int status = hdr.get_info_attr("NOTAINFOMEMBER", &attr); EXPECT_NE(status, 0); if (fid) htslib::hts_close(fid); @@ -217,35 +221,39 @@ TEST(TestBcfHeader, BcfHdrNull) { } +TEST(TestBcfHeader, Kfmt) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; -TEST(TestReadBcf, Constructor) { - bcfio::ReadBcf bcf { VCF_NAME }; - EXPECT_EQ(bcf.n_samples(), N_SAMPS); - EXPECT_EQ(bcf.k_fmt("HD"), K_FOUNDERS); -} + // DS is alt allele dosage, which is more clearly defined as the expected + // count of alt alleles under the trained HMM + EXPECT_EQ(hdr.k_fmt("DS"), 1); + EXPECT_EQ(hdr.k_fmt("HD"), K_FOUNDERS); + // error detection + EXPECT_TRUE(hdr.k_fmt("WRONG_ID") < 0); + EXPECT_TRUE(hdr.k_fmt("") < 0); + EXPECT_TRUE(hdr.k_fmt(nullptr) < 0); +} -TEST(TestReadBcf, K_fmt) { - bcfio::ReadBcf bcf { VCF_NAME }; +TEST(TestBcfHeader, Nsamples) { + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; // DS is alt allele dosage, which is more clearly defined as the expected // count of alt alleles under the trained HMM - EXPECT_EQ(bcf.k_fmt("DS"), 1); - - // error detection - EXPECT_TRUE(bcf.k_fmt("WRONG_ID") < 0); - EXPECT_TRUE(bcf.k_fmt("") < 0); - EXPECT_TRUE(bcf.k_fmt(nullptr) < 0); + EXPECT_EQ(hdr.n_samples(), N_SAMPS); } -TEST(TestReadBcf, VcfSampNames) { - bcfio::ReadBcf bcf { VCF_NAME }; +TEST(TestBcfHeader, VcfSampNames) { + htslib::htsFile *fid = htslib::hts_open(VCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; - std::unique_ptr s = bcf.sample_names(); + const std::unique_ptr s = hdr.sample_names(); char samp_name[] = "S01"; - for (int i = 0; i < bcf.n_samples(); i++) { + for (int i = 0; i < hdr.n_samples(); i++) { snprintf(samp_name, 4, "S%02d", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } @@ -253,13 +261,14 @@ TEST(TestReadBcf, VcfSampNames) { TEST(TestReadBcf, VcfGzSampNames) { - bcfio::ReadBcf bcf { VCFGZ_NAME }; + htslib::htsFile *fid = htslib::hts_open(VCFGZ_NAME, "r"); + bcfio::BcfHeader hdr { fid }; - std::unique_ptr s = bcf.sample_names(); + const std::unique_ptr s = hdr.sample_names(); char samp_name[] = "S01"; - for (int i = 0; i < bcf.n_samples(); i++) { + for (int i = 0; i < hdr.n_samples(); i++) { snprintf(samp_name, 4, "S%02d", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } @@ -267,19 +276,63 @@ TEST(TestReadBcf, VcfGzSampNames) { TEST(TestReadBcf, BcfSampNames) { - bcfio::ReadBcf bcf { BCF_NAME }; + htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); + bcfio::BcfHeader hdr { fid }; - std::unique_ptr s = bcf.sample_names(); + const std::unique_ptr s = hdr.sample_names(); char samp_name[] = "S01"; - for (int i = 0; i < bcf.n_samples(); i++) { + for (int i = 0; i < hdr.n_samples(); i++) { snprintf(samp_name, 4, "S%02d", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } } + +// ************************************************************************ +// Test bcfio::BcfFloatRecord +// ************************************************************************ + +TEST(TestBcfFloatRecord, Constructor) { + bcfio::BcfFloatRecord brec {}; + + EXPECT_EQ(brec.size(), 0); + EXPECT_EQ(brec.get(1, 3), std::nullopt); +} + + + +// ************************************************************************ +// Test bcfio::ReadBcf +// ************************************************************************ + +TEST(TestReadBcf, Constructor) { + bcfio::ReadBcf bcf { VCF_NAME }; + EXPECT_EQ(bcf.n_samples(), N_SAMPS); + EXPECT_EQ(bcf.k_fmt("HD"), K_FOUNDERS); +} + + +TEST(TestReadBcf, Kfmt) { + bcfio::ReadBcf bcf { VCF_NAME }; + + // DS is alt allele dosage, which is more clearly defined as the expected + // count of alt alleles under the trained HMM + EXPECT_EQ(bcf.k_fmt("DS"), 1); + EXPECT_EQ(bcf.k_fmt("HD"), K_FOUNDERS); + + // TODO: what happens if I submit "GT", it exists but is a string + // not float + // error detection + EXPECT_TRUE(bcf.k_fmt("WRONG_ID") < 0); + EXPECT_TRUE(bcf.k_fmt("") < 0); + EXPECT_TRUE(bcf.k_fmt(nullptr) < 0); +} + + + // TEST(TestHaplotypeVCFParser, LoadRecord) { // // HaplotypeVcfParser vcf { VCF_NAME }; From 119d3a3e25cf0910a7a39bac9faab3276b5d18eb Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 19 Dec 2025 14:38:00 -0500 Subject: [PATCH 17/58] increase code warnings and errors at compile time. --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 82f58e2..fbfc520 100644 --- a/Makefile +++ b/Makefile @@ -17,14 +17,16 @@ ifneq ($(shell which clang++),) CXX = clang++ +CXXFLAGS = -pedantic -Wextra else ifneq ($(shell which g++),) CXX = g++ +CXXFLAGS = -Wpedantic -Wextra else $(error "Couldn't establish either clang or gcc compiler availability") endif -CXXFLAGS = -g -std=c++17 -Wall -Werror +CXXFLAGS += -g -std=c++17 -Wall -Werror ifndef VIM CXXFLAGS += -fdiagnostics-color=always From 919904cac241a42db75cd14b1c0c5946f978f5f6 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 23 Dec 2025 13:47:43 -0500 Subject: [PATCH 18/58] Intermediate and temporary update --- Makefile | 4 ++-- src/bcfio.cpp | 4 ++-- src/main.cpp | 44 ++++++++++++++++++++++++++++++++----------- src/use_haplotype.cpp | 4 ++++ tests/test_bcfio.cpp | 16 ++++++++-------- 5 files changed, 49 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index fbfc520..0b85d71 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ ifneq ($(shell which clang++),) CXX = clang++ -CXXFLAGS = -pedantic -Wextra +CXXFLAGS = -pedantic # -Wextra else ifneq ($(shell which g++),) CXX = g++ CXXFLAGS = -Wpedantic -Wextra @@ -32,6 +32,7 @@ ifndef VIM CXXFLAGS += -fdiagnostics-color=always endif +# Recall that -c flag prevents the compiler linking object files OBJ_OUTPUT_OPTIONS = -c -MMD -MP -o $@ AR = ar AR_FLAGS = crs @@ -86,7 +87,6 @@ $(TARGET): $(SRC_DIR)/main.cpp $(APP_OBJS) $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -largparse -lhts -# Recall that -c flag prevents the compiler linking object files $(BUILD_DIR)/%.o: $(SRC_DIR)/%.cpp | $(BUILD_DIR) $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(OBJ_OUTPUT_OPTIONS) $< diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 1b9c160..0103a5e 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -71,7 +71,7 @@ const std::unique_ptr bcfio::BcfHeader::sample_names() const { std::unique_ptr samp_names = std::make_unique(n_samples()); - for (int i = 0; i < n_samples(); i++) + for (size_t i = 0; i < n_samples(); i++) samp_names[i] = std::string(*(hdr_->samples + i)); return samp_names; @@ -90,7 +90,7 @@ bcfio::BcfFloatRecord::~BcfFloatRecord() { std::optional bcfio::BcfFloatRecord::get(const size_t row_idx, const size_t col_idx) const { - if ((row_idx * col_idx + col_idx) >= ndst_) return std::nullopt; + if ((row_idx * col_idx + col_idx) >= size()) return std::nullopt; return *(dst_ + row_idx * col_idx + col_idx); } diff --git a/src/main.cpp b/src/main.cpp index 4509283..ba860e6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -58,15 +58,26 @@ int main(int argc, char* argv[]) " include a single sample filename, and if necessary file system" " path, per line."); - parser.add_arg("--use_genotypes", + parser.add_arg("--gt", argparse::ArgType::BOOLEAN, "Use sample genotypes to compute the relationship matrix"); - parser.add_arg("--use_both", + parser.add_arg("--eac", argparse::ArgType::BOOLEAN, - "Use both genotypes and haplotypes to compute relationship matrix"); + "Use sample expected alt allele count to compute the" + " relationship matrix"); - parser.add_arg("vcf", + parser.add_arg("-b", + argparse::ArgType::BOOLEAN, + "Use both the expected alternative allele and haplotype counts to" + " compute relationship matrix"); + + parser.add_arg("--loco", + argparse::ArgType::STRING, + "Directory with chromosome matrix files to compute the" + " leave-one-chromosome-out (LOCO) relationship matrix.") + + parser.add_arg("--vcf", argparse::ArgType::STRING, "the path and filename of the vcf in which the hgrm is computed."); @@ -101,20 +112,27 @@ int main(int argc, char* argv[]) std::optional tmp_bool {}; - if ((tmp_bool = parser.get("use_genotypes")) == std::nullopt) { + if ((tmp_bool = parser.get("gt")) == std::nullopt) { fprintf(stderr, "Error retrieving relationship matrix type.\n"); exit(EXIT_FAILURE); } - bool use_genotypes { tmp_bool.value() }; + bool use_gt { tmp_bool.value() }; - if ((tmp_bool = parser.get("use_both")) == std::nullopt) { + if ((tmp_bool = parser.get("b")) == std::nullopt) { fprintf(stderr, "Error retrieving relationship matrix type.\n"); exit(EXIT_FAILURE); } bool use_both { tmp_bool.value() }; - if (use_genotypes && use_both) { - fprintf(stderr, "user must specify either use_genotypes, use_both," + if ((tmp_bool = parser.get("eac")) == std::nullopt) { + fprintf(stderr, "Error retrieving relationship matrix type.\n"); + exit(EXIT_FAILURE); + } + bool use_eac { tmp_bool.value() }; + + + if ((use_gt && use_both) || (use_gt && use_ds) || (use_both && use_ds)) { + fprintf(stderr, "user must specify either use_gt, use_both, use_ds," " or omit both options to compute the haplotype based" " relationship matrix."); exit(EXIT_FAILURE); @@ -136,11 +154,15 @@ int main(int argc, char* argv[]) bcfio::ReadBcf bfid { vcf_fname.c_str() }; Matrix cov { bfid.n_samples(), bfid.n_samples() }; - if (use_genotypes) { + if (use_gt) { log.info("Relationship matrix: genotype"); status = compute_genotype_matrix(); + } else if (use_ds) { + log.info("Relationship matrix: expected alt allele count + status = compute_eac_matrix(); } else if (use_both) { - log.info("Relationship matrix: genotype and haplotype"); + log.info("Relationship matrix: expected alt allele and haplotype" + " counts"); status = compute_geno_and_haplo_matrix(); } else { log.info("Relationship matrix: haplotype"); diff --git a/src/use_haplotype.cpp b/src/use_haplotype.cpp index 1157371..716e3a5 100644 --- a/src/use_haplotype.cpp +++ b/src/use_haplotype.cpp @@ -29,10 +29,14 @@ int compute_haplotype_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov) { for (idx_row = 0; idx_row < n_samples; idx_row++) { for (idx_hap = 0; idx_hap < k_haps; idx_hap++) { + if ((val = rec.get(idx_row, idx_hap)) == std::nullopt) { printf("IDX: (%zu, %zu) = null\n", idx_row, idx_hap); return -1; } + + (*cov)(idx_row, idx_row) += val.value(); + printf("%f\t ", val.value()); } diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index 6adf7a4..af56356 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -17,7 +17,7 @@ extern "C" { char VCF_NAME[] { "build/geno_test_data.vcf" }; char VCFGZ_NAME[] { "build/geno_test_data.vcf.gz" }; char BCF_NAME[] { "build/geno_test_data.bcf" }; -int32_t K_FOUNDERS = 8; +size_t K_FOUNDERS = 8; size_t N_SAMPS = 11; @@ -253,8 +253,8 @@ TEST(TestBcfHeader, VcfSampNames) { char samp_name[] = "S01"; - for (int i = 0; i < hdr.n_samples(); i++) { - snprintf(samp_name, 4, "S%02d", i+1); + for (size_t i = 0; i < hdr.n_samples(); i++) { + snprintf(samp_name, 4, "S%02zu", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } } @@ -268,8 +268,8 @@ TEST(TestReadBcf, VcfGzSampNames) { char samp_name[] = "S01"; - for (int i = 0; i < hdr.n_samples(); i++) { - snprintf(samp_name, 4, "S%02d", i+1); + for (size_t i = 0; i < hdr.n_samples(); i++) { + snprintf(samp_name, 4, "S%02zu", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } } @@ -283,8 +283,8 @@ TEST(TestReadBcf, BcfSampNames) { char samp_name[] = "S01"; - for (int i = 0; i < hdr.n_samples(); i++) { - snprintf(samp_name, 4, "S%02d", i+1); + for (size_t i = 0; i < hdr.n_samples(); i++) { + snprintf(samp_name, 4, "S%02zu", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } } @@ -298,7 +298,7 @@ TEST(TestReadBcf, BcfSampNames) { TEST(TestBcfFloatRecord, Constructor) { bcfio::BcfFloatRecord brec {}; - EXPECT_EQ(brec.size(), 0); + EXPECT_EQ(brec.size(), static_cast(0)); EXPECT_EQ(brec.get(1, 3), std::nullopt); } From 81d1eb6c255772bd4532b1b9b0068ce950ed8c05 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 2 Jan 2026 15:39:02 -0500 Subject: [PATCH 19/58] updated subcommands for both contig matrix calculation and loco --- README.md | 60 +++---- include/calc.h | 6 +- include/logger.h | 23 +-- src/logger.cpp | 100 +++++++----- src/main.cpp | 216 ++++++++++++++----------- src/use_both.cpp | 5 +- src/use_eac.cpp | 6 + src/{use_haplotype.cpp => use_ehc.cpp} | 2 +- src/{use_genotype.cpp => use_gt.cpp} | 1 + 9 files changed, 230 insertions(+), 189 deletions(-) create mode 100644 src/use_eac.cpp rename src/{use_haplotype.cpp => use_ehc.cpp} (98%) rename src/{use_genotype.cpp => use_gt.cpp} (65%) diff --git a/README.md b/README.md index c763477..afa72c4 100644 --- a/README.md +++ b/README.md @@ -3,23 +3,32 @@ # Compute the genetic relationship matrix using expected haplotype counts +The genetic relationship matrix (GRM) describes the genetic relationship between +pairs of samples. WRITE MORE -The genetic relationship matrix (GRM) is the covariance between samples over -the measured genetic markers. It's utility is in accounting for -relatedness among samples, as random effects in a Linear Mixed Effects -Model, when performing a Genome Wide Association Study (GWAS) [1,2] and computing -the heritability of complex traits [3]. The GRM is traditionally computed using -SNP genotypes, but here we are interested in haplotype based covariance. -## Running the software -The program is ran by supplying the path and filename of a VCF. Note, as -of now this software does not support data streams with UNIX pipe or -bgzip, gzip, etc. compression. The output is printed to standard out. +## Compute the genetic relationship matrix + +The GRM calculation requires the SNPs or haplotypes to jbe in the bcf family +of file formats, i.e. vcf, vcf.gz, or bcf. By default, the GRM is computed +using the expected haplotype counts with FORMAT ID = "HD". + +``` +grm chrm +``` + +will produce a binary `.mat` file that stores the GRM and relavent meta data. +Other options include + + + + +## Compute LOCO matrices ``` -hgrm path/to/my_vcf > grm +grm loco path/to/file/with/grm_filename_and_path_per_line ``` @@ -27,26 +36,10 @@ hgrm path/to/my_vcf > grm The program is only available as source from this repository and requires -* `cmake` (>= 3.31.4) -* `make` +* `GNU make` +* `htslib` * `clang` or `gcc` C++17 compiler -To install navigate to the top level directory of this repository, -and make the `build` directory -```bash -mkdir build -``` -then use `cmake` to generate `make` files, etc., -```bash -cmake -S . -B build/ -``` -followed by navigating to the build directory and running GNU `make` -```bash -make -``` -In the build directory you should now have a binary file called `hgrm`, -this is the executable program. Move it to a directory in your -shell's search path. ## Contributing @@ -55,13 +48,8 @@ I am using [GoogleTest](https://google.github.io/googletest/) framework for organizing tests. If you contribute, please make tests for your contributions. To run tests, `build` directory and build the project ``` -cmake -S ../ -B . -make -``` -Then use `cmake`'s utility -``` -ctest -``` +make check + ## Acknowledgement diff --git a/include/calc.h b/include/calc.h index b9c3052..345b9ba 100644 --- a/include/calc.h +++ b/include/calc.h @@ -10,9 +10,11 @@ int compute_genotype_matrix(); +int compute_eac_matrix(); + // -int compute_haplotype_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov); +int compute_ehc_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov); -int compute_geno_and_haplo_matrix(); +int compute_eac_and_ehc_matrix(); #endif diff --git a/include/logger.h b/include/logger.h index 78665c9..49504a5 100644 --- a/include/logger.h +++ b/include/logger.h @@ -6,6 +6,8 @@ #include #include #include +#include + class Logger { public: @@ -14,20 +16,19 @@ class Logger { // TODO: right now only accepts a single msg string, I should make // this arbitrary message elements using va_list, this makes the // interface match that of sprintf - int info(const char *format, const char *msg); - int warn(const char *format, const char *msg); - int error(const char *format, const char *msg); + int info(const char *format, ...); + int warn(const char *format, ...); + int error(const char *format, ...); - int info(const char *msg); - int warn(const char *msg); - int error(const char *msg); + // int info(const char *msg); + // int warn(const char *msg); + // int error(const char *msg); private: time_t t_; tm *time_point_; - int msg_len_ { 0 }; - size_t time_len_ { 0 }; + int status_ { 0 }; static constexpr size_t time_buf_len_ { 30 }; static constexpr size_t str_buf_len_ { 500 }; @@ -40,8 +41,10 @@ class Logger { static constexpr char warn_str_[] = { "WARN" }; static constexpr char info_str_[] = { "INFO" }; - int print_(FILE *stream, const char *log_type, - const char *format, const char *msg); + int load_time_buf_(); + void empty_bufs_(); + void vprintf_(FILE *stream, const char *log_type, + const char *format, va_list arg_ptr); }; #endif diff --git a/src/logger.cpp b/src/logger.cpp index 9f07ae3..ed0ffdc 100644 --- a/src/logger.cpp +++ b/src/logger.cpp @@ -4,70 +4,86 @@ Logger::Logger(): t_(time(nullptr)), - time_point_(localtime(&t_)) { + time_point_(localtime(&t_)) { empty_bufs_(); }; - std::memset(time_buf_, '\0', time_buf_len_); + +void Logger::empty_bufs_() { + std::memset(time_buf_, '\0', time_buf_len_); std::memset(str_buf_, '\0', str_buf_len_); -}; +} -int Logger::print_(FILE *stream, - const char *log_type, - const char *format, - const char *msg) { +int Logger::load_time_buf_() { // get time and format time string t_ = time(nullptr); time_point_ = localtime(&t_); - // strftime returns the number of characters written to buffer, // a 0 returned indicates an error has occured. - time_len_ = strftime(time_buf_, time_buf_len_,"%FT%H:%M:%S", time_point_); - - if(time_len_ == 0) { - std::memset(time_buf_, '\0', time_buf_len_); + return strftime(time_buf_, time_buf_len_,"%FT%H:%M:%S", time_point_); +} - fprintf(stderr, "%s\t%s\t%s\n", time_buf_, - err_str_, "logger time buf failure, please notify maintainer."); - return -1; - } - // construct logging message - // TODO: truncation of msg notification when msg exceeds buffer - // Recall that snprintf returns int less than 0 if an error occurs - msg_len_ = snprintf(str_buf_, max_str_, format, msg); - if (msg_len_ < 0) { - fprintf(stderr, "%s\t%s\t%s\n", time_buf_, - err_str_, "logger msg failure, please notify maintainer."); - return -1; +void Logger::vprintf_(FILE *stream, const char *log_type, const char *format, + va_list arg_ptr) { + + if ((status_ = load_time_buf_()) <= 0) + strncpy(str_buf_, + "logger time buffer failure, please notify maintainer.", + max_str_); + else if ((status_ = vsnprintf(str_buf_, max_str_, format, arg_ptr)) <= 0) + strncpy(str_buf_, + "logger msg failure, please notify maintainer.", + max_str_); + else if (status_ >= max_str_) { + status_ = -status_; + strncpy(str_buf_, + "logger msg too long, please shorten msg.", + max_str_); } - fprintf(stream, "%s\t%s\t%s\n", time_buf_, log_type, str_buf_); + if (status_ <= 0) + fprintf(stderr, "%s\t%s\t%s\n", time_buf_, err_str_, str_buf_); + else + fprintf(stream, "%s\t%s\t%s\n", time_buf_, log_type, str_buf_); - return 0; + empty_bufs_(); } -int Logger::info(const char *format, const char *msg) { - return print_(stdout, info_str_, format, msg); -} -int Logger::warn(const char *format, const char *msg) { - return print_(stdout, warn_str_, format, msg); +int Logger::info(const char *format, ...) { + va_list arg_ptr; + va_start(arg_ptr, format); + vprintf_(stdout, info_str_, format, arg_ptr); + va_end(arg_ptr); + return status_; } -int Logger::error(const char *format, const char *msg) { - return print_(stderr, err_str_, format, msg); +int Logger::warn(const char *format, ...) { + va_list arg_ptr; + va_start(arg_ptr, format); + vprintf_(stdout, warn_str_, format, arg_ptr); + va_end(arg_ptr); + return status_; } -int Logger::info(const char *msg) { - return print_(stdout, info_str_, "%s", msg); +int Logger::error(const char *format, ...) { + va_list arg_ptr; + va_start(arg_ptr, format); + vprintf_(stderr, err_str_, format, arg_ptr); + va_end(arg_ptr); + return status_; } -int Logger::warn(const char *msg) { - return print_(stdout, warn_str_, "%s", msg); -} - -int Logger::error(const char *msg) { - return print_(stderr, err_str_, "%s", msg); -} +// int Logger::info(const char *msg) { +// return info("%s", msg); +// } +// +// int Logger::warn(const char *msg) { +// return warn("%s", msg); +// } +// +// int Logger::error(const char *msg) { +// return error("%s", msg); +// } diff --git a/src/main.cpp b/src/main.cpp index ba860e6..cb00a8b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -40,137 +40,161 @@ int main(int argc, char* argv[]) // } argparse::ArgParser parser { - "hgrm: Haplotype Genetic Relationship Matrix", - "This program computes the haplotype genetic relationship matrix" - " from the expected haplotype counts per locus per sample and stored" - " as a text file in the variant call format (VCF)." + "grm: Genetic Relationship Matrix", + "This program provides tools for computing the genetic relationship" + " matrix (GRM) and the leave-one-chromosome-out (LOCO) matrices for" + " linear mixed effect based association studies. The GRM may be" + " computed using called genotypes, expected alternative allele counts," + " expected haplotype counts, or both expected alternative allele" + " and haplotype counts. By default the expected alternative allele" + " counts are used." }; - parser.add_arg("-o", + argparse::CmdDef *contig_cmd = parser.add_cmd("contig"); + + contig_cmd->add_arg("-o", argparse::ArgType::STRING, "the path and filename that the resulting haplotype genetic" " relationship matrix is printed."); - parser.add_arg("--sample_names", + contig_cmd->add_arg("--sample_names", argparse::ArgType::STRING, "The path and name of the file containing sample names to be" " included in computing the relationship matrix. The file must" " include a single sample filename, and if necessary file system" " path, per line."); - parser.add_arg("--gt", + contig_cmd->add_arg("--gt", argparse::ArgType::BOOLEAN, "Use sample genotypes to compute the relationship matrix"); - parser.add_arg("--eac", + contig_cmd->add_arg("--ehc", argparse::ArgType::BOOLEAN, - "Use sample expected alt allele count to compute the" - " relationship matrix"); + "Use sample expected haplotype count to compute the the genetic" + " relationship matrix."); - parser.add_arg("-b", + contig_cmd->add_arg("-b", argparse::ArgType::BOOLEAN, "Use both the expected alternative allele and haplotype counts to" - " compute relationship matrix"); - - parser.add_arg("--loco", - argparse::ArgType::STRING, - "Directory with chromosome matrix files to compute the" - " leave-one-chromosome-out (LOCO) relationship matrix.") + " compute the genetic relationship matrix"); - parser.add_arg("--vcf", + contig_cmd->add_arg("bcf", argparse::ArgType::STRING, - "the path and filename of the vcf in which the hgrm is computed."); - - if (parser.parse_args(argc, argv) != argparse::ArgStatus::SUCCESS) { - fprintf(stderr, "Error: couldn't parse command line args, exiting\n"); - exit(EXIT_FAILURE); - } + "The path and filename of the genetic data to compute the GRM. The" + " data may be in any of the htslib supported formats, i.e. vcf," + " vcf.gz, or bcf."); - // TODO: Update below to use logger - // - std::optional tmp_str {}; - if((tmp_str = parser.get("vcf")) == std::nullopt) { - fprintf(stderr, "Error retrieving vcf name"); - exit(EXIT_FAILURE); - } - std::string vcf_fname { tmp_str.value() }; - if ((tmp_str = parser.get("o")) == std::nullopt) { - fprintf(stderr, "Error retrieving output name"); - exit(EXIT_FAILURE); - } - std::string out_fname { tmp_str.value() }; + argparse::CmdDef *loco_cmd = parser.add_cmd("loco"); + loco_cmd->add_arg("filename", + argparse::ArgType::STRING, + "Name, and path, of file that stores the name and paths of matrix" + " files used to compute leave-one-chromosome-out (LOCO) relationship" + " matrix."); - if (out_fname.size() == 0) - out_fname = vcf_fname + ".mat"; - if ((tmp_str = parser.get("sample_names")) == std::nullopt) { - fprintf(stderr, "Error retrieving sample_names file.\n"); - exit(EXIT_FAILURE); - } - std::string samp_fname { tmp_str.value() }; - - std::optional tmp_bool {}; - if ((tmp_bool = parser.get("gt")) == std::nullopt) { - fprintf(stderr, "Error retrieving relationship matrix type.\n"); - exit(EXIT_FAILURE); - } - bool use_gt { tmp_bool.value() }; - - if ((tmp_bool = parser.get("b")) == std::nullopt) { - fprintf(stderr, "Error retrieving relationship matrix type.\n"); - exit(EXIT_FAILURE); - } - bool use_both { tmp_bool.value() }; - - if ((tmp_bool = parser.get("eac")) == std::nullopt) { - fprintf(stderr, "Error retrieving relationship matrix type.\n"); - exit(EXIT_FAILURE); - } - bool use_eac { tmp_bool.value() }; + Logger log {}; + int status = FAILED_CALC; + argparse::ArgStatus arg_status = parser.parse_args(argc, argv); + // PARSE ARGUMENTS + if (arg_status == argparse::ArgStatus::HELP) + return 0; - if ((use_gt && use_both) || (use_gt && use_ds) || (use_both && use_ds)) { - fprintf(stderr, "user must specify either use_gt, use_both, use_ds," - " or omit both options to compute the haplotype based" - " relationship matrix."); + if (arg_status != argparse::ArgStatus::SUCCESS) { + log.error("Error: couldn't parse command line args, exiting\n"); exit(EXIT_FAILURE); } - - Logger log {}; + // EXTRACT ARGS + if (parser.is_sub_cmd("contig")) { + + std::optional tmp_str {}; + if((tmp_str = parser.get("bcf")) == std::nullopt) { + log.error("Error retrieving vcf name"); + exit(EXIT_FAILURE); + } + std::string bcf_fname { tmp_str.value() }; + + if ((tmp_str = parser.get("o")) == std::nullopt) { + log.error("Error retrieving output name"); + exit(EXIT_FAILURE); + } + std::string out_fname { tmp_str.value() }; + + if (out_fname.size() == 0) + out_fname = bcf_fname + ".mat"; + + if ((tmp_str = parser.get("sample_names")) == std::nullopt) { + log.error("Error retrieving sample_names file.\n"); + exit(EXIT_FAILURE); + } + std::string samp_fname { tmp_str.value() }; + + + std::optional tmp_bool {}; + if ((tmp_bool = parser.get("gt")) == std::nullopt) { + log.error("Error retrieving relationship matrix type.\n"); + exit(EXIT_FAILURE); + } + bool use_gt { tmp_bool.value() }; + + if ((tmp_bool = parser.get("ehc")) == std::nullopt) { + log.error("Error retrieving relationship matrix type.\n"); + exit(EXIT_FAILURE); + } + bool use_ehc { tmp_bool.value() }; + + if ((tmp_bool = parser.get("b")) == std::nullopt) { + log.error("Error retrieving relationship matrix type.\n"); + exit(EXIT_FAILURE); + } + bool use_both { tmp_bool.value() }; + + + + if ((use_gt && use_both) || (use_gt && use_ehc) || (use_both && use_ehc)) { + log.error("user must specify either use_gt, use_both, use_ds," + " or omit both options to compute the haplotype based" + " relationship matrix."); + exit(EXIT_FAILURE); + } - log.info("BCF/VCF file name: %s", vcf_fname.c_str()); - if (samp_fname.size() == 0) - log.info("Sample file: None, use all samples"); - else - log.info("Sample file: %s", samp_fname.c_str()); - - log.info("Output matrix file: %s", out_fname.c_str()); - - int status = FAILED_CALC; + log.info("BCF/VCF file name: %s", bcf_fname.c_str()); + if (samp_fname.size() == 0) + log.info("Sample file: None, use all samples"); + else + log.info("Sample file: %s", samp_fname.c_str()); + + log.info("Output matrix file: %s", out_fname.c_str()); + + + bcfio::ReadBcf bfid { bcf_fname.c_str() }; + Matrix cov { bfid.n_samples(), bfid.n_samples() }; + + if (use_gt) { + log.info("Relationship matrix: genotype"); + status = compute_genotype_matrix(); + } else if (use_ehc) { + log.info("Relationship matrix: expected haplotype count"); + status = compute_ehc_matrix(&log, &bfid, &cov); + } else if (use_both) { + log.info("Relationship matrix: expected alt allele and haplotype" + " counts"); + status = compute_eac_and_ehc_matrix(); + } else { + log.info("Relationship matrix: expected alternative allele counts"); + status = compute_eac_matrix(); + } + + if (status == FAILED_CALC) + log.error("Computation failed"); - bcfio::ReadBcf bfid { vcf_fname.c_str() }; - Matrix cov { bfid.n_samples(), bfid.n_samples() }; - - if (use_gt) { - log.info("Relationship matrix: genotype"); - status = compute_genotype_matrix(); - } else if (use_ds) { - log.info("Relationship matrix: expected alt allele count - status = compute_eac_matrix(); - } else if (use_both) { - log.info("Relationship matrix: expected alt allele and haplotype" - " counts"); - status = compute_geno_and_haplo_matrix(); - } else { - log.info("Relationship matrix: haplotype"); - status = compute_haplotype_matrix(&log, &bfid, &cov); } - if (status == FAILED_CALC) - log.error("Computation failed"); + if (parser.is_sub_cmd("loco")) + printf("loco selected\n"); return status; diff --git a/src/use_both.cpp b/src/use_both.cpp index f3011c0..807097e 100644 --- a/src/use_both.cpp +++ b/src/use_both.cpp @@ -1,6 +1,7 @@ - +// Compute GRM with expected alt allele and haplotype counts +// #include -int compute_geno_and_haplo_matrix() { +int compute_eac_and_ehc_matrix() { return -1; } diff --git a/src/use_eac.cpp b/src/use_eac.cpp new file mode 100644 index 0000000..984ebdf --- /dev/null +++ b/src/use_eac.cpp @@ -0,0 +1,6 @@ + +#include + +int compute_eac_matrix() { + return -1; +} diff --git a/src/use_haplotype.cpp b/src/use_ehc.cpp similarity index 98% rename from src/use_haplotype.cpp rename to src/use_ehc.cpp index 716e3a5..45a4d8b 100644 --- a/src/use_haplotype.cpp +++ b/src/use_ehc.cpp @@ -2,7 +2,7 @@ #include -int compute_haplotype_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov) { +int compute_ehc_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov) { int output_status = 0; diff --git a/src/use_genotype.cpp b/src/use_gt.cpp similarity index 65% rename from src/use_genotype.cpp rename to src/use_gt.cpp index 99cb7f7..5abc58f 100644 --- a/src/use_genotype.cpp +++ b/src/use_gt.cpp @@ -1,3 +1,4 @@ +// Compute GRM with called genotypes #include From 7b94ce05d33be2dad28e2999a14f41be883eb890 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 3 Jan 2026 09:15:37 -0500 Subject: [PATCH 20/58] intermediate and incomplete update of grm writer call in main function. --- src/main.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main.cpp b/src/main.cpp index cb00a8b..824fa89 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -191,6 +191,9 @@ int main(int argc, char* argv[]) if (status == FAILED_CALC) log.error("Computation failed"); + log.info("Writing to file"); + + grmio::write( cov); } if (parser.is_sub_cmd("loco")) From a710901c57d0d47d22f5d046b57bd4951af03e1c Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 3 Jan 2026 09:33:13 -0500 Subject: [PATCH 21/58] Changed genetic matrix to grm specific so that I can include the binary file I/O as class methods. --- include/bcfio.h | 2 +- include/calc.h | 4 ++-- include/{matrix.h => grm.h} | 25 ++++++++++++++++--------- src/{matrix.cpp => grm.cpp} | 34 +++++++++++++++++++++++----------- src/main.cpp | 6 +++--- src/use_ehc.cpp | 2 +- tests/test_matrix.cpp | 26 +++++++++++++------------- 7 files changed, 59 insertions(+), 40 deletions(-) rename include/{matrix.h => grm.h} (57%) rename src/{matrix.cpp => grm.cpp} (61%) diff --git a/include/bcfio.h b/include/bcfio.h index f02b602..6cfb625 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -16,7 +16,7 @@ #include #include -#include +#include namespace htslib { extern "C" { diff --git a/include/calc.h b/include/calc.h index 345b9ba..284e4b7 100644 --- a/include/calc.h +++ b/include/calc.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include int compute_genotype_matrix(); @@ -13,7 +13,7 @@ int compute_genotype_matrix(); int compute_eac_matrix(); // -int compute_ehc_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov); +int compute_ehc_matrix(Logger *log, bcfio::ReadBcf *bfid, Grm *cov); int compute_eac_and_ehc_matrix(); diff --git a/include/matrix.h b/include/grm.h similarity index 57% rename from include/matrix.h rename to include/grm.h index dce983c..59065d4 100644 --- a/include/matrix.h +++ b/include/grm.h @@ -11,23 +11,24 @@ // (Jan 2025), with minor recommendations incorporated. // // -#ifndef HEADER_MATRIX_H -#define HEADER_MATRIX_H +#ifndef HEADER_GRM_H +#define HEADER_GRM_H +#include #include #include #include #include #include -class Matrix -{ + +class Grm { public: - Matrix(const size_t, const size_t); - Matrix(const Matrix&); // copy constructor - Matrix(Matrix&&); // move constructor - Matrix& operator=(const Matrix&)=delete; // copy assignment - Matrix& operator=(Matrix&&)=delete; // move assignment + Grm(const size_t, const size_t); + Grm(const Grm&); // copy constructor + Grm(Grm&&); // move constructor + Grm& operator=(const Grm&)=delete; // copy assignment + Grm& operator=(Grm&&)=delete; // move assignment double operator()(const size_t&, const size_t&) const; @@ -36,6 +37,11 @@ class Matrix size_t size() const; std::array dims() const; + int write(const char *filename) const; + int write(const std::string& filename) const; + static int read(const char *filename, Grm *grm); + static int read(const std::string& filename, Grm *grm); + private: const size_t nrow_; const size_t mcol_; @@ -43,4 +49,5 @@ class Matrix size_t mat_idx_to_array_(const size_t&, const size_t&) const; }; + #endif diff --git a/src/matrix.cpp b/src/grm.cpp similarity index 61% rename from src/matrix.cpp rename to src/grm.cpp index 3449980..028291b 100644 --- a/src/matrix.cpp +++ b/src/grm.cpp @@ -13,15 +13,15 @@ // // -#include +#include // default constructor -Matrix::Matrix(const size_t nrow, const size_t mcol) +Grm::Grm(const size_t nrow, const size_t mcol) : nrow_(nrow), mcol_(mcol), data_(nrow_ > 0 && mcol_ > 0 ? std::make_unique(size()) : nullptr) { if (nrow_ == 0 || mcol_ == 0) - throw std::runtime_error("Matrix must have minimum size of 1"); + throw std::runtime_error("Grm must have minimum size of 1"); // set default values to zero for (size_t i = 0; i < size(); i++) @@ -31,34 +31,34 @@ Matrix::Matrix(const size_t nrow, const size_t mcol) // copy constructor // -Matrix::Matrix(const Matrix& other) +Grm::Grm(const Grm& other) : nrow_(other.nrow_), mcol_(other.mcol_), data_(std::make_unique(other.size())) { - // Matrix values have already been validated + // Grm values have already been validated for (size_t i = 0; i < size(); i++) data_[i] = other.data_[i]; } // TODO: check this. -Matrix::Matrix(Matrix&& other) +Grm::Grm(Grm&& other) : nrow_(other.nrow_), mcol_(other.mcol_), data_(std::move(other.data_)) {}; -double Matrix::operator()(const size_t& i, const size_t& j) const { +double Grm::operator()(const size_t& i, const size_t& j) const { return data_[mat_idx_to_array_(i, j)]; } -double& Matrix::operator()(const size_t& i, const size_t& j) { +double& Grm::operator()(const size_t& i, const size_t& j) { return data_[mat_idx_to_array_(i, j)]; } -std::array Matrix::dims() const { +std::array Grm::dims() const { return {nrow_, mcol_}; } -size_t Matrix::mat_idx_to_array_(const size_t& i, const size_t& j) const { +size_t Grm::mat_idx_to_array_(const size_t& i, const size_t& j) const { if (i >= nrow_ || j >= mcol_) throw std::runtime_error("Indices must be postive integers or zero."); @@ -66,4 +66,16 @@ size_t Matrix::mat_idx_to_array_(const size_t& i, const size_t& j) const { } -size_t Matrix::size() const { return nrow_ * mcol_; }; +size_t Grm::size() const { return nrow_ * mcol_; }; + +int Grm::write(const std::string& filename) const { + return -1; +} + +int Grm::write(const char *filename) const { + return -1; +} + +int Grm::read(const char *filename, Grm *grm) { + return -1; +} diff --git a/src/main.cpp b/src/main.cpp index 824fa89..7a9d8bb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -21,7 +21,7 @@ #include #include -#include +#include #define FAILED_CALC -1 @@ -171,7 +171,7 @@ int main(int argc, char* argv[]) bcfio::ReadBcf bfid { bcf_fname.c_str() }; - Matrix cov { bfid.n_samples(), bfid.n_samples() }; + Grm cov { bfid.n_samples(), bfid.n_samples() }; if (use_gt) { log.info("Relationship matrix: genotype"); @@ -193,7 +193,7 @@ int main(int argc, char* argv[]) log.info("Writing to file"); - grmio::write( cov); + cov.write(out_fname); } if (parser.is_sub_cmd("loco")) diff --git a/src/use_ehc.cpp b/src/use_ehc.cpp index 45a4d8b..79d0081 100644 --- a/src/use_ehc.cpp +++ b/src/use_ehc.cpp @@ -2,7 +2,7 @@ #include -int compute_ehc_matrix(Logger *log, bcfio::ReadBcf *bfid, Matrix *cov) { +int compute_ehc_matrix(Logger *log, bcfio::ReadBcf *bfid, Grm *cov) { int output_status = 0; diff --git a/tests/test_matrix.cpp b/tests/test_matrix.cpp index 7fda809..c95575a 100644 --- a/tests/test_matrix.cpp +++ b/tests/test_matrix.cpp @@ -1,13 +1,13 @@ #include -#include +#include #include -TEST(TestMatrix, Init) { +TEST(TestGrm, Init) { size_t n_row { 3 }; size_t m_col { 2 }; - Matrix a { n_row, m_col }; + Grm a { n_row, m_col }; std::array dims { a.dims() }; EXPECT_EQ(dims[0], n_row); EXPECT_EQ(dims[1], m_col); @@ -21,27 +21,27 @@ TEST(TestMatrix, Init) { EXPECT_THROW({ size_t n_row = 0; size_t m_col = 2; - Matrix b(n_row, m_col); + Grm b(n_row, m_col); }, std::runtime_error); EXPECT_ANY_THROW({ size_t n_row = -1; size_t m_col = 2; - Matrix b(n_row, m_col); + Grm b(n_row, m_col); }); - EXPECT_THROW({Matrix b(1, 0);}, std::runtime_error); - EXPECT_ANY_THROW({Matrix b(1, -1);}); + EXPECT_THROW({Grm b(1, 0);}, std::runtime_error); + EXPECT_ANY_THROW({Grm b(1, -1);}); } -TEST(TestMatrix, Vals) { +TEST(TestGrm, Vals) { size_t n_row { 3 }; size_t m_col { 5 }; - Matrix a { n_row, m_col }; + Grm a { n_row, m_col }; std::array dims { a.dims() }; @@ -57,11 +57,11 @@ TEST(TestMatrix, Vals) { } -TEST(TestMatrix, OutOfBounds) { +TEST(TestGrm, OutOfBounds) { size_t n_row { 3 }; size_t m_col { 5 }; - Matrix a { n_row, m_col }; + Grm a { n_row, m_col }; EXPECT_THROW({ a(4, 3); }, std::runtime_error); EXPECT_THROW({ a(3, 5); }, std::runtime_error); @@ -72,11 +72,11 @@ TEST(TestMatrix, OutOfBounds) { } -TEST(TestMatrix, DimAndSize) { +TEST(TestGrm, DimAndSize) { size_t n_row { 3 }; size_t m_col { 5 }; - Matrix a { n_row, m_col }; + Grm a { n_row, m_col }; std::array dims { a.dims() }; EXPECT_EQ(dims[0], n_row); From ab2294949f26e01da8d2c75a78e1c90af6a03c17 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 7 Jan 2026 08:10:48 -0500 Subject: [PATCH 22/58] unfinished update --- include/grm.h | 118 ++++++++++++++++++++++++--- include/textio.h | 40 ++++++++++ src/grm.cpp | 203 ++++++++++++++++++++++++++++++++++++++--------- src/textio.cpp | 67 ++++++++++++++++ 4 files changed, 379 insertions(+), 49 deletions(-) create mode 100644 include/textio.h create mode 100644 src/textio.cpp diff --git a/include/grm.h b/include/grm.h index 59065d4..51e60a9 100644 --- a/include/grm.h +++ b/include/grm.h @@ -16,12 +16,102 @@ #include #include -#include +#include +#include +#include #include -#include #include +namespace grm { + +namespace details { + +// @title: Count the number of non empty lines in text file +// +// @param fid: pointer to C file stream, i.e. that returned by fopen +// @param num_lines: the number of lines written at this address +// @return -1: file I/O error as determined by ferror(fid), or +// -2: end of file not reached, reason undetermined, or +// -3: error in returning file handle to beginning of file +// 0: success +int num_lines_in_file(FILE *fid, size_t *num_lines); + + +int chars_to_size_t(FILE *fid, size_t *val); + +} + + +enum class STATUS { + SUCCESS, + FAILED, + UNKNOWN_FAILURE, + ERROR_IDX_ARR_BOUNDS, + ERROR_FOPEN, + ERROR_EOF_NOT_REACHED, + ERROR_ON_WRITE, +}; + + +struct Dims { + Dims(size_t nrow_in, size_t mcol_in): + nrow(nrow_in), mcol(mcol_in) {}; + + const size_t nrow; + const size_t mcol; +}; + + +// @title: Store genomic coordinates and mange binary I/O +// @description: +struct Coordinates { + + // @param pos_filename: The name, and path if necessary, of the text + // file specifying variant positions on the specified contig + // to be included for the grm. + // @param contig_name: The name of the contig, e.g. chrm1 + // + Coordinates(const char *contig_name, const size_t len); + Coordinates(const std::string& contig_name, const size_t len); + + size_t operator[](size_t idx) const; + size_t& operator[](size_t idx); + + STATUS write(FILE *fid); + static STATUS read(FILE *fid, Coordinates *coords); + + const size_t len; + const std::string contig; + std::unqiue_ptr *pos; +}; + + +struct Samples { + Samples(const char *sample_filename); + Samples(const std::string sample_filename); + + const std::string filename; + const size_t len; + std::unique_ptr *names; + + STATUS bin_write(FILE *fid); + static STATUS bin_read(FILE *fid, Samples *samples); +} + + +struct Hdr { + const std::string version; + const std::string data_type; + const Coordinates *coords; + const Samples *samples; + + STATUS bin_write(FILE *fid); + static STATUS bin_read(FILE *fid, Hdr *hdr); + +} + + class Grm { public: Grm(const size_t, const size_t); @@ -30,24 +120,26 @@ class Grm { Grm& operator=(const Grm&)=delete; // copy assignment Grm& operator=(Grm&&)=delete; // move assignment + // Unchecked indexes when setting and getting of matrix values + float operator()(const size_t i, const size_t j) const; + float& operator()(const size_t i, const size_t j); - double operator()(const size_t&, const size_t&) const; - double& operator()(const size_t&, const size_t&); + // Checked indexes when setting and getting of matrix values + STATUS set(const size_t i, const size_t j, const float val); + STATUS get(const size_t i, const size_t j, float *val) const; size_t size() const; - std::array dims() const; + const Dims& dims() const; - int write(const char *filename) const; - int write(const std::string& filename) const; - static int read(const char *filename, Grm *grm); - static int read(const std::string& filename, Grm *grm); + STATUS write(const char *filename, const char Hdr *hdr) const; + static STATUS read(const char *filename, Grm *grm); private: - const size_t nrow_; - const size_t mcol_; - std::unique_ptr data_; - size_t mat_idx_to_array_(const size_t&, const size_t&) const; + const Dims dims_; + std::unique_ptr data_; + size_t midx_to_arr_(const size_t&, const size_t&) const; }; +} #endif diff --git a/include/textio.h b/include/textio.h new file mode 100644 index 0000000..5e91ed3 --- /dev/null +++ b/include/textio.h @@ -0,0 +1,40 @@ + +#include +#include + + +#ifndef HEADER_TEXTIO_H +#define HEADER_TEXTIO_H + +namespace details { +const size_t DEFAULT_BUF_SIZE = 100; +} + + +// @title: Parsing text files +// @description: This class manages the lifetime of a C-style file stream +// by RAII, line retrieval, and getting line unumber. +class TextIO { +public: + + TextIO(const char *filename, const char *mode); + TextIO(const char *filename, const char *mode, const size_t buf_size); + ~TextIO(); + + int num_lines(); + int get_line(); + +private: + std::string fname_; + const size_t buf_size_; + char *buf_; + size_t buf_line_len_ = 0; + + FILE *fid_ = nullptr; + +}; + + +TextIO text_open(const char *filename, const char *mode); + +#endif diff --git a/src/grm.cpp b/src/grm.cpp index 028291b..feea726 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -15,67 +15,198 @@ #include +int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { + // TODO: errno, need to reset? + + size_t line_num = 0; + size_t word_len = 0; + int c; + while ((c = fgetc(fid)) != EOF) { + + if (c == '\n' && word_len != 0) { + line_num++; + word_len = 0; + } else if (c != '\n') + word_len++; + } + + if (ferror(fid)) + return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; + + if (feof(fid) == 0) + return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; + + *num_lines = line_num; + return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; +} + + +int grm::details::get_size_t(FILE *fid, size_t *val) { + + std::string s { "" }; + while (std::getline(fid, s)) + if (s.size() < ) + + int c; + for (int i = 0; (c = fgetc(fid)) != EOF && i < max_bitsize_size_t; i++) { + if (c == '\n') + break; + s[i] = c; + } + s[i] = '\0'; + + return 0; +} + + +grm::Coordinates::Coordinates(const char *contig_name): + contig(contig_name) {}; + + +grm::Coordinates::Coordinates(const std::string& contig_name): + contig(contig_name) {}; + + +grm::STATUS grm::Coordinates::parse_input_file(const char *pos_filename) { + + fid = fopen(pos_filename, "r"); + if (ferror(fid)) + return grm::STATUS::ERROR_FOPEN; + + + grm::STATUS status { grm::STATUS::UNKNOWN_FAILURE }; + + // Use switch the interpret, and act accordingly, to the returned interger + // status code + switch (grm::details::num_lines_in_file(fid, &len)) { + case -1: + return grm::STATUS::ERROR_FOPEN; + case -2: + return grm::STATUS::ERROR_EOF_NOT_REACHED; + case 0: + status = grm::STATUS::SUCCESS; + break; + default: + return status; + } + + pos = make_unique(len); + + size_t num_bits_size_t = static_cast(CHAR_BIT * sizeof(size_t)); + std::string s {}; + + for (size_t i = 0; i < len; i++) { + + + if (s.size() >= num_bits_size_t) + return STATUS::FAILED; + + pos[i] = static_cast(std::strtoull(s)); + } + + fid +} + +grm::STATUS grm::Coordinates::parse_input_file(const std::string& pos_filename) { + return grm::Coordinates::parse_input_file(pos_filename.c_str()); +} + + // default constructor -Grm::Grm(const size_t nrow, const size_t mcol) - : nrow_(nrow), mcol_(mcol), - data_(nrow_ > 0 && mcol_ > 0 ? std::make_unique(size()) : nullptr) { - - if (nrow_ == 0 || mcol_ == 0) - throw std::runtime_error("Grm must have minimum size of 1"); +grm::Grm::Grm(const size_t nrow, const size_t mcol) + : dims_(nrow, mcol), + data_(size() != 0 ? std::make_unique(size()) : nullptr) { - // set default values to zero - for (size_t i = 0; i < size(); i++) - data_[i] = 0; - }; + if (data_) + std::memset(data_.get(), 0, size()); +} // copy constructor // -Grm::Grm(const Grm& other) - : nrow_(other.nrow_), mcol_(other.mcol_), - data_(std::make_unique(other.size())) { - - // Grm values have already been validated - for (size_t i = 0; i < size(); i++) - data_[i] = other.data_[i]; +grm::Grm::Grm(const grm::Grm& other) + : nrow_(dims.other.nrow_), mcol_(dims.other.mcol_), + data_(std::make_unique(other.size())) { + std::memset(data_.get(), 0, size()); } + // TODO: check this. -Grm::Grm(Grm&& other) - : nrow_(other.nrow_), mcol_(other.mcol_), data_(std::move(other.data_)) {}; +grm::Grm::Grm(grm::Grm&& other) + : nrow_(dims_.other.nrow_), dims.mcol_(other.mcol_), + data_(std::move(other.data_)) {}; -double Grm::operator()(const size_t& i, const size_t& j) const { - return data_[mat_idx_to_array_(i, j)]; +float grm::Grm::operator()(const size_t i, const size_t j) const { + return data_[i*mcol_ + j]; } -double& Grm::operator()(const size_t& i, const size_t& j) { - return data_[mat_idx_to_array_(i, j)]; + +float& grm::Grm::operator()(const size_t i, const size_t j) { + return data_[i*mcol_ + j]; } -std::array Grm::dims() const { - return {nrow_, mcol_}; + +grm::STATUS grm::Grm::get(const size_t i, const size_t j, float *val) const { + size_t idx = 0; + grm::STATUS status = grm::STATUS::FAILED; + if ((status = midx_to_arr_(i, j, &idx)) != grm::STATUS::SUCCESS) + return status; + + *val = data_[idx]; + + return status; } -size_t Grm::mat_idx_to_array_(const size_t& i, const size_t& j) const { - if (i >= nrow_ || j >= mcol_) - throw std::runtime_error("Indices must be postive integers or zero."); +grm::STATUS grm::Grm::set(const size_t i, const size_t j, const float val) { + size_t idx = 0; + grm::STATUS status = grm::STATUS::FAILED; + if ((status = midx_to_arr_(i, j, &idx)) != grm::STATUS::SUCCESS) + return status; - return i*mcol_ + j; + data_[idx] = val; + + return status; } -size_t Grm::size() const { return nrow_ * mcol_; }; +const grm::Dims& grm::Grm::dims() const { return dims_; }; + -int Grm::write(const std::string& filename) const { - return -1; +grm::STATUS grm::Grm::midx_to_arr_(const size_t i, const size_t j, size_t *idx) const { + + if (i >= nrow_ || j >= mcol_) + return grm::STATUS::ERROR_IDX_ARR_BOUNDS; + + *idx = i*mcol_ + j; + return grm::STATUS::SUCCESS; } -int Grm::write(const char *filename) const { - return -1; + +size_t grm::Grm::size() const { return dims_.nrow_ * dims_.mcol_; }; + + +grm::STATUS grm::Grm::write(const char *filename) const { + + std::unique_ptr fid = make_unique(fopen(filename, "wb")); + + size_t size_written = fwrite(&dims_, sizeof(Dims), 1, fid.get()); + if (size_written < 1) + return grm::STATUS::ERROR_ON_WRITE; + + size_written = fwrite(data_.get(), + sizeof(float), + size(), + fid.get()); + + if (size_written < size()) + return grm::STATUS::ERROR_ON_WRITE; + + return grm::STATUS::SUCCESS; } -int Grm::read(const char *filename, Grm *grm) { - return -1; + +grm::STATUS grm::Grm::read(const char *filename, grm::Grm *grm) { + return grm::STATUS; } diff --git a/src/textio.cpp b/src/textio.cpp new file mode 100644 index 0000000..88478aa --- /dev/null +++ b/src/textio.cpp @@ -0,0 +1,67 @@ + +#include + +TextIO::TextIO(const char *filename): + fname_(filename), + buf_size_(details::DEFAULT_BUF_SIZE), + buf_(new char[buf_size_]) { + + std::memset(buf_, '\0', buf_size_); +}; + + +TextIO::TextIO(const char *filename, const size_t buf_size): + fname_(filename), + buf_size_(buf_size), + buf_(new char[buf_size_]) { + + std::memset(buf_, '\0', buf_size_); +}; + + +int TextIO::num_lines() { + + size_t line_num = 0; + size_t word_len = 0; + int c; + while (get_line(fid)) { + + if (c == '\n' && word_len != 0) { + line_num++; + word_len = 0; + } else if (c != '\n') + word_len++; + } + + if (ferror(fid)) + return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; + + if (feof(fid) == 0) + return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; + + *num_lines = line_num; + return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; +} + + +int get_line(FILE *fid) { + int c; + while ((c = fgetc(fid)) != EOF) { + + if (c == '\n' && word_len != 0) { + line_num++; + word_len = 0; + } else if (c != '\n') + word_len++; + } + + if (ferror(fid)) + return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; + + if (feof(fid) == 0) + return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; + + *num_lines = line_num; + return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; + +} From 04c78558b8558397b70b4b5c0a687cf5d3f4521c Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 13 Jan 2026 10:23:48 -0500 Subject: [PATCH 23/58] intermediate commit to move code to another dev environment. --- include/bcfio.h | 5 ++++- src/bcfio.cpp | 30 +++++++++--------------------- src/main.cpp | 21 ++++++++++++++++++--- tests/samples_test | 5 +++++ 4 files changed, 36 insertions(+), 25 deletions(-) create mode 100644 tests/samples_test diff --git a/include/bcfio.h b/include/bcfio.h index 6cfb625..fc38538 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -12,6 +12,7 @@ #define HEADER_PARSE_HTS_H #include +#include #include #include #include @@ -30,6 +31,7 @@ extern "C" { namespace bcfio { + // @title The meta data on a BCF attribute // @description BCF, VCF, and VCF.GZ files hold metadata in the header that // specify the type and format of data in records. I call each unique @@ -173,7 +175,6 @@ class ReadBcf public: // TODO: Review C++ idioms the rule of three and five ReadBcf(const char *bcfname); - ReadBcf(const char *bcfname, const char *sample_fname); ReadBcf()=delete; ReadBcf(const ReadBcf&)=delete; @@ -195,6 +196,8 @@ class ReadBcf // dictionaries in the VCF. BCF_DT_SAMPLE, provides the index of n // that correspondes to the number of samples. size_t n_samples() const { return hdr_.n_samples(); }; + + int set_samples(const char *filename); // TODO: sample_names const std::unique_ptr sample_names() const { diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 0103a5e..a3494ff 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -131,37 +131,25 @@ bcfio::ReadBcf::ReadBcf(const char *bcfname) hdr_(fid_) {}; +bcfio::ReadBcf::~ReadBcf() { + if (fid_) htslib::hts_close(fid_); +} + + // TODO: subset samples by those in sample_fname file -bcfio::ReadBcf::ReadBcf(const char *bcfname, const char *sample_fname) - : fname_(bcfname), - fid_(htslib::hts_open(bcfname, "r")), - hdr_(fid_) { +int bcfio::ReadBcf::set_samples(const char *sample_fname) { int status { 0 }; // Subset samples with those found in the file sample_fname - if (!sample_fname || *sample_fname == '\0') + if (!sample_fname || *sample_fname == '\0') { fprintf(stdout, "No file with sample names detected, retreiving" " records for all samples.\n"); - else - status = hdr_.subset_samples(sample_fname); - - if (status < 0) { - fprintf(stderr, "Error: Couldn't read sample file\n"); - exit(EXIT_FAILURE); - } else if (status > 0) { - fprintf(stderr, "Error: A subset of samples in sample file are not" - " found in the VCF,BCF, or VCF.GZ file.\n"); - exit(EXIT_FAILURE); + return -1; } - - // get number of characters in data record for line buffer size + return hdr_.subset_samples(sample_fname); }; -bcfio::ReadBcf::~ReadBcf() { - if (fid_) htslib::hts_close(fid_); -} - // title: load next record int bcfio::ReadBcf::next_record(bcfio::BcfFloatRecord *ptr, const char *id) { diff --git a/src/main.cpp b/src/main.cpp index 7a9d8bb..9bcc23c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -130,6 +130,7 @@ int main(int argc, char* argv[]) log.error("Error retrieving sample_names file.\n"); exit(EXIT_FAILURE); } + std::string samp_fname { tmp_str.value() }; @@ -162,15 +163,29 @@ int main(int argc, char* argv[]) } log.info("BCF/VCF file name: %s", bcf_fname.c_str()); + + + bcfio::ReadBcf bfid { bcf_fname.c_str() }; + + int bstatus = 0; if (samp_fname.size() == 0) log.info("Sample file: None, use all samples"); - else + else if ((bstatus = bfid.set_samples(samp_fname.c_str())) == 0) log.info("Sample file: %s", samp_fname.c_str()); + else if (bstatus < 0) { + log.error("Subsetting by sample file, %s, resulted in error", + samp_fname.c_str()); + return -1; + } else if (bstatus > 0) { + log.error("One or more samples specified in sample file, %s," + " do not %s", + samp_fname.c_str(), + bcf_fname.c_str()); + return -1; + } log.info("Output matrix file: %s", out_fname.c_str()); - - bcfio::ReadBcf bfid { bcf_fname.c_str() }; Grm cov { bfid.n_samples(), bfid.n_samples() }; if (use_gt) { diff --git a/tests/samples_test b/tests/samples_test new file mode 100644 index 0000000..fc07ba5 --- /dev/null +++ b/tests/samples_test @@ -0,0 +1,5 @@ +S03 +S05 +S06 +S08 +S11 From 9ebb4b9e6593ca9a39957114877ab23995ddb72b Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 15 Jan 2026 21:05:02 +0000 Subject: [PATCH 24/58] intermediate update. --- Makefile | 6 +- include/grm.h | 38 ++++++++----- include/textio.h | 79 ++++++++++++++++++++------ src/bcfio.cpp | 1 - src/grm.cpp | 34 ++++++------ src/textio.cpp | 141 +++++++++++++++++++++++++++++++++-------------- 6 files changed, 205 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index 0b85d71..67ea8d0 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,7 @@ TEST_TARGET_PRG = $(BUILD_DIR)/runtests # Executable Build Rules ###################################################################### -TARGET = $(BUILD_DIR)/hgrm +TARGET = $(BUILD_DIR)/grm .PHONY: all all: $(TARGET) $(TEST_TARGET_PRG) data @@ -149,10 +149,10 @@ check: .PHONY: help help: - -@echo "build hgrm" + -@echo "build grm" -@echo "2025 Palmer Lab" -@echo "" - -@echo "make hgrm executable" + -@echo "make grm executable" -@echo "make libargparse" diff --git a/include/grm.h b/include/grm.h index 51e60a9..c0ee860 100644 --- a/include/grm.h +++ b/include/grm.h @@ -21,6 +21,7 @@ #include #include #include +#include namespace grm { @@ -38,7 +39,7 @@ namespace details { int num_lines_in_file(FILE *fid, size_t *num_lines); -int chars_to_size_t(FILE *fid, size_t *val); +// int chars_to_size_t(FILE *fid, size_t *val); } @@ -83,33 +84,31 @@ struct Coordinates { const size_t len; const std::string contig; - std::unqiue_ptr *pos; + std::unique_ptr *pos; }; struct Samples { - Samples(const char *sample_filename); - Samples(const std::string sample_filename); - - const std::string filename; const size_t len; - std::unique_ptr *names; + std::unique_ptr names; + + STATUS write(FILE *fid); + static STATUS read(FILE *fid, Samples *samples); +}; - STATUS bin_write(FILE *fid); - static STATUS bin_read(FILE *fid, Samples *samples); -} + +STATUS load_samples(const char *filename, Samples *samples); struct Hdr { - const std::string version; + const std::string program_version; const std::string data_type; const Coordinates *coords; const Samples *samples; STATUS bin_write(FILE *fid); static STATUS bin_read(FILE *fid, Hdr *hdr); - -} +}; class Grm { @@ -131,7 +130,18 @@ class Grm { size_t size() const; const Dims& dims() const; - STATUS write(const char *filename, const char Hdr *hdr) const; + // @title: Write meta-data and computed grm elements to file + // @description: The binary file written contains a header and payload: + // * Header + // - an instance of grm::Header + // * Payload + // - grm data in row major order + // @param filename: name of file that the data are written + // @param hdr: an instance of grm::Hdr with important meta data + // @return grm::STATUS: + // + STATUS write(const char *filename, const Hdr *hdr) const; + static STATUS read(const char *filename, Grm *grm); private: diff --git a/include/textio.h b/include/textio.h index 5e91ed3..6c8749e 100644 --- a/include/textio.h +++ b/include/textio.h @@ -1,40 +1,83 @@ #include #include +#include #ifndef HEADER_TEXTIO_H #define HEADER_TEXTIO_H -namespace details { -const size_t DEFAULT_BUF_SIZE = 100; -} + +namespace textio { + +enum STATUS { + SUCCESS, + FERROR, + FEOF, + INVALID_ARG_ERROR, + FSEEK_ERROR, + FEOF_ERROR, + END_OF_BUF_ERROR +}; // @title: Parsing text files // @description: This class manages the lifetime of a C-style file stream // by RAII, line retrieval, and getting line unumber. -class TextIO { -public: - - TextIO(const char *filename, const char *mode); - TextIO(const char *filename, const char *mode, const size_t buf_size); +struct TextIO { + TextIO(FILE *fid); ~TextIO(); - int num_lines(); - int get_line(); + int bseek(); -private: - std::string fname_; - const size_t buf_size_; - char *buf_; - size_t buf_line_len_ = 0; + FILE *fid; +}; - FILE *fid_ = nullptr; -}; +std::unique_ptr open(const char *filename, const char *mode); + + +struct FileStats { + size_t nchar = 0; + size_t nwords = 0; + size_t nlines = 0; + size_t nblanklines = 0; +} + +STATUS wc(TextIO *tio, FileStats *fs); -TextIO text_open(const char *filename, const char *mode); +template +struct Array { + Array(size_t size_in): size(size_in), + data(size > 0 ? new T[size] : nullptr) {}; + + ~Array() { if (data) delete[] data; }; + + size_t size; + T *data; + size_t len = 0; + + //unsafe referencing + T operator[](size_t i) { return data[i]; }; + T& operator[](size_t i) { return data[i]; }; + + STATUS append(T val) { + if (len >= size-1) + return END_OF_BUF_ERROR; + + data[len++] = val; + return SUCCESS; + } + + void fill(T val) { + std::memset(data, val, size); + len = 0; + } +} + + +STATUS getline(TextIO *tio, Array linebuf); +} #endif diff --git a/src/bcfio.cpp b/src/bcfio.cpp index a3494ff..76620ff 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -139,7 +139,6 @@ bcfio::ReadBcf::~ReadBcf() { // TODO: subset samples by those in sample_fname file int bcfio::ReadBcf::set_samples(const char *sample_fname) { - int status { 0 }; // Subset samples with those found in the file sample_fname if (!sample_fname || *sample_fname == '\0') { fprintf(stdout, "No file with sample names detected, retreiving" diff --git a/src/grm.cpp b/src/grm.cpp index feea726..cce45a5 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -41,23 +41,23 @@ int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { } -int grm::details::get_size_t(FILE *fid, size_t *val) { - - std::string s { "" }; - while (std::getline(fid, s)) - if (s.size() < ) - - int c; - for (int i = 0; (c = fgetc(fid)) != EOF && i < max_bitsize_size_t; i++) { - if (c == '\n') - break; - s[i] = c; - } - s[i] = '\0'; - - return 0; -} - +// int grm::details::get_size_t(FILE *fid, size_t *val) { +// +// std::string s { "" }; +// while (std::getline(fid, s)) +// if (s.size() < ) +// +// int c; +// for (int i = 0; (c = fgetc(fid)) != EOF && i < max_bitsize_size_t; i++) { +// if (c == '\n') +// break; +// s[i] = c; +// } +// s[i] = '\0'; +// +// return 0; +// } +// grm::Coordinates::Coordinates(const char *contig_name): contig(contig_name) {}; diff --git a/src/textio.cpp b/src/textio.cpp index 88478aa..8ca674d 100644 --- a/src/textio.cpp +++ b/src/textio.cpp @@ -1,67 +1,126 @@ #include -TextIO::TextIO(const char *filename): - fname_(filename), - buf_size_(details::DEFAULT_BUF_SIZE), - buf_(new char[buf_size_]) { - - std::memset(buf_, '\0', buf_size_); -}; -TextIO::TextIO(const char *filename, const size_t buf_size): - fname_(filename), - buf_size_(buf_size), - buf_(new char[buf_size_]) { +textio::TextIO::TextIO(FILE *fileid): fid(fileid) {}; - std::memset(buf_, '\0', buf_size_); -}; +textio::TextIO::~TextIO() { + if (fid) { + fclose(fid); + fid = nullptr; + } +} +textio::TextIO::bseek() { return fseek(fid, 0, SEEK_SET); }; -int TextIO::num_lines() { - size_t line_num = 0; +std::unique_ptr textio::open(const char *filename, + const char *mode) { + + FILE *fid = fopen(filename, mode); + if (ferror(fid)) { + fclose(fid); + return nullptr; + } + + std::unique_ptr tio = std::make_unique(fid); + + return std::move(tio); +} + + +textio::STATUS wc(textio::TextIO *tio, textio::FileStats *fs) { + if (!fs) + return textio::INVALID_ARG_ERROR; + + size_t nchar = 0; + size_t nwords = 0; + size_t nlines = 0; + size_t nblanklines = 0; + size_t word_len = 0; - int c; - while (get_line(fid)) { - - if (c == '\n' && word_len != 0) { - line_num++; - word_len = 0; - } else if (c != '\n') - word_len++; + + FILE *fid = tio->fid; + + int c = '\0'; + while ((c = fgetc(fid)) != EOF) { + + switch (c) { + case '\n': + nlines++; + + if (word_len == 0) + nblanklines++; + else { + nwords++; + word_len = 0; + } + break; + case ';': + case ':': + case ',': + case '!': + case '?': + case '(': + case ')': + case '\"': + case '\t': + case ' ': + if (word_len == 0) + break; + + nwords++; + word_len = 0; + + break; + default: + nchar++; + word_len++; + } + } - if (ferror(fid)) - return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; + if (ferror(fid)) { + fs = nullptr; + return tio->bseek() == 0 ? textio::FERROR : textio::FSEEK_ERROR; + } - if (feof(fid) == 0) - return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; + if (feof(fid) == 0) { + fs = nullptr; + return tio->bseek() == 0 ? textio::FEOF_ERROR : textio::FSEEK_ERROR; + } - *num_lines = line_num; - return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; + fs->nchar = nchar; + fs->nwords = nwords; + fs->nlines = nlines; + fs->nblanklines = nblanklines; + + return tio->bseek() ? textio::SUCCESS : textio::FSEEK_ERROR; } -int get_line(FILE *fid) { - int c; + +textio::STATUS textio::getline(textio::TextIO *tio, textio::Array *buf) { + buf->fill('\0'); + + FILE *fid = buf->fid; + + int c = 0; while ((c = fgetc(fid)) != EOF) { - if (c == '\n' && word_len != 0) { - line_num++; - word_len = 0; - } else if (c != '\n') - word_len++; + if (c == '\n') { + buf->append('\0'); + return textio::SUCCESS; + + buf->append(c) } if (ferror(fid)) - return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; + return textio::FERROR; if (feof(fid) == 0) - return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; - - *num_lines = line_num; - return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; + return textio::FEOF_ERROR; + return textio::SUCCESS; } From adbb2ffd5867ed1f088f46226592b0f3bf35a762 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:08:51 +0000 Subject: [PATCH 25/58] intermediate commit --- include/grm.h | 32 +++++++++++------------------ include/textio.h | 29 +++++++++++++++++++++++---- src/grm.cpp | 52 ++---------------------------------------------- src/textio.cpp | 7 ------- 4 files changed, 39 insertions(+), 81 deletions(-) diff --git a/include/grm.h b/include/grm.h index c0ee860..f5fffdf 100644 --- a/include/grm.h +++ b/include/grm.h @@ -63,40 +63,32 @@ struct Dims { const size_t mcol; }; +struct GrmInfo { + virtual void len() = 0; + virtual void items() = 0; +} // @title: Store genomic coordinates and mange binary I/O // @description: -struct Coordinates { - - // @param pos_filename: The name, and path if necessary, of the text - // file specifying variant positions on the specified contig - // to be included for the grm. - // @param contig_name: The name of the contig, e.g. chrm1 - // - Coordinates(const char *contig_name, const size_t len); - Coordinates(const std::string& contig_name, const size_t len); - - size_t operator[](size_t idx) const; - size_t& operator[](size_t idx); - - STATUS write(FILE *fid); - static STATUS read(FILE *fid, Coordinates *coords); - +struct Coordinates: public GrmInfo { const size_t len; const std::string contig; std::unique_ptr *pos; }; -struct Samples { +struct Samples: public GrmInfo { const size_t len; std::unique_ptr names; - - STATUS write(FILE *fid); - static STATUS read(FILE *fid, Samples *samples); }; +STATUS write(FILE *fid, const GrmInfo *ginfo); + +static STATUS read(FILE *fid, GrmInfo *ginfo); + + + STATUS load_samples(const char *filename, Samples *samples); diff --git a/include/textio.h b/include/textio.h index 6c8749e..4659e71 100644 --- a/include/textio.h +++ b/include/textio.h @@ -21,22 +21,34 @@ enum STATUS { }; -// @title: Parsing text files +// @title: file object // @description: This class manages the lifetime of a C-style file stream -// by RAII, line retrieval, and getting line unumber. +// by RAII. To contruct an instance of the class use the "open" function +// below. +// @param fid: an opened C-style file stream struct TextIO { TextIO(FILE *fid); - ~TextIO(); + ~TextIO() { if (fid) { fclose(fid); fid = nullptr; } }; - int bseek(); + // Move the current file stream to the beginning of the file. + int bseek() { return fseek(fid, 0, SEEK_SET); }; FILE *fid; }; +// @title: open a file and instantiate a TextIO object +// @description: +// @param filename: name and path of file to open +// @param mode: a mode in the set of those in the C library function fopen +// @return a unique_ptr if the file stream was successfully opened +// and TextIO instance created. Otherwise, return a nullptr. std::unique_ptr open(const char *filename, const char *mode); +// @title: File statistics +// @description: This object is returned by any function meant to calculate +// file character statistics. struct FileStats { size_t nchar = 0; size_t nwords = 0; @@ -44,6 +56,15 @@ struct FileStats { size_t nblanklines = 0; } + +// @title: word count +// @description: Similar to the UNIX/Linux wc command line program, wc +// calculates the number of characters, words, lines, etc. that +// the specified file contains. +// @param tio: an instance of TextIO +// @param fs: the structure that the file statistics will be stored +// @return a STATUS code that specifies whether the function was successful +// or failed. STATUS wc(TextIO *tio, FileStats *fs); diff --git a/src/grm.cpp b/src/grm.cpp index cce45a5..5f0e379 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -59,58 +59,10 @@ int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { // } // -grm::Coordinates::Coordinates(const char *contig_name): - contig(contig_name) {}; +grm::Coordinates::Coordinates(const char *contig, const size_t len): + contig(contig), len(len) {} -grm::Coordinates::Coordinates(const std::string& contig_name): - contig(contig_name) {}; - - -grm::STATUS grm::Coordinates::parse_input_file(const char *pos_filename) { - - fid = fopen(pos_filename, "r"); - if (ferror(fid)) - return grm::STATUS::ERROR_FOPEN; - - - grm::STATUS status { grm::STATUS::UNKNOWN_FAILURE }; - - // Use switch the interpret, and act accordingly, to the returned interger - // status code - switch (grm::details::num_lines_in_file(fid, &len)) { - case -1: - return grm::STATUS::ERROR_FOPEN; - case -2: - return grm::STATUS::ERROR_EOF_NOT_REACHED; - case 0: - status = grm::STATUS::SUCCESS; - break; - default: - return status; - } - - pos = make_unique(len); - - size_t num_bits_size_t = static_cast(CHAR_BIT * sizeof(size_t)); - std::string s {}; - - for (size_t i = 0; i < len; i++) { - - - if (s.size() >= num_bits_size_t) - return STATUS::FAILED; - - pos[i] = static_cast(std::strtoull(s)); - } - - fid -} - -grm::STATUS grm::Coordinates::parse_input_file(const std::string& pos_filename) { - return grm::Coordinates::parse_input_file(pos_filename.c_str()); -} - // default constructor grm::Grm::Grm(const size_t nrow, const size_t mcol) diff --git a/src/textio.cpp b/src/textio.cpp index 8ca674d..60c514a 100644 --- a/src/textio.cpp +++ b/src/textio.cpp @@ -5,14 +5,7 @@ textio::TextIO::TextIO(FILE *fileid): fid(fileid) {}; -textio::TextIO::~TextIO() { - if (fid) { - fclose(fid); - fid = nullptr; - } -} -textio::TextIO::bseek() { return fseek(fid, 0, SEEK_SET); }; std::unique_ptr textio::open(const char *filename, From 0e1c9a97c8345b7cf97e66d40b6e8e2b03c4862d Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 23 Jan 2026 22:10:31 +0000 Subject: [PATCH 26/58] incomplete update and reorganization of grm and i/o. does not compile, intermediate update --- .gitignore | 1 + include/grm.h | 136 +++++++++++++++++++++---------------- include/{textio.h => io.h} | 33 +++++---- src/grm.cpp | 58 ++++++++-------- src/{textio.cpp => io.cpp} | 0 src/main.cpp | 18 ++--- 6 files changed, 132 insertions(+), 114 deletions(-) rename include/{textio.h => io.h} (85%) rename src/{textio.cpp => io.cpp} (100%) diff --git a/.gitignore b/.gitignore index 03e5e7c..cd8cd0a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ build/ *.vscode/ scratch/ data/ +tags diff --git a/include/grm.h b/include/grm.h index f5fffdf..3452799 100644 --- a/include/grm.h +++ b/include/grm.h @@ -1,16 +1,33 @@ +// Palmer Lab at UCSD // -// By: Robert Vogel -// Affiliation: Palmer Lab at UCSD -// Date: 2025-01-09 +// This library provides the data structure of a genetic relationship matrix +// and functions for file I/O. +// +// GRM BINARY FILE SPECIFICATION // +// A computed GRM is stored in a custom binary format. The extension ".grm" +// of these files is mandatory. The file is divided into two components, a +// header with meta-data necessary to reproduce the grm calculation and the +// the computed grm values, named the payload. // -// Acknowledgment +// The .grm file header is defined by the struct Hdr, and contains, at a +// minimum, the following information: +// * program_version: grm program version number +// * data_type: alt_count, expected_alt_count, expected_haplotype_count, +// both expected_alt_count and expected_haplotype_count. +// * coords: Genomic coordinates used in the grm calculation. +// * samples: list of sample id's in order of the grm +// the coords and samples are defined by their own structs with field pointers +// to heap allocated memory addresses. Reading and writing such heap allocated +// structs make use of runtime polymorphism of function "read" and "write" +// +// +// ACKNOWLEDGMENT // // Code design and original version completed by Robert Vogel, // reviewed by Claude Sonnet, the AI assistant from Anthropic // (Jan 2025), with minor recommendations incorporated. -// -// +// #ifndef HEADER_GRM_H #define HEADER_GRM_H @@ -23,28 +40,12 @@ #include #include +#include "io.h" -namespace grm { - -namespace details { - -// @title: Count the number of non empty lines in text file -// -// @param fid: pointer to C file stream, i.e. that returned by fopen -// @param num_lines: the number of lines written at this address -// @return -1: file I/O error as determined by ferror(fid), or -// -2: end of file not reached, reason undetermined, or -// -3: error in returning file handle to beginning of file -// 0: success -int num_lines_in_file(FILE *fid, size_t *num_lines); - - -// int chars_to_size_t(FILE *fid, size_t *val); - -} +namespace grm { -enum class STATUS { +enum STATUS { SUCCESS, FAILED, UNKNOWN_FAILURE, @@ -55,6 +56,25 @@ enum class STATUS { }; +namespace details { + + // @title: Count the number of non empty lines in text file + // + // @param fid: pointer to C file stream, i.e. that returned by fopen + // @param num_lines: the number of lines written at this address + // @return -1: file I/O error as determined by ferror(fid), or + // -2: end of file not reached, reason undetermined, or + // -3: error in returning file handle to beginning of file + // 0: success + int num_lines_in_file(FILE *fid, size_t *num_lines); + + + // int chars_to_size_t(FILE *fid, size_t *val); + // + +} + + struct Dims { Dims(size_t nrow_in, size_t mcol_in): nrow(nrow_in), mcol(mcol_in) {}; @@ -63,33 +83,32 @@ struct Dims { const size_t mcol; }; -struct GrmInfo { - virtual void len() = 0; - virtual void items() = 0; -} -// @title: Store genomic coordinates and mange binary I/O +// @title: Store genomic coordinates used in GRM calculation // @description: -struct Coordinates: public GrmInfo { - const size_t len; +struct Coordinates { const std::string contig; - std::unique_ptr *pos; -}; - - -struct Samples: public GrmInfo { const size_t len; - std::unique_ptr names; + std::unique_ptr *pos; }; -STATUS write(FILE *fid, const GrmInfo *ginfo); - -static STATUS read(FILE *fid, GrmInfo *ginfo); +STATUS write(io::FileIO *fio, Coordinates *coords); +STATUS read(io::FileIO *fio, Coordinates *coords); +// Storage in binary format. Sample names are comma separated +// [size_t len][names[0],names[1],names[2]...names[len-1]\0] +struct Samples { + Samples(const size_t len): + len(len), + names(len == 0 ? nullptr : std::make_unique(len)){}; + const size_t len; //number of samples + std::unique_ptr names; +}; -STATUS load_samples(const char *filename, Samples *samples); +STATUS write(io::FileIO *fio, Samples *samples); +STATUS read(io::FileIO *fio, Samples *samples); struct Hdr { @@ -97,12 +116,12 @@ struct Hdr { const std::string data_type; const Coordinates *coords; const Samples *samples; - - STATUS bin_write(FILE *fid); - static STATUS bin_read(FILE *fid, Hdr *hdr); }; +STATUS write(FILE *fid, Hdr *hdr); +STATUS read(FILE *fid, Hdr *hdr); + class Grm { public: Grm(const size_t, const size_t); @@ -122,26 +141,25 @@ class Grm { size_t size() const; const Dims& dims() const; - // @title: Write meta-data and computed grm elements to file - // @description: The binary file written contains a header and payload: - // * Header - // - an instance of grm::Header - // * Payload - // - grm data in row major order - // @param filename: name of file that the data are written - // @param hdr: an instance of grm::Hdr with important meta data - // @return grm::STATUS: - // - STATUS write(const char *filename, const Hdr *hdr) const; - - static STATUS read(const char *filename, Grm *grm); - private: const Dims dims_; std::unique_ptr data_; size_t midx_to_arr_(const size_t&, const size_t&) const; }; +// @title: Write meta-data and computed grm elements to file +// @description: The binary file written contains a header and payload: +// * Header +// - an instance of grm::Header +// * Payload +// - grm data in row major order +// @param filename: name of file that the data are written +// @param hdr: an instance of grm::Hdr with important meta data +// @return grm::STATUS: +// +STATUS write(io.FileIO *fio, Grm *grm, Hdr *hdr) const; +STATUS read(io.FileIO *fio, Grm *grm, Hdr *hdr); + } #endif diff --git a/include/textio.h b/include/io.h similarity index 85% rename from include/textio.h rename to include/io.h index 4659e71..6e4466e 100644 --- a/include/textio.h +++ b/include/io.h @@ -8,7 +8,7 @@ #define HEADER_TEXTIO_H -namespace textio { +namespace io { enum STATUS { SUCCESS, @@ -21,21 +21,20 @@ enum STATUS { }; +struct FileIO { + FileIO(FILE *fid): fid(fid) {}; + ~FileIO() { if (fid) { fclose(fid); fid = nullptr; } }; + + FILE *fid; +}; + + // @title: file object // @description: This class manages the lifetime of a C-style file stream // by RAII. To contruct an instance of the class use the "open" function // below. // @param fid: an opened C-style file stream -struct TextIO { - TextIO(FILE *fid); - ~TextIO() { if (fid) { fclose(fid); fid = nullptr; } }; - - // Move the current file stream to the beginning of the file. - int bseek() { return fseek(fid, 0, SEEK_SET); }; - - FILE *fid; -}; - +int bseek(FileIO *fio); // @title: open a file and instantiate a TextIO object // @description: @@ -43,7 +42,17 @@ struct TextIO { // @param mode: a mode in the set of those in the C library function fopen // @return a unique_ptr if the file stream was successfully opened // and TextIO instance created. Otherwise, return a nullptr. -std::unique_ptr open(const char *filename, const char *mode); +FileIO *open(const char *filename, const char *mode) { + if (!mode or !filename) + return nullptr; + + fid = fopen(filename, mode); + if (ferror(fid)) + return nullptr; + + if (*mode == 'b') + return +} // @title: File statistics diff --git a/src/grm.cpp b/src/grm.cpp index 5f0e379..c43e5bb 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -1,11 +1,7 @@ -// MAtrix +// Palmer Lab at UCSD // -// By: Robert Vogel -// Affiliation: Palmer Lab at UCSD -// Date: 2025-01-10 // -// -// Acknowledgment +// ACKNOWLEDGMENT // // Code design and original version completed by Robert Vogel, // reviewed by Claude Sonnet, the AI assistant from Anthropic @@ -15,30 +11,30 @@ #include -int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { - // TODO: errno, need to reset? - - size_t line_num = 0; - size_t word_len = 0; - int c; - while ((c = fgetc(fid)) != EOF) { - - if (c == '\n' && word_len != 0) { - line_num++; - word_len = 0; - } else if (c != '\n') - word_len++; - } - - if (ferror(fid)) - return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; - - if (feof(fid) == 0) - return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; - - *num_lines = line_num; - return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; -} +// int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { +// // TODO: errno, need to reset? +// +// size_t line_num = 0; +// size_t word_len = 0; +// int c; +// while ((c = fgetc(fid)) != EOF) { +// +// if (c == '\n' && word_len != 0) { +// line_num++; +// word_len = 0; +// } else if (c != '\n') +// word_len++; +// } +// +// if (ferror(fid)) +// return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; +// +// if (feof(fid) == 0) +// return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; +// +// *num_lines = line_num; +// return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; +// } // int grm::details::get_size_t(FILE *fid, size_t *val) { @@ -60,7 +56,7 @@ int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { // grm::Coordinates::Coordinates(const char *contig, const size_t len): - contig(contig), len(len) {} + contig(contig), len(len), pos(std::make_unique(len)) {} diff --git a/src/textio.cpp b/src/io.cpp similarity index 100% rename from src/textio.cpp rename to src/io.cpp diff --git a/src/main.cpp b/src/main.cpp index 9bcc23c..022b9b6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,17 +1,11 @@ // Compute the genomic relationship matrix using haplotypes // -// By: Robert Vogel -// Affiliation: Palmer Lab at UCSD -// Date: 2025-01-09 +// Palmer Lab at UCSD // -// Input argument -// filename: vcf with haplotpye -// -// This program performs a single-pass computation of the -// haplotype based genomic relationship matrix. The approach -// is well defined for the covariance, however under my definition -// of the haplotype based covariance I had to derive the recursion -// relations myself. +// This program performs a single-pass computation of the genetic relationship +// matrix (GRM, GR matrix). GR matrices may be constructed using alt allele +// counts, expected alt allele counts, expected haplotype counts, or a +// combination of both expected alt allele and haplotype counts. // #include #include @@ -186,7 +180,7 @@ int main(int argc, char* argv[]) log.info("Output matrix file: %s", out_fname.c_str()); - Grm cov { bfid.n_samples(), bfid.n_samples() }; + grm.Grm cov { bfid.n_samples(), bfid.n_samples() }; if (use_gt) { log.info("Relationship matrix: genotype"); From 821a58a6f9cacd94ab497621e2b6aad232cd2fb7 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Mon, 26 Jan 2026 17:10:46 +0000 Subject: [PATCH 27/58] updating writing for better use of claude code, the agentic AI coding assistant. --- CLAUDE.md | 40 ++++++++++++++++++++++++++++++++++++++++ README.md | 40 +++++++++++++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..84133aa --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,40 @@ +# Purpose +- Compute the genetic similarity matrix from genetic data stored in the +vcf, vcf.gz, and bcf files. +- Use htslib C library to query / read genentic data. +- Write data to a binary file with a header and payload. + * header data must include all information that is + required to reproduce the calculation. + * payload consists of the computed values +- Easy to use command line interface + + +# External libraries +- htslib, code available on github at: https://github.com/samtools/htslib +- argparse, code available on github at: https://github.com/robert-vogel/argparse + + +# Code style +- variable, function, class, etc. names: + * should be descriptive (self-documenting) and short. + * classes and structs use Upper camel case + * functions and variables use snake case + - global constants use all upper case +- keep orthogonal services of the code in distinct modules +- module header files + * written to the 'include' directory. + * API documentation should be on line(s) proceeding the entities they + describe. +- implementation files in the 'src' directory, make sure to describe the +purpose / design of complex code blocks in the comments +- Unit tests should be written in the 'tests' directory and use Google +test and mock frameworks + + +# File and directory structure +- 'scratch' directory is not tracked by version control and is for testing +ideas by implementing in small programs +- 'build' directory is the target path for program and unit test builds +and should not be tracked by version control. +- 'include' directory for header files +- 'src' directory for 'main.cpp' file and module implementation files diff --git a/README.md b/README.md index afa72c4..462e41a 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,38 @@ # 🏗️ Being built 🏗️ -# Compute the genetic relationship matrix using expected haplotype counts - - -The genetic relationship matrix (GRM) describes the genetic relationship between -pairs of samples. WRITE MORE - - +# Compute the genetic relationship matrix + + +The genetic relationship matrix (GRM) describes the genetic relationship +between pairs of samples. Its computation is dependent on the random effects +defined by the linear mixed model mapping genetic features to phenotype. For +example, suppose that we are interested in account for polygenic SNP effects +using measured genotypes. The LMM is + +$$ +Y = x_j\beta + \mathbf{Z}_j U_j + \epsilon +$$ + +With $Y$ being a random $N$ sample column vector, $x_j$ is the $N$ sample +genotype vector at locus $j$, $\mathbf{Z}_j$ is an $N\times M$ marker matrix +of genotypes that do not include locus $j$, +$U_j\sim\mathcal{N}\left(0,\sigma^2_g \mathbf{I}_{M\times M}\right)$ +independent genetic random effects, and +$\epsilon\sim\mathcal{N}\left(0,\sigma^2_e\mathbf{I}_{N\times N}\right)$ +independent environmental random effects. From which it follows that + +$$ +\text{cov}(Y) = \sigma^2_g\mathbf{Z}\mathbf{Z}^T ++ \sigma^2_e\mathbf{I}_{N\times N} +$$ + +where the genetic component of the phenotype covariance tells us the how +to compute the GRM, i.e. $\mathbf{Z}\mathbf{Z}^T$. + +This program provides the tools to compute the GRM genotypes (alt allele +count 0,1,2), the expected alt allele count under the imputation model, +the expected haplotype count, and a combination of the expected alt allele +count with the expected haplotype count. ## Compute the genetic relationship matrix From 60942e43c9e28b3ebfb99281dfed5d1d17592d29 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 27 Jan 2026 00:00:56 +0000 Subject: [PATCH 28/58] small incomplete update to equations --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 462e41a..91204ba 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,20 @@ The genetic relationship matrix (GRM) describes the genetic relationship between pairs of samples. Its computation is dependent on the random effects defined by the linear mixed model mapping genetic features to phenotype. For -example, suppose that we are interested in account for polygenic SNP effects -using measured genotypes. The LMM is +example, suppose that we are interested in accounting for polygenic SNP effects +using measured genotypes. Let the number of samples be $N$, the number of +loci genotyped $M+1$, $Y\in\mathbb{R}^{N\times 1}$ $$ -Y = x_j\beta + \mathbf{Z}_j U_j + \epsilon +\begin{align} +Y &= x_j\beta + \mathbf{Z}_j U_j + \epsilon\\ +U_j &\sim \mathcal{N}\left(0, \sigma_g^2 \mathbf{I}_{M\times M}\right)\\ +\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2 \mathbf{I}_{N\times N}\right) +\end{align} $$ -With $Y$ being a random $N$ sample column vector, $x_j$ is the $N$ sample -genotype vector at locus $j$, $\mathbf{Z}_j$ is an $N\times M$ marker matrix -of genotypes that do not include locus $j$, +genotype vector at locus $j$, $\mathbf{Z}_j$ is an $N\times M$ genotype matrix +consisting of a set loci that do not include locus $j$, $U_j\sim\mathcal{N}\left(0,\sigma^2_g \mathbf{I}_{M\times M}\right)$ independent genetic random effects, and $\epsilon\sim\mathcal{N}\left(0,\sigma^2_e\mathbf{I}_{N\times N}\right)$ From eb32f5aefb46ff0e41a2fa1d7f4b5d52a575edc0 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 27 Jan 2026 00:11:24 +0000 Subject: [PATCH 29/58] Intermediate progress on equations, commiting to test equation typesetting. --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91204ba..005b0b3 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,26 @@ between pairs of samples. Its computation is dependent on the random effects defined by the linear mixed model mapping genetic features to phenotype. For example, suppose that we are interested in accounting for polygenic SNP effects using measured genotypes. Let the number of samples be $N$, the number of -loci genotyped $M+1$, $Y\in\mathbb{R}^{N\times 1}$ +loci with polygenic effects $M$, and the quantiative phenotypes +$Y\in\mathbb{R}^{N\times 1}$ under the LMM be + +$$ +Y = x_j\beta_j + \mathbf{Z}_j U_j + \epsilon.\\ +$$ + +Here, the fixed effects at locus $j$ are modeled using the alt allele count +$x_j\in \{0,1,2\}^{N\times 1}$ and fixed effect size $\beta_j$. The +random polygenic genetic and environmental effects have properties $$ \begin{align} -Y &= x_j\beta + \mathbf{Z}_j U_j + \epsilon\\ U_j &\sim \mathcal{N}\left(0, \sigma_g^2 \mathbf{I}_{M\times M}\right)\\ \epsilon &\sim \mathcal{N}\left(0, \sigma_e^2 \mathbf{I}_{N\times N}\right) \end{align} $$ +and $\mathbf{Z}_j\in\{0,1,2\}^{N\times M}$ is the genotypes of the. + genotype vector at locus $j$, $\mathbf{Z}_j$ is an $N\times M$ genotype matrix consisting of a set loci that do not include locus $j$, $U_j\sim\mathcal{N}\left(0,\sigma^2_g \mathbf{I}_{M\times M}\right)$ From 3a98669785d98217d279bb33ad5736f59f743988 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 29 Jan 2026 17:39:01 +0000 Subject: [PATCH 30/58] Update: * Writing, which is still in progress, a more detailed README. The details I am currently working on is defining the distinct type of GRMs, table of contents, referencing sections in the document. * Adding C libraries to the command line program entry point. --- README.md | 132 ++++++++++++++++++++++++++++++++++++++------------- src/main.cpp | 2 - 2 files changed, 99 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 005b0b3..2c2873a 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,124 @@ -# 🏗️ Being built 🏗️ +# `grm` a tool for computing genetic relationship matrices -# Compute the genetic relationship matrix + 🏗️ **Under construction** 🏗️ +## Table of Contents + +1. [About](#about) +1. [Subprograms: Compute GRM and leave-one-chromosome-out](#subprog) +2. [Command line user interface](#cli) +3. [Installation and requirements](#install) +4. [References](#refs) + +# About + The genetic relationship matrix (GRM) describes the genetic relationship between pairs of samples. Its computation is dependent on the random effects defined by the linear mixed model mapping genetic features to phenotype. For example, suppose that we are interested in accounting for polygenic SNP effects using measured genotypes. Let the number of samples be $N$, the number of -loci with polygenic effects $M$, and the quantiative phenotypes -$Y\in\mathbb{R}^{N\times 1}$ under the LMM be +loci that contribute to polygenic effects $M$, and the quantiative phenotypes +of $N$ samples $Y\in\mathbb{R}^{N\times 1}$. Under the LMM [[1]](#refs) $$ Y = x_j\beta_j + \mathbf{Z}_j U_j + \epsilon.\\ $$ -Here, the fixed effects at locus $j$ are modeled using the alt allele count -$x_j\in \{0,1,2\}^{N\times 1}$ and fixed effect size $\beta_j$. The -random polygenic genetic and environmental effects have properties +with the fixed effect at locus $j$ being the alternative allele count, denoted +$x_j\in \{0,1,2\}^{N\times 1}$, and fixed effect size $\beta_j$. The +random polygenic and environmental effects have properties $$ \begin{align} -U_j &\sim \mathcal{N}\left(0, \sigma_g^2 \mathbf{I}_{M\times M}\right)\\ -\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2 \mathbf{I}_{N\times N}\right) +U_j &\sim \mathcal{N}\left(0, \sigma_g^2\; \mathbf{I}_{M\times M}\right)\\ +\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2\; \mathbf{I}_{N\times N}\right) \end{align} $$ -and $\mathbf{Z}_j\in\{0,1,2\}^{N\times M}$ is the genotypes of the. +with $\mathbf{Z}_j\in\{0,1,2\}^{N\times M}$ being the matrix of alt allele +counts of the $N$ samples and the set of $M$ markers in which locus $j$ is +not a member. + +Under this model the sample phenotype covariance matrix +decomposes into genetic and environmental terms + +$$ +\text{cov}(Y) = \overbrace{\sigma^2_g\mathbf{Z}\mathbf{Z}^T}^{\text{genetic}} ++ +\underbrace{\sigma^2_e\mathbf{I}_{N\times N}}_{\text{Environment}} +$$ + +where the genetic component of the phenotype covariance tells us how +to compute the GRM, i.e. $\mathbf{Z}\mathbf{Z}^T$. + +In general, the alt allele count polygenic random effects are not the only +genetic effects that we may account for. This program includes GRMs +computed from genotypes, the expected alt allele count under the imputation +model, the expected haplotype count, and a combination of the expected alt +allele count with the expected haplotype count. + + +## Genetic relationship matrices + +The genetic relationship matrices modeling distinct genetic random effects +are derived as outlined above. Here we enumerate the GRM for each type +of random effect consider. + + +### SNP GRM + +The SNP GRM is presented in the [about](#about) section. Let $A_\text{SNP}$ be the +GRM computed by polygenic SNP effects, then + +$$ +G = ZZ^T +$$ + +with $Z$ being the $N\times M$ matrix of alt allele counts. + -genotype vector at locus $j$, $\mathbf{Z}_j$ is an $N\times M$ genotype matrix -consisting of a set loci that do not include locus $j$, -$U_j\sim\mathcal{N}\left(0,\sigma^2_g \mathbf{I}_{M\times M}\right)$ -independent genetic random effects, and -$\epsilon\sim\mathcal{N}\left(0,\sigma^2_e\mathbf{I}_{N\times N}\right)$ -independent environmental random effects. From which it follows that +### Expected alt allele count GRM + +In many cases, as is the case in the Palmer Lab, SNP are imputed. If the +imputation method estimates the genotype probabilities of each locus of each +sample as out method of choice, STITCH [[2]](#refs), then we are able to +compute the expected alt allele count $$ -\text{cov}(Y) = \sigma^2_g\mathbf{Z}\mathbf{Z}^T -+ \sigma^2_e\mathbf{I}_{N\times N} +\begin{align} +\mathbb{E}[X_{ji} | p_{ji}] = \sum_{x_{ji} = 0}^3 x_{ji}\;\mathbb{P}(X_{ji}) +\end{align} $$ -where the genetic component of the phenotype covariance tells us the how -to compute the GRM, i.e. $\mathbf{Z}\mathbf{Z}^T$. +where $p + +### Expected haplotype count GRM + +## Subprograms: Compute GRM and leave-one-chromosome-out + +The `grm` program consists of two subprograms: + +* `grm contig`: the computation of the GRM of a named contig +* `grm loco`: the aggregation of contig GRMs into a + "leave-one-chromosome-out" matrix (denoted the "loco" matrix). + +each of which have there own options enumerated in the next section.4= + + + +## Installation and requirements + +The program is only available as source from this repository and requires + +* `GNU make` +* `htslib` https://github.com/samtools/htslib +* `argparse` https://github.com/robert-vogel/argparse +* `clang` or `gcc` C++17 compiler + + +## Genetic Relationship Matrix types -This program provides the tools to compute the GRM genotypes (alt allele -count 0,1,2), the expected alt allele count under the imputation model, -the expected haplotype count, and a combination of the expected alt allele -count with the expected haplotype count. ## Compute the genetic relationship matrix @@ -72,13 +144,6 @@ grm loco path/to/file/with/grm_filename_and_path_per_line ``` -## Installation and availability - -The program is only available as source from this repository and requires - -* `GNU make` -* `htslib` -* `clang` or `gcc` C++17 compiler @@ -97,11 +162,12 @@ Code design and original version completed by Robert Vogel, reviewed by Claude Sonnet, the AI assistant from Anthropic (Jan 2025), with minor recommendations incorporated. -## References +## References + +[1] [Yang et al. Nature Genetics 42, 565-569 (2010)](https://www.nature.com/articles/ng.608) [1] [Kang et al. Genetics 178: 1709-1723 (2008)](https://academic.oup.com/genetics/article/178/3/1709/6061473) [2] [Kang et al. Nature Genetics 42 348-354 (2010)](https://www.nature.com/articles/ng.548) -[3] [Yang et al. Nature Genetics 42, 565-569 (2010)](https://www.nature.com/articles/ng.608) diff --git a/src/main.cpp b/src/main.cpp index 022b9b6..d588bb7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,8 +8,6 @@ // combination of both expected alt allele and haplotype counts. // #include -#include -#include #include #include From f178f4d9ee004045fb0b2ae42c0db21ba548adb7 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 30 Jan 2026 19:20:30 +0000 Subject: [PATCH 31/58] Updating the README.md. Explaining the distinct GRMs that can be calculated. --- README.md | 131 +++++++++++++++++++++++++++++------------------------- 1 file changed, 70 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 2c2873a..b056a52 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,14 @@ ## Table of Contents 1. [About](#about) -1. [Subprograms: Compute GRM and leave-one-chromosome-out](#subprog) -2. [Command line user interface](#cli) -3. [Installation and requirements](#install) -4. [References](#refs) +2. [Genetic relationship matrices](#grm) +3. [Command line interface](#cli) +4. [Installation and requirements](#install) +4. [Contributing](#contributing) +4. [A.I. Acknowledgement](#ai) +5. [References](#refs) -# About +## About The genetic relationship matrix (GRM) describes the genetic relationship between pairs of samples. Its computation is dependent on the random effects @@ -52,19 +54,30 @@ $$ where the genetic component of the phenotype covariance tells us how to compute the GRM, i.e. $\mathbf{Z}\mathbf{Z}^T$. -In general, the alt allele count polygenic random effects are not the only -genetic effects that we may account for. This program includes GRMs -computed from genotypes, the expected alt allele count under the imputation -model, the expected haplotype count, and a combination of the expected alt -allele count with the expected haplotype count. +The alt allele count polygenic random effects are one example of genetic +effects. We do not need to limit ourselves to this model, and instead +account for any measurable genetic signals. This program uses several +signals to compute a GRM: +* genotypes i.e. the alternative allele count, +* the expected alt allele count under a probabilistic model, useful for +imputed genetic signals, +* ancestral haplotypes, i.e. a K dimensional vector of expected haplotype +counts, +* or some combination of the aforementioned signals. -## Genetic relationship matrices +This program provides an means to compute the GRM of genetic signals in +general. -The genetic relationship matrices modeling distinct genetic random effects -are derived as outlined above. Here we enumerate the GRM for each type -of random effect consider. +## Genetic relationship matrices + +The genetic relationship matrices that this program computes are as follows: +imputed SNP genotypes considered above, the expected alternative allele +counts under a probabilistic model, the expected ancestral haplotype counts, +and the combined expected alternative allele and expected haplotype counts. +The subsections the follow define the model and the calculation for the +aforementioned GRMs. ### SNP GRM @@ -72,30 +85,43 @@ The SNP GRM is presented in the [about](#about) section. Let $A_\text{SNP}$ be GRM computed by polygenic SNP effects, then $$ -G = ZZ^T +A_\text{SNP} = \mathbf{Z}\mathbf{Z}^T $$ -with $Z$ being the $N\times M$ matrix of alt allele counts. +with $\mathbf{Z}$ being the $N\times M$ matrix of alt allele counts. + +### Expected alternative allele count GRM -### Expected alt allele count GRM +In many cases, as is the case in the Palmer Lab, SNPs are ***imputed***. If +the imputation method estimates the genotype probabilities at each locus of +each sample, as does our method of choice STITCH [[2]](#refs), then it may +be more appropriate consider the expected alternative allele counts (EAC) +under the imputation model rather than the imputed genotype calls. -In many cases, as is the case in the Palmer Lab, SNP are imputed. If the -imputation method estimates the genotype probabilities of each locus of each -sample as out method of choice, STITCH [[2]](#refs), then we are able to -compute the expected alt allele count +Let $Z_{ik}\in\{0, 1, 2\}$ be an element of the design matrix of random +polygenic effects for sample $i$ at locus $k\neq j$. We consider $Z_{ik}$ +to be random as the genotypes are imputed under a probistic model. This +makes the expected value $ \mathbb{E}[Z_{ik} |\mathcal{O} ]$ given the +experimentally observed reads the most appropriate genetic signal to +consider. Let's call the matrix of conditionally expected alternative +allele counts $\mathbf{C}$, meaning that the GRM over expected alternative +allele counts $\mathbf{A}_\text{EAC}$ is $$ -\begin{align} -\mathbb{E}[X_{ji} | p_{ji}] = \sum_{x_{ji} = 0}^3 x_{ji}\;\mathbb{P}(X_{ji}) -\end{align} +\mathbf{A}_\text{EAC} =\mathbf{C}\mathbf{C}^T. $$ -where $p - ### Expected haplotype count GRM -## Subprograms: Compute GRM and leave-one-chromosome-out +[] TODO + +### Leave one chromosome out (loco) GRM + +[] TODO + + +## Command line interface The `grm` program consists of two subprograms: @@ -106,23 +132,6 @@ The `grm` program consists of two subprograms: each of which have there own options enumerated in the next section.4= - -## Installation and requirements - -The program is only available as source from this repository and requires - -* `GNU make` -* `htslib` https://github.com/samtools/htslib -* `argparse` https://github.com/robert-vogel/argparse -* `clang` or `gcc` C++17 compiler - - -## Genetic Relationship Matrix types - - - -## Compute the genetic relationship matrix - The GRM calculation requires the SNPs or haplotypes to jbe in the bcf family of file formats, i.e. vcf, vcf.gz, or bcf. By default, the GRM is computed using the expected haplotype counts with FORMAT ID = "HD". @@ -136,38 +145,38 @@ Other options include +## Installation and requirements -## Compute LOCO matrices - -``` -grm loco path/to/file/with/grm_filename_and_path_per_line -``` - - +The program is only available as source from this repository and requires +* `GNU make` +* `htslib` https://github.com/samtools/htslib +* `argparse` https://github.com/robert-vogel/argparse +* `clang` or `gcc` C++17 compiler -## Contributing +## Contributing I am using [GoogleTest](https://google.github.io/googletest/) framework for organizing tests. If you contribute, please make tests for your contributions. To run tests, `build` directory and build the project -``` make check -## Acknowledgement -Code design and original version completed by Robert Vogel, -reviewed by Claude Sonnet, the AI assistant from Anthropic -(Jan 2025), with minor recommendations incorporated. +## A.I. Acknowledgement + +The problem statement and overall design of the code base was by +Robert Vogel. He has made use of Claude for review and Claude Code, +the AI assistant by Anthropic, to implement sum features. + ## References [1] [Yang et al. Nature Genetics 42, 565-569 (2010)](https://www.nature.com/articles/ng.608) - -[1] [Kang et al. Genetics 178: 1709-1723 (2008)](https://academic.oup.com/genetics/article/178/3/1709/6061473) +[2] [Davies et al. Nature Genetics 48, 965-969 (2016)](https://www.nature.com/articles/ng.3594) + From 2a545b00971c2f8592445c9645603e03a76ebe1f Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 30 Jan 2026 20:15:37 +0000 Subject: [PATCH 32/58] Simplifying equations in the README.md to ease understanding. Trying to debug the issue of spacing between variables. In tex, '\;' will add some white space between variables in an equation. I think this is suppose to work in GitHub Markdown but am not sure. So I am trying out different approaches. --- README.md | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index b056a52..0f379dc 100644 --- a/README.md +++ b/README.md @@ -19,12 +19,14 @@ The genetic relationship matrix (GRM) describes the genetic relationship between pairs of samples. Its computation is dependent on the random effects defined by the linear mixed model mapping genetic features to phenotype. For example, suppose that we are interested in accounting for polygenic SNP effects -using measured genotypes. Let the number of samples be $N$, the number of -loci that contribute to polygenic effects $M$, and the quantiative phenotypes -of $N$ samples $Y\in\mathbb{R}^{N\times 1}$. Under the LMM [[1]](#refs) +using measured genotypes. Let the number of samples be $N$, the set of loci of +contributing to polygenic effects be $\Omega$ with $|\Omega|= M$, and the +quantitative phenotypes of $N$ samples $Y\in\mathbb{R}^{N\times 1}$. Note that +locus $j$ is not a member of the set of loci $\Omega$. Under the +LMM [[1]](#refs) $$ -Y = x_j\beta_j + \mathbf{Z}_j U_j + \epsilon.\\ +Y = x_j\beta_j + \mathbf{Z} U + \epsilon.\\ $$ with the fixed effect at locus $j$ being the alternative allele count, denoted @@ -33,22 +35,26 @@ random polygenic and environmental effects have properties $$ \begin{align} -U_j &\sim \mathcal{N}\left(0, \sigma_g^2\; \mathbf{I}_{M\times M}\right)\\ -\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2\; \mathbf{I}_{N\times N}\right) +U &\sim \mathcal{N}\left(0, \sigma_g^2 \; \mathbf{I}_{M\times M}\right)\\ +\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2 \; \mathbf{I}_{N\times N}\right) \end{align} $$ -with $\mathbf{Z}_j\in\{0,1,2\}^{N\times M}$ being the matrix of alt allele -counts of the $N$ samples and the set of $M$ markers in which locus $j$ is +with $\mathbf{Z}\in\{0,1,2\}^{N\times M}$ being the matrix of alt allele +counts of the $N$ samples and and $M$ loci in the set $\Omega$, not a member. Under this model the sample phenotype covariance matrix decomposes into genetic and environmental terms $$ -\text{cov}(Y) = \overbrace{\sigma^2_g\mathbf{Z}\mathbf{Z}^T}^{\text{genetic}} +\text{cov}(Y) = \overbrace{ + \sigma^2_g \; \mathbf{Z}\mathbf{Z}^T +}^{\text{genetic}} + -\underbrace{\sigma^2_e\mathbf{I}_{N\times N}}_{\text{Environment}} +\underbrace{ + \sigma^2_e \; \mathbf{I}_{N\times N} +}_{\text{environment}} $$ where the genetic component of the phenotype covariance tells us how @@ -100,8 +106,8 @@ be more appropriate consider the expected alternative allele counts (EAC) under the imputation model rather than the imputed genotype calls. Let $Z_{ik}\in\{0, 1, 2\}$ be an element of the design matrix of random -polygenic effects for sample $i$ at locus $k\neq j$. We consider $Z_{ik}$ -to be random as the genotypes are imputed under a probistic model. This +polygenic effects for sample $i$ at locus $k\in\Omega$. We consider $Z_{ik}$ +to be random as the genotypes are imputed under a probistic model. This makes the expected value $ \mathbb{E}[Z_{ik} |\mathcal{O} ]$ given the experimentally observed reads the most appropriate genetic signal to consider. Let's call the matrix of conditionally expected alternative From 2317e05ef1662a6e1cb0f1b3ccc44dc3de9d854a Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 30 Jan 2026 20:20:51 +0000 Subject: [PATCH 33/58] To add space in an equation in GitHub markdown I learned that I need to use '\;' and not '\;' as I had been using. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0f379dc..bc61e9a 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ random polygenic and environmental effects have properties $$ \begin{align} -U &\sim \mathcal{N}\left(0, \sigma_g^2 \; \mathbf{I}_{M\times M}\right)\\ -\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2 \; \mathbf{I}_{N\times N}\right) +U &\sim \mathcal{N}\left(0, \sigma_g^2 \\: \mathbf{I}_{M\times M}\right)\\ +\epsilon &\sim \mathcal{N}\left(0, \sigma_e^2 \\: \mathbf{I}_{N\times N}\right) \end{align} $$ @@ -49,11 +49,11 @@ decomposes into genetic and environmental terms $$ \text{cov}(Y) = \overbrace{ - \sigma^2_g \; \mathbf{Z}\mathbf{Z}^T + \sigma^2_g \\; \mathbf{Z}\mathbf{Z}^T }^{\text{genetic}} + \underbrace{ - \sigma^2_e \; \mathbf{I}_{N\times N} + \sigma^2_e \\; \mathbf{I}_{N\times N} }_{\text{environment}} $$ From fe277a8c817e9de62ed669b31c0c1fafa3c905e6 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 30 Jan 2026 21:02:04 +0000 Subject: [PATCH 34/58] added a draft of the haplotype grm description to README --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index bc61e9a..bc09721 100644 --- a/README.md +++ b/README.md @@ -105,13 +105,14 @@ each sample, as does our method of choice STITCH [[2]](#refs), then it may be more appropriate consider the expected alternative allele counts (EAC) under the imputation model rather than the imputed genotype calls. -Let $Z_{ik}\in\{0, 1, 2\}$ be an element of the design matrix of random -polygenic effects for sample $i$ at locus $k\in\Omega$. We consider $Z_{ik}$ -to be random as the genotypes are imputed under a probistic model. This -makes the expected value $ \mathbb{E}[Z_{ik} |\mathcal{O} ]$ given the -experimentally observed reads the most appropriate genetic signal to -consider. Let's call the matrix of conditionally expected alternative -allele counts $\mathbf{C}$, meaning that the GRM over expected alternative +Let $Z_{im}\in\{0, 1, 2\}$ be an element of the design matrix of random +polygenic effects for sample $i$ at locus $m\in\Omega$. We consider $Z_{im}$ +to be random as the genotypes are imputed under a probabilistic model. This +makes the expected value $\mathbb{E}[Z_{im} |\mathcal{O}]$ of alternative +allele counts given the experimentally observed reads the an appropriate +genetic signal to consider. Let's call the matrix of expected alternative +allele counts $\mathbf{C} := \mathbb{E}[\mathbf{Z} |\mathcal{O}]$. Given +this, the GRM over expected alternative allele counts $\mathbf{A}_\text{EAC}$ is $$ @@ -120,7 +121,45 @@ $$ ### Expected haplotype count GRM -[] TODO +The expected haplotype count GRM, $\mathbf{A}_\text{EHC}$, needs +motivation. The reason is that at each locus there is not a single +allele that we are counting, but instead we are counting the +number of copies of each haplotype $k\in\{1, 2, \dots, K\}$ at any +specified locus. Indeed, when $K=2$ we can cast the problem to +SNP case by identifying one of the two haplotypes as an alternative +allele. However when $K>2$ the genetic data is no longer a scalar +but a $K$ dimensional vector $h\in \{0,1,2\}^{K\times 1}$ such that +$\sum_{k=1}^K h_k=2$. Moreover, this implies that at each locus $j$ +there are $K$ effect sizes, that can be expressed as the column +vector $\boldsymbol{\beta}_j\in \mathbb{R}^{K\times 1}$. As we can +see the haplotype model is more complex as the fixed effects are +in a $K$ dimensional space as opposed to a 1 dimensional space. + +An important question when working under the haplotype model is what +type of random genetic effects do we want to account for. If we +only care about the polygenic SNP effects, then we should make use of +GRMs $\mathbf{A}$ or $\mathbf{A}_\text{EHC}$. Another choice would +be to account for poly-haplotype effects, that the LMM mapping genotype +to phenotype at locus $j$ for sample $i$ becomes, + +$$ +Y_j = h_{ij}^T\,\boldsymbol{\beta}_j + \mathbf{W}_1U_1 ++ \mathbf{W}_2 U_2 ++ \dots ++ \mathbf{W}_K U_K ++ \epsilon_j. +$$ + +Here, the difference between the polygenic SNP and haplotype effects +are explicit. Instead of a single design matrix $\mathbf{Z}$ accounting +for polygenic effects there are now $K$ matrices. Consequently, the +haplotype GRM is + +$$ +\mathbf{A}_\text{EHC} = \sum_{k=1}^K \mathbf{W}_k\mathbf{W}^T_k +$$ + +the sum of the similarity matrices of each haplotype. ### Leave one chromosome out (loco) GRM From 7fef10f4e078270a1f43b69c1a80c6a917c7d41c Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 30 Jan 2026 16:18:34 -0500 Subject: [PATCH 35/58] Update README.md debugging haplotype model equations --- README.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index bc09721..2b744ab 100644 --- a/README.md +++ b/README.md @@ -116,20 +116,20 @@ this, the GRM over expected alternative allele counts $\mathbf{A}_\text{EAC}$ is $$ -\mathbf{A}_\text{EAC} =\mathbf{C}\mathbf{C}^T. +\mathbf{A}_\text{EAC} =\mathbf{C}\mathbf{C}^T $$ ### Expected haplotype count GRM -The expected haplotype count GRM, $\mathbf{A}_\text{EHC}$, needs +The expected haplotype count GRM $\mathbf{A}_\text{EHC}$ needs motivation. The reason is that at each locus there is not a single allele that we are counting, but instead we are counting the number of copies of each haplotype $k\in\{1, 2, \dots, K\}$ at any specified locus. Indeed, when $K=2$ we can cast the problem to SNP case by identifying one of the two haplotypes as an alternative allele. However when $K>2$ the genetic data is no longer a scalar -but a $K$ dimensional vector $h\in \{0,1,2\}^{K\times 1}$ such that -$\sum_{k=1}^K h_k=2$. Moreover, this implies that at each locus $j$ +but a $K$ dimensional vector $h\in \\{0,1,2\\}^{K\times 1}$ such that +$\sum _{k=1}^K h_k = 2$. Moreover, this implies that at each locus $j$ there are $K$ effect sizes, that can be expressed as the column vector $\boldsymbol{\beta}_j\in \mathbb{R}^{K\times 1}$. As we can see the haplotype model is more complex as the fixed effects are @@ -143,11 +143,10 @@ be to account for poly-haplotype effects, that the LMM mapping genotype to phenotype at locus $j$ for sample $i$ becomes, $$ -Y_j = h_{ij}^T\,\boldsymbol{\beta}_j + \mathbf{W}_1U_1 -+ \mathbf{W}_2 U_2 -+ \dots -+ \mathbf{W}_K U_K -+ \epsilon_j. +\begin{equation} +Y_i = h_{ij}^T\\,\boldsymbol{\beta}_j + \mathbf{W}_1 U_1 + \mathbf{W}_2 U_2 + +\dots + \mathbf{W}_K U_K + \epsilon_i. +\end{equation} $$ Here, the difference between the polygenic SNP and haplotype effects From b2426159a734035eb7be213b77c9d7df0e14c672 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Mon, 2 Feb 2026 16:38:34 +0000 Subject: [PATCH 36/58] Working on README documentation. Finished a draft of the loco GRM descrtiption. Writing binary grm file specification. --- README.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2b744ab..323e3a1 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ 2. [Genetic relationship matrices](#grm) 3. [Command line interface](#cli) 4. [Installation and requirements](#install) +5. [`.grm` file format](#grmspec) 4. [Contributing](#contributing) 4. [A.I. Acknowledgement](#ai) 5. [References](#refs) @@ -162,7 +163,22 @@ the sum of the similarity matrices of each haplotype. ### Leave one chromosome out (loco) GRM -[] TODO +In a genome wide association study (GWAS) polygenic effects are often +accounted for from all loci except those of the chromosome that fixed +effect sizes are being estimated. This approach is referred to as +"leave-one-chromosome-out", or loco for short, for which we will refer +the resulting GRM as the loco GRM. This matrix can be computed easily +from the set of all chromosome GRMs. + +Let $M_c$ be the number of markers used for computing the GRM $A_c$. +Then it follows that the loco GRM $L_u$ that is used for effect size +estimation of all loci on chromosome $u$ is, + +$$ +\begin{align} +L_u &= \sum_{\forall c \neq u} A_c. +\end{align} +$$ ## Command line interface @@ -173,20 +189,31 @@ The `grm` program consists of two subprograms: * `grm loco`: the aggregation of contig GRMs into a "leave-one-chromosome-out" matrix (denoted the "loco" matrix). -each of which have there own options enumerated in the next section.4= +to read documentation on the respective subprogram simply + +``` +grm [contig | loco] --help +``` + +The GRM calculation requires tha the genetic information is in the +the bcf family of file formats, i.e. vcf, vcf.gz, or bcf. While +the loco subprogram requires all chromosome matrices to be in the +`.grm` binary file format defined below. +### Computing a contig GRM -The GRM calculation requires the SNPs or haplotypes to jbe in the bcf family -of file formats, i.e. vcf, vcf.gz, or bcf. By default, the GRM is computed +By default, the GRM is computed using the expected haplotype counts with FORMAT ID = "HD". ``` grm chrm ``` - will produce a binary `.mat` file that stores the GRM and relavent meta data. Other options include +### Computing a loco GRM + + ## Installation and requirements @@ -199,6 +226,63 @@ The program is only available as source from this repository and requires * `clang` or `gcc` C++17 compiler +## The `.grm` file format + +The `.grm` file format is a binary data format consisting of meta data +and a payload. The meta data includes: + +* program version number +``` +struct version { + uint32_t major: 10; + uint32_t minor: 10; + uint32_t micro: 10; +``` +* date that the program launched +``` +struct date { + uint32_t year : 12; + uint32_t month : 4; + uint32_t day : 5; + uint32_t hour : 5; + uint32_t sec : 6; +} +``` +* user name of person that launched the program +``` +Array +``` +* chromosome, or more generally, the contigs name +``` +Array +``` +* the set of marker positions used for computing the GRM +``` +Array +``` +* the sample id's in order of the column number of the GRM +``` +Array> +``` + +where the `Array` template is defined by: +``` +templat +struct Array { + uint32_t len; + char *data; +`` +that is to say that the array is a simple dynamic data structure +whose data is allocated on the heap. + +The payload is the upper triangular and diagonal components in an +$M \, (M+1) / 2$ element array of 32 bit floating point numbers +While the genotype based GRM will not produce fractional values, +the expected counts will, making `float32` an acceptable choice. +order. + + + ## Contributing I am using [GoogleTest](https://google.github.io/googletest/) framework From ae1ba67b1726eda9c0f7a58f6c943719bad847d2 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Mon, 2 Feb 2026 16:40:50 +0000 Subject: [PATCH 37/58] Debug README for proper rendering on github. I had made a simple mistake of terminating a code block with two back ticks instead of three. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 323e3a1..04c0b2d 100644 --- a/README.md +++ b/README.md @@ -271,7 +271,7 @@ templat struct Array { uint32_t len; char *data; -`` +``` that is to say that the array is a simple dynamic data structure whose data is allocated on the heap. From 41e6dd050105faa365c9e3127d182d3a0252a4f6 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Mon, 2 Feb 2026 11:42:02 -0500 Subject: [PATCH 38/58] Update README.md Forgot terminating brace on definition of array. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 04c0b2d..d1f286f 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,7 @@ templat struct Array { uint32_t len; char *data; +} ``` that is to say that the array is a simple dynamic data structure whose data is allocated on the heap. From 2fef20b50248288901f6ee8c6f92f3e8d5c84339 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Mon, 2 Feb 2026 17:13:30 +0000 Subject: [PATCH 39/58] Start work on Claude's recommendation for readme. Mainly updating the grm file specification for clarity of exposition. --- README.md | 66 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index d1f286f..6bc53f8 100644 --- a/README.md +++ b/README.md @@ -229,16 +229,36 @@ The program is only available as source from this repository and requires ## The `.grm` file format The `.grm` file format is a binary data format consisting of meta data -and a payload. The meta data includes: +and a payload. + +### Defined types + +The `Array` type is a minimal dynamic data storage structure where +the length of the array is known and its address on the heap saved in +a pointer. +``` +template +struct Array { + uint32_t len; + T data[len]; +}; +``` + +The version structure stores is bit packed with fields specifying +the `grm` program version number. -* program version number ``` struct version { uint32_t major: 10; uint32_t minor: 10; uint32_t micro: 10; + uint32_t : 2; +}; ``` -* date that the program launched + +The data structure stores is bit packed with fields specifying when +the `grm` program was launched. + ``` struct date { uint32_t year : 12; @@ -246,35 +266,21 @@ struct date { uint32_t day : 5; uint32_t hour : 5; uint32_t sec : 6; -} -``` -* user name of person that launched the program -``` -Array -``` -* chromosome, or more generally, the contigs name -``` -Array -``` -* the set of marker positions used for computing the GRM -``` -Array -``` -* the sample id's in order of the column number of the GRM -``` -Array> +}; ``` -where the `Array` template is defined by: -``` -templat -struct Array { - uint32_t len; - char *data; -} -``` -that is to say that the array is a simple dynamic data structure -whose data is allocated on the heap. +### Meta data + +| offset | field | type | size (bytes) | description | +| --- | --- | --- | --- | --- | +| 0 | magic | uint32_t | 4 | File signature | +| 4 | version | struct version | 4 | major/minor/micro | +| 8 | date | struct date | 4 | year/month/day/hour/sec | +| 16 | user | Array | || + + + +### Payload The payload is the upper triangular and diagonal components in an $M \, (M+1) / 2$ element array of 32 bit floating point numbers From 9889fa0572aa8a8a0c65725ae5c765cf585c6ce5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Feb 2026 17:17:08 +0000 Subject: [PATCH 40/58] Complete .grm file format specification in README. Fix grammatical errors in type descriptions, complete the meta data table with all fields (magic, version, date, user, contig, markers, samples), correct offset error, and clarify payload section. https://claude.ai/code/session_0116D5VsHW8prM7LEGxa4TVk --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6bc53f8..6c7fd00 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,7 @@ struct Array { }; ``` -The version structure stores is bit packed with fields specifying +The version structure is bit packed with fields specifying the `grm` program version number. ``` @@ -256,7 +256,7 @@ struct version { }; ``` -The data structure stores is bit packed with fields specifying when +The date structure is bit packed with fields specifying when the `grm` program was launched. ``` @@ -272,21 +272,24 @@ struct date { ### Meta data | offset | field | type | size (bytes) | description | -| --- | --- | --- | --- | --- | -| 0 | magic | uint32_t | 4 | File signature | -| 4 | version | struct version | 4 | major/minor/micro | -| 8 | date | struct date | 4 | year/month/day/hour/sec | -| 16 | user | Array | || +| --- | --- | --- | --- | --- | +| 0 | magic | uint32_t | 4 | File signature (0x47524D00 = "GRM\0") | +| 4 | version | struct version | 4 | Program version: major/minor/micro | +| 8 | date | struct date | 4 | Launch time: year/month/day/hour/sec | +| 12 | user | Array\ | 4 + len | Username of person who ran the program | +| varies | contig | Array\ | 4 + len | Chromosome or contig name | +| varies | markers | Array\ | 4 + 4×len | Marker positions used for GRM computation | +| varies | samples | Array\\> | 4 + Σ(4 + len_i) | Sample IDs in column order of the GRM | ### Payload -The payload is the upper triangular and diagonal components in an -$M \, (M+1) / 2$ element array of 32 bit floating point numbers -While the genotype based GRM will not produce fractional values, -the expected counts will, making `float32` an acceptable choice. -order. +The payload is the upper triangular and diagonal components of the +$N \times N$ GRM stored as an array of $N(N+1)/2$ 32-bit floating point +numbers in row-major order. While the genotype-based GRM will not produce +fractional values, the expected count GRMs will, making `float32` an +acceptable choice for all GRM types. From 12924df164a0842ca7703af21dba6e25befb43f6 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Tue, 17 Mar 2026 21:53:08 -0400 Subject: [PATCH 41/58] intermediate update. Trying to write the binary file io for matrix. updated to only store upper triangular matrix of GRM --- include/grm.h | 107 +++++++++++++++++++++++++++++------------------- include/io.h | 54 ++++++++++++++++-------- src/grm.cpp | 111 +++++++++++++++++++++++++++++++++----------------- src/main.cpp | 27 +++++++----- 4 files changed, 193 insertions(+), 106 deletions(-) diff --git a/include/grm.h b/include/grm.h index 3452799..f8f12a3 100644 --- a/include/grm.h +++ b/include/grm.h @@ -43,6 +43,36 @@ #include "io.h" +// The algorithm for getting the array idx from matrix indexes is simply +// +// i * n_samples - n_skipped_idxs + j +// +// where i is the matrix row index and j is the matrix column index. +// interesting term is n_skipped_idxs, this is the number of elements +// that referencing (i, j) skip when only storing upper triangle. For +// example, consider the following table with matrix to array indexes +// +// i j num_skipped idx +// 0 0 0 0 +// 0 5 0 5 +// 1 0 0 1n - 0 +// 2 0 1 2n - 1 +// 3 0 3 3n - 3 +// 4 0 6 4n - 6 +// +// we see that number skipped is the number of lower triangular elements +// of a matrix constructed from i rows, (i-1) * i / 2. Here, we see an +// obvious problem, that when i = 0 we get a negative number, which doesn't +// make sense. This can be avoided by using the equivalent formulat +// +// n_skipped_idxs = i * (i + 1) / 2 - i +// +// making the equation above read +// +// i * (n_samples + 1) - i * (i+1)/2 + j +#define MATRIX_IDX_TO_ARRAY(i, j, n) ((i) * (n + 1) - (i)*(i+1)/2 + j) + + namespace grm { enum STATUS { @@ -53,36 +83,16 @@ enum STATUS { ERROR_FOPEN, ERROR_EOF_NOT_REACHED, ERROR_ON_WRITE, + ERROR_FILE_NOT_OPEN, }; -namespace details { - - // @title: Count the number of non empty lines in text file - // - // @param fid: pointer to C file stream, i.e. that returned by fopen - // @param num_lines: the number of lines written at this address - // @return -1: file I/O error as determined by ferror(fid), or - // -2: end of file not reached, reason undetermined, or - // -3: error in returning file handle to beginning of file - // 0: success - int num_lines_in_file(FILE *fid, size_t *num_lines); - - - // int chars_to_size_t(FILE *fid, size_t *val); - // - -} - - -struct Dims { - Dims(size_t nrow_in, size_t mcol_in): - nrow(nrow_in), mcol(mcol_in) {}; - - const size_t nrow; - const size_t mcol; -}; - +enum GrmType { + EHC, // Expected Haplotype Count + EAC, // Expected Alternative Allele Count + BOTH, // Both EHC AND EAC + DS, // Dosage, i.e. Called Alternative Allele Count +}; // @title: Store genomic coordinates used in GRM calculation // @description: @@ -93,8 +103,8 @@ struct Coordinates { }; -STATUS write(io::FileIO *fio, Coordinates *coords); -STATUS read(io::FileIO *fio, Coordinates *coords); +STATUS write(io::FileIO* fio, Coordinates* coords); +STATUS read(io::FileIO* fio, Coordinates* coords); // Storage in binary format. Sample names are comma separated @@ -111,24 +121,37 @@ STATUS write(io::FileIO *fio, Samples *samples); STATUS read(io::FileIO *fio, Samples *samples); +// Header struct Hdr { - const std::string program_version; - const std::string data_type; + // const std::string program_version; + const size_t n_samples; + const GrmType grm_type; const Coordinates *coords; const Samples *samples; }; -STATUS write(FILE *fid, Hdr *hdr); -STATUS read(FILE *fid, Hdr *hdr); +STATUS write(io::FileIO *fio, const Hdr *hdr); +STATUS read(io::FileIO *fio, Hdr *hdr); + +// Grm class manages storage and access of GRM matrix +// +// The GRM as an n_sample by n_sample symmetric, positive semi-definite +// matrix. Let Z represent the n_sample by m_marker data genetic data. +// From these data the GRM is computed as GRM = ZZ^T. +// +// @param n_samples of the GRM. +// class Grm { public: - Grm(const size_t, const size_t); - Grm(const Grm&); // copy constructor - Grm(Grm&&); // move constructor - Grm& operator=(const Grm&)=delete; // copy assignment - Grm& operator=(Grm&&)=delete; // move assignment + // + Grm(const size_t n_samples); + + Grm(const Grm&)=delete; + Grm(Grm&&)=delete; + Grm& operator=(const Grm&)=delete; + Grm& operator=(Grm&&)=delete; // Unchecked indexes when setting and getting of matrix values float operator()(const size_t i, const size_t j) const; @@ -139,10 +162,9 @@ class Grm { STATUS get(const size_t i, const size_t j, float *val) const; size_t size() const; - const Dims& dims() const; private: - const Dims dims_; + const size_t n_samples_; std::unique_ptr data_; size_t midx_to_arr_(const size_t&, const size_t&) const; }; @@ -157,8 +179,9 @@ class Grm { // @param hdr: an instance of grm::Hdr with important meta data // @return grm::STATUS: // -STATUS write(io.FileIO *fio, Grm *grm, Hdr *hdr) const; -STATUS read(io.FileIO *fio, Grm *grm, Hdr *hdr); +STATUS write(io.FileIO *fio, const Hdr *hdr, const Grm *grmatrix) const; +STATUS read(io.FileIO *fio, const Hdr *hdr, Grm *grmatrix); + } diff --git a/include/io.h b/include/io.h index 6e4466e..702e7ef 100644 --- a/include/io.h +++ b/include/io.h @@ -22,39 +22,59 @@ enum STATUS { struct FileIO { - FileIO(FILE *fid): fid(fid) {}; + FileIO(FILE* fid): fid(fid) {}; ~FileIO() { if (fid) { fclose(fid); fid = nullptr; } }; + FileIO(const FileIO&) = delete; + FileIO& operator=(const FileIO&) = delete; + + FileIO(FileIO&& other) noexcept : fid(other.fid) { + other.fid = nullptr; + } + + FileIO& operator=(FileIO&& other) { + // protect against self assignment + if (this == &other) return *this; + + if (fid) fclose(fid); + + fid = other.fid; + other.fid = nullptr; + + return *this; + } + FILE *fid; }; -// @title: file object -// @description: This class manages the lifetime of a C-style file stream -// by RAII. To contruct an instance of the class use the "open" function -// below. -// @param fid: an opened C-style file stream -int bseek(FileIO *fio); - -// @title: open a file and instantiate a TextIO object +// @title: open a file // @description: // @param filename: name and path of file to open // @param mode: a mode in the set of those in the C library function fopen -// @return a unique_ptr if the file stream was successfully opened -// and TextIO instance created. Otherwise, return a nullptr. -FileIO *open(const char *filename, const char *mode) { - if (!mode or !filename) +// @return a pointer to opened file +FileIO open(const char *filename, const char *mode) { + if (!mode || !filename) return nullptr; - fid = fopen(filename, mode); - if (ferror(fid)) + FILE *fid = fopen(filename, mode); + if (!fid) return nullptr; - if (*mode == 'b') - return + FileIO fio = FileIO(fid); + return std::move(fio); } +// @title: file object +// @description: This class manages the lifetime of a C-style file stream +// by RAII. To contruct an instance of the class use the "open" function +// below. +// @param fid: an opened C-style file stream +// int bseek(FileIO *fio); + + + // @title: File statistics // @description: This object is returned by any function meant to calculate // file character statistics. diff --git a/src/grm.cpp b/src/grm.cpp index c43e5bb..5e130d8 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -55,50 +55,103 @@ // } // +//////////////////////////////////////////////////////////////////// +// Coordinates class +//////////////////////////////////////////////////////////////////// + grm::Coordinates::Coordinates(const char *contig, const size_t len): - contig(contig), len(len), pos(std::make_unique(len)) {} + contig(contig), len(len), pos(std::make_unique(len)) {}; + + +grm::STATUS write(io::FileIO *fio, const Coordinates *coords) { + + if (!fio->fid) + return grm::ERROR_FILE_NOT_OPEN; + + // write contig name to file + size_t nwritten = 0; + size_t nchar = coords->contig.size(); + nwritten = fwrite(&nchar, sizeof(nchar), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + nwritten = fwrite(coords->contig.c_str(), + sizeof(coords->name[0]), + nchar, + fio->fid); + if (nwritten != nchar) + return grm::ERROR_ON_WRITE; + + // write positions + size_t npos = coords->len; + nwritten = fwrite(&npos, sizeof(npos), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + nwritten = fwrite(coords->pos.get(), + sizeof(coords->pos[0]), + npos, + fio->fid); + if (nwritten != pos) + return grm::ERROR_ON_WRITE; + + return grm::SUCCESS; +} +grm::STATUS read(io::FileIO* fio, Coordinates* coords) { +} + -// default constructor -grm::Grm::Grm(const size_t nrow, const size_t mcol) - : dims_(nrow, mcol), +//////////////////////////////////////////////////////////////////// +// GRM class +//////////////////////////////////////////////////////////////////// +// +// Recall that the GRM is a symmetric matrix, therefore we only need +// to store the upper triagonal and diagonal element values. +// Consequently, the size of the array storing the data is n*(n +1)/2. +// +grm::Grm::Grm(const size_t n_samples) + : n_samples_(n_samples) data_(size() != 0 ? std::make_unique(size()) : nullptr) { if (data_) std::memset(data_.get(), 0, size()); } +// The number of upper diagonal + diagonal elements of the GRM +size_t grm::Grm::size() const { return n_samples * (n_samples + 1) / 2; }; -// copy constructor -// -grm::Grm::Grm(const grm::Grm& other) - : nrow_(dims.other.nrow_), mcol_(dims.other.mcol_), - data_(std::make_unique(other.size())) { - std::memset(data_.get(), 0, size()); -} +grm::STATUS grm::Grm::midx_to_arr_(const size_t i, const size_t j, size_t *idx) const { + + if (i >= n_samples_ || j >= n_samples_) + return grm::ERROR_IDX_ARR_BOUNDS; -// TODO: check this. -grm::Grm::Grm(grm::Grm&& other) - : nrow_(dims_.other.nrow_), dims.mcol_(other.mcol_), - data_(std::move(other.data_)) {}; + // remember that by symmetry, the matrix is equal to its transpose + if (i <= j) + *idx = MATRIX_IDX_TO_ARRAY(i, j, n_samples_); + else + *idx = MATRIX_IDX_TO_ARRAY(j, i, n_samples_); + + return grm::SUCCESS; +} float grm::Grm::operator()(const size_t i, const size_t j) const { - return data_[i*mcol_ + j]; + return data_[MATRIX_IDX_TO_ARRAY(i, j, n_samples_)]; } float& grm::Grm::operator()(const size_t i, const size_t j) { - return data_[i*mcol_ + j]; + return data_[MATRIX_IDX_TO_ARRAY(i, j, n_samples_)]; } grm::STATUS grm::Grm::get(const size_t i, const size_t j, float *val) const { size_t idx = 0; - grm::STATUS status = grm::STATUS::FAILED; - if ((status = midx_to_arr_(i, j, &idx)) != grm::STATUS::SUCCESS) + grm::STATUS status = grm::FAILED; + if ((status = midx_to_arr_(i, j, &idx)) != grm::SUCCESS) return status; *val = data_[idx]; @@ -119,23 +172,7 @@ grm::STATUS grm::Grm::set(const size_t i, const size_t j, const float val) { } -const grm::Dims& grm::Grm::dims() const { return dims_; }; - - -grm::STATUS grm::Grm::midx_to_arr_(const size_t i, const size_t j, size_t *idx) const { - - if (i >= nrow_ || j >= mcol_) - return grm::STATUS::ERROR_IDX_ARR_BOUNDS; - - *idx = i*mcol_ + j; - return grm::STATUS::SUCCESS; -} - - -size_t grm::Grm::size() const { return dims_.nrow_ * dims_.mcol_; }; - - -grm::STATUS grm::Grm::write(const char *filename) const { +grm::STATUS grm::Grm::write(io.FileIO *fio, const Hdr *hdr) const { std::unique_ptr fid = make_unique(fopen(filename, "wb")); @@ -155,6 +192,6 @@ grm::STATUS grm::Grm::write(const char *filename) const { } -grm::STATUS grm::Grm::read(const char *filename, grm::Grm *grm) { +grm::Grm grm::Grm::read(io.FileIO *fio) { return grm::STATUS; } diff --git a/src/main.cpp b/src/main.cpp index d588bb7..d440da4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -99,7 +99,9 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } - // EXTRACT ARGS + // PARSE ARGS FOR RESPECTIVE SUBPROGRAMS AND RUN + // + // Compute the GRM for the specified contig if (parser.is_sub_cmd("contig")) { std::optional tmp_str {}; @@ -146,9 +148,8 @@ int main(int argc, char* argv[]) bool use_both { tmp_bool.value() }; - if ((use_gt && use_both) || (use_gt && use_ehc) || (use_both && use_ehc)) { - log.error("user must specify either use_gt, use_both, use_ds," + log.error("user must specify either use_gt, use_both, use_ehc," " or omit both options to compute the haplotype based" " relationship matrix."); exit(EXIT_FAILURE); @@ -167,25 +168,25 @@ int main(int argc, char* argv[]) else if (bstatus < 0) { log.error("Subsetting by sample file, %s, resulted in error", samp_fname.c_str()); - return -1; + exit(EXIT_FAILURE); } else if (bstatus > 0) { log.error("One or more samples specified in sample file, %s," " do not %s", samp_fname.c_str(), bcf_fname.c_str()); - return -1; + exit(EXIT_FAILURE); } log.info("Output matrix file: %s", out_fname.c_str()); - grm.Grm cov { bfid.n_samples(), bfid.n_samples() }; + grm::Grm grmatrix { bfid.n_samples() }; if (use_gt) { log.info("Relationship matrix: genotype"); status = compute_genotype_matrix(); } else if (use_ehc) { log.info("Relationship matrix: expected haplotype count"); - status = compute_ehc_matrix(&log, &bfid, &cov); + status = compute_ehc_matrix(&log, &bfid, &grmatrix); } else if (use_both) { log.info("Relationship matrix: expected alt allele and haplotype" " counts"); @@ -200,11 +201,17 @@ int main(int argc, char* argv[]) log.info("Writing to file"); - cov.write(out_fname); + grmatrix.write(out_fname); + } else if (parser.is_sub_cmd("loco")) { + // Compute the leave-one-chromosome-out matrix given a set of + // matricies. + // + printf("loco selected\n"); + } else if (parser.is_sub_cmd("assoc")) { + printf("association statistics selected\n"); } - if (parser.is_sub_cmd("loco")) - printf("loco selected\n"); + return status; From 4bfd157741aa31c33a5813f3d0fdf03b80addb12 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 18 Mar 2026 16:05:02 -0400 Subject: [PATCH 42/58] intermediate progress on read and write operations for data structures. --- include/constants.h | 11 ++ include/grm.h | 108 ++++++++++++++---- include/io.h | 28 ----- src/grm.cpp | 269 ++++++++++++++++++++++++++++++++++++++++++-- src/io.cpp | 237 +++++++++++++++++++------------------- 5 files changed, 472 insertions(+), 181 deletions(-) create mode 100644 include/constants.h diff --git a/include/constants.h b/include/constants.h new file mode 100644 index 0000000..339a9f1 --- /dev/null +++ b/include/constants.h @@ -0,0 +1,11 @@ + +#ifndef HEADER_CONSTANTS_H +#define HEADER_CONSTANTS_H + +#include + +namespace constants { + std::string version = std::string("0.0.1"); +} + +#endif diff --git a/include/grm.h b/include/grm.h index f8f12a3..0348267 100644 --- a/include/grm.h +++ b/include/grm.h @@ -25,8 +25,8 @@ // ACKNOWLEDGMENT // // Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. +// reviewed by Claude Opus 4.6, the AI assistant from Anthropic. +// Some recommendations have been incorporated. // #ifndef HEADER_GRM_H #define HEADER_GRM_H @@ -40,7 +40,7 @@ #include #include -#include "io.h" +#include "constants.h" // The algorithm for getting the array idx from matrix indexes is simply @@ -83,7 +83,10 @@ enum STATUS { ERROR_FOPEN, ERROR_EOF_NOT_REACHED, ERROR_ON_WRITE, + ERROR_ON_READ, ERROR_FILE_NOT_OPEN, + ERROR_NULLPTR_ARG, + ERROR_INVALID_ARG, }; @@ -95,42 +98,101 @@ enum GrmType { }; // @title: Store genomic coordinates used in GRM calculation -// @description: struct Coordinates { - const std::string contig; - const size_t len; - std::unique_ptr *pos; + Coordinates(): contig(""), len(0), pos(nullptr) {}; + Coordinates(const char* contig, const size_t len) + : contig(contig), + len(len), + pos(std::make_unique(len)) {}; + + Coordinates(Coordinates&) = delete; + Coordinates& operator=(Coordinates&) = delete; + + Coordinates(Coordinates&& other); + Coordinates& operator=(Coordinates&& other); + + // Data Fields + std::string contig; + size_t len; + std::unique_ptr pos; }; -STATUS write(io::FileIO* fio, Coordinates* coords); +// Coordinates Storage Layout +// +// type number description +// -------------------------------------------------------------------- +// size_t 1 number of characters (n) in contig name +// char n characters for contig name without null character +// size_t 1 number of genomic positions (npos) +// size_t npos the positions used for computation of the grm +// +STATUS write(io::FileIO* fio, const Coordinates* coords); STATUS read(io::FileIO* fio, Coordinates* coords); -// Storage in binary format. Sample names are comma separated -// [size_t len][names[0],names[1],names[2]...names[len-1]\0] +// Samples stores sample id strings and the number of samples +// struct Samples { - Samples(const size_t len): - len(len), - names(len == 0 ? nullptr : std::make_unique(len)){}; - const size_t len; //number of samples - std::unique_ptr names; + Samples(): len(0), names(nullptr); + Samples(size_t n_samples): + len(n_samples), + names(len == 0 ? nullptr : std::make_unique(len)) {}; + + Samples(const Samples&) = delete; + Samples& operator=(const Samples&) = delete; + + Samples(Samples&& other); + Samples& operator=(Samples&& other); + + // Data Fields + size_t len; //number of samples + std::unique_ptr names; + }; -STATUS write(io::FileIO *fio, Samples *samples); -STATUS read(io::FileIO *fio, Samples *samples); + +// Sample Storage Layout +// +// type number description +// -------------------------------------------------------------------- +// size_t 1 represents number of samples +// size_t 1 the number of characters of longest sample id +// size_t 1 number of characters (n_1) in first sample id +// char n_1 characters of sample id 1 without terminal null '\0' +// size_t 1 number of characters (n_2) in second sample id +// char n_2 characters of sample id 2 without terminal null '\0' +// ... +// size_t 1 number of characters (n_N) in N^{th} sample id +// char n_N characters of sample id N without terminal null '\0' + +STATUS write(io::FileIO* fio, const Samples* samples); +STATUS read(io::FileIO* fio, Samples* samples); // Header struct Hdr { - // const std::string program_version; - const size_t n_samples; - const GrmType grm_type; - const Coordinates *coords; - const Samples *samples; -}; + // Data Fields + GrmType grm_type; + Coordinates* coords; + Samples* samples; + + const std::string version = constants::version; +}; +// Header Storage Layout +// +// type number description +// -------------------------------------------------------------------- +// size_t 1 number of characters (n) in version string +// char n version string without null terminator +// GrmType 1 type of grm +// +// call write for coordinates +// +// call write for samples +// STATUS write(io::FileIO *fio, const Hdr *hdr); STATUS read(io::FileIO *fio, Hdr *hdr); diff --git a/include/io.h b/include/io.h index 702e7ef..9eed6c0 100644 --- a/include/io.h +++ b/include/io.h @@ -97,34 +97,6 @@ struct FileStats { STATUS wc(TextIO *tio, FileStats *fs); -template -struct Array { - Array(size_t size_in): size(size_in), - data(size > 0 ? new T[size] : nullptr) {}; - - ~Array() { if (data) delete[] data; }; - - size_t size; - T *data; - size_t len = 0; - - //unsafe referencing - T operator[](size_t i) { return data[i]; }; - T& operator[](size_t i) { return data[i]; }; - - STATUS append(T val) { - if (len >= size-1) - return END_OF_BUF_ERROR; - - data[len++] = val; - return SUCCESS; - } - - void fill(T val) { - std::memset(data, val, size); - len = 0; - } -} STATUS getline(TextIO *tio, Array linebuf); diff --git a/src/grm.cpp b/src/grm.cpp index 5e130d8..b9506db 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -4,8 +4,8 @@ // ACKNOWLEDGMENT // // Code design and original version completed by Robert Vogel, -// reviewed by Claude Sonnet, the AI assistant from Anthropic -// (Jan 2025), with minor recommendations incorporated. +// reviewed by Claude Opus 4.6, the AI assistant from Anthropic +// with minor recommendations incorporated. // // @@ -56,27 +56,55 @@ // //////////////////////////////////////////////////////////////////// -// Coordinates class +// COORDINATES CLASS //////////////////////////////////////////////////////////////////// -grm::Coordinates::Coordinates(const char *contig, const size_t len): - contig(contig), len(len), pos(std::make_unique(len)) {}; +grm::Coordinates::Coordinates(Coordinates&& other) + : len(0), contig(""), pos(nullptr) { + len = other.len; + contig = other.contig; + pos = std::move(other.pos); + other.pos=nullptr; + other.len = 0; + other.contig = ""; +} + +grm::Coordinates& grm::Coordinates::operator=(Coordinates&& other) { + if (this == &other) + return *this; + + len = other.len; + contig = other.contig; + pos = std::move(other.pos); + + other.pos=nullptr; + other.len = 0; + other.contig = ""; +} + +// remember that Coordinates* should be uninstantiated +grm::STATUS write(io::FileIO* fio, const Coordinates* coords) { -grm::STATUS write(io::FileIO *fio, const Coordinates *coords) { + if (!fio) + return grm::ERROR_NULLPTR_ARG; if (!fio->fid) - return grm::ERROR_FILE_NOT_OPEN; + return grm::ERROR_NULLPTR_ARG; + + if (!coords) + return grm::ERROR_NULLPTR_ARG; - // write contig name to file size_t nwritten = 0; + + // write contig name to file size_t nchar = coords->contig.size(); nwritten = fwrite(&nchar, sizeof(nchar), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; nwritten = fwrite(coords->contig.c_str(), - sizeof(coords->name[0]), + sizeof(char), nchar, fio->fid); if (nwritten != nchar) @@ -89,10 +117,10 @@ grm::STATUS write(io::FileIO *fio, const Coordinates *coords) { return grm::ERROR_ON_WRITE; nwritten = fwrite(coords->pos.get(), - sizeof(coords->pos[0]), + sizeof(size_t), npos, fio->fid); - if (nwritten != pos) + if (nwritten != npos) return grm::ERROR_ON_WRITE; return grm::SUCCESS; @@ -100,11 +128,228 @@ grm::STATUS write(io::FileIO *fio, const Coordinates *coords) { grm::STATUS read(io::FileIO* fio, Coordinates* coords) { + if (!fio) + return grm::ERROR_NULLPTR_ARG; + + if (!fio->fid) + return grm::ERROR_NULLPTR_ARG; + + if (!coords) + return grm::ERROR_NULLPTR_ARG; + + // I create a temporary Coordinates class, because I don't want + // the input coords instance to partial update upon an error + grm::Coordinates tmpc {}; + + // will store the number of bytes read at each step + size_t nread = 0; + + // read contig name + size_t size_contig_name = 0; + nread = fread(&size_contig_name, sizeof(size_t), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + std::unique_ptr buffer = std::make_unique(size_contig_name + 1); + std::memset(buffer.get(), '\0', size_contig_name + 1); + + nread = fread(buffer, sizeof(char), size_contig_name, fio->fid); + if (nread != size_contig_name) + return grm::ERROR_ON_READ; + + tmpc.contig = std::string(buffer); + + // read in positions + size_t npos = 0; + nread = fread(&npos, sizeof(size_t), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + tmpc.len = npos; + + tmpc.pos = std::make_unique(npos); + nread = fread(tmpc.pos.get(), sizeof(size_t), npos, fio->fid); + if (nread != npos) + return grm::ERROR_ON_READ; + + *coords = std::move(tmpc); + + return grm::SUCCESS; +} + +//////////////////////////////////////////////////////////////////// +// SAMPLES CLASS +//////////////////////////////////////////////////////////////////// + +grm::Samples::Samples(grm::Samples&& other) + : len(other.len), names(nullptr) { + names = std::move(other.names); + other.len = 0; + other.names = nullptr; +} + + +grp::Samples& grm::Samples::operator=(grm::Samples&& other) { + if (this == &other) + return *this; + + len = other.len; + names = std::move(other.names); + + other.len = 0; + other.names = nullptr; + + return *this; } +grm::STATUS write(io::FileIO* fio, const grm::Samples* samples) { + if (!fio) + return grm::ERROR_NULLPTR_ARG; + + if (!fio->fid) + return grm::ERROR_NULLPTR_ARG; + + if (!samples) + return grm::ERROR_NULLPTR_ARG; + + size_t nwritten = 0; + size_t nsamps = samples->len; + nwritten = fwrite(&nsamps, sizeof(size_t), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + // When it comes time to read the data, I need to make a character + // buffer to temporarily place the read string. To make this + // buffer, I need to know the length of string with the greatest + // number of characters. Here I find that number and store in + // the binary file. + + size_t nchar_max = 0; + size_t tmp = 0; + for (size_t n = 0; n < nsamps; n++) + if ((tmp = samples->name[n].size()) > nchar_max) nchar_max = tmp; + + if (nchar_max == 0) + return grm::ERROR_INVALID_ARG; + + nwritten = fwrite(&nchar_max, sizeof(size_t), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + // Write each string to file; + size_t nchar = 0; + for (size_t n = 0; n < nsamps; n++) { + nchar = samples->names[n].size(); + + nwritten = fwrite(&nchar, sizeof(size_t), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + nwritten = fwrite(samples->names[n].c_str(), + sizeof(char), + nchar, + fio->fid); + if (nwritten != nchar) + return grm::ERROR_ON_WRITE; + } + + return grm::SUCCESS; +} + +grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { + if (!fio) + return grm::ERROR_NULLPTR_ARG; + + if (!fio->fid) + return grm::ERROR_NULLPTR_ARG; + + // I create a temporary Sample class, because I don't want the + // input samples instance to partial update upon an error + grm::Samples tmp_samps(); + + size_t nread; + size_t n_samples = 0; + + nread = fread(&n_samples, sizeof(size_t), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + tmp_samps->len = n_samples; + + // Get the number of characters of the longest string + size_t nchar_max = 0; + nread = fread(&nchar_max, sizeof(size_t), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + + std::unique_ptr buffer = std::make_unique(nchar_max + 1); + std::memset(buffer.get(), '\0', nchar_max + 1); + + size_t nchar = 0; + for (size_t n = 0; n < n_samples: n++) { + nread = fread(&nchar, sizeof(size_t), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + nread = fread(buffer.get(), sizeof(char), nchar, fio->fid); + if (nread != nchar) + return grm::ERROR_ON_READ; + + tmp_samps->names[n] = std::string(buffer); + + nchar = 0; + std::memset(buffer.get(), '\0', nchar); + } + + *samples = std::move(tmp_samps); + + return grm::SUCCESS; +} + +//////////////////////////////////////////////////////////////////// +// HDR CLASS +//////////////////////////////////////////////////////////////////// + + +grm::STATUS write(io::FileIO* fio, const Hdr* hdr) { + if (!fio) + return grm::ERROR_NULLPTR_ARG; + + if (!fio->fid) + return grm::ERROR_NULLPTR_ARG; + + if (!hdr) + return grm::ERROR_NULLPTR_ARG; + + fwrite(&hdr->version.size(), sizeof(size_t), 1, fio->fid); + fwrite(hdr->version.c_str(), sizeof(char), hdr->version.size(), fio->fid); + + fwrite(&hdr->grm_type, sizeof(GrmType), 1, fio->fid); + grm::STATUS status; + if ((status = write(fio, hdr->coords)) != grm::SUCCESS) + return status; + + if ((status = write(fio, hdr->samples)) != grm::SUCCESS) + return status; + + return grm::SUCCESS; +} + +grm::STATUS read(io::FileIO* fio, Hdr* hdr) { + if (!fio) + return grm::ERROR_NULLPTR_ARG; + + if (!fio->fid) + return grm::ERROR_NULLPTR_ARG; + + if (!hdr) + return grm::ERROR_NULLPTR_ARG; + +} //////////////////////////////////////////////////////////////////// -// GRM class +// GRM CLASS //////////////////////////////////////////////////////////////////// // // Recall that the GRM is a symmetric matrix, therefore we only need diff --git a/src/io.cpp b/src/io.cpp index 60c514a..6a91f5b 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -1,119 +1,120 @@ -#include - - - -textio::TextIO::TextIO(FILE *fileid): fid(fileid) {}; - - - - -std::unique_ptr textio::open(const char *filename, - const char *mode) { - - FILE *fid = fopen(filename, mode); - if (ferror(fid)) { - fclose(fid); - return nullptr; - } - - std::unique_ptr tio = std::make_unique(fid); - - return std::move(tio); -} - - -textio::STATUS wc(textio::TextIO *tio, textio::FileStats *fs) { - if (!fs) - return textio::INVALID_ARG_ERROR; - - size_t nchar = 0; - size_t nwords = 0; - size_t nlines = 0; - size_t nblanklines = 0; - - size_t word_len = 0; - - FILE *fid = tio->fid; - - int c = '\0'; - while ((c = fgetc(fid)) != EOF) { - - switch (c) { - case '\n': - nlines++; - - if (word_len == 0) - nblanklines++; - else { - nwords++; - word_len = 0; - } - break; - case ';': - case ':': - case ',': - case '!': - case '?': - case '(': - case ')': - case '\"': - case '\t': - case ' ': - if (word_len == 0) - break; - - nwords++; - word_len = 0; - - break; - default: - nchar++; - word_len++; - } - - } - - if (ferror(fid)) { - fs = nullptr; - return tio->bseek() == 0 ? textio::FERROR : textio::FSEEK_ERROR; - } - - if (feof(fid) == 0) { - fs = nullptr; - return tio->bseek() == 0 ? textio::FEOF_ERROR : textio::FSEEK_ERROR; - } - - fs->nchar = nchar; - fs->nwords = nwords; - fs->nlines = nlines; - fs->nblanklines = nblanklines; - - return tio->bseek() ? textio::SUCCESS : textio::FSEEK_ERROR; -} - - - -textio::STATUS textio::getline(textio::TextIO *tio, textio::Array *buf) { - buf->fill('\0'); - - FILE *fid = buf->fid; - - int c = 0; - while ((c = fgetc(fid)) != EOF) { - - if (c == '\n') { - buf->append('\0'); - return textio::SUCCESS; - - buf->append(c) - } - - if (ferror(fid)) - return textio::FERROR; - - if (feof(fid) == 0) - return textio::FEOF_ERROR; - - return textio::SUCCESS; -} +#include + + + + +// textio::TextIO::TextIO(FILE *fileid): fid(fileid) {}; +// +// +// +// +// std::unique_ptr textio::open(const char *filename, +// const char *mode) { +// +// FILE *fid = fopen(filename, mode); +// if (ferror(fid)) { +// fclose(fid); +// return nullptr; +// } +// +// std::unique_ptr tio = std::make_unique(fid); +// +// return std::move(tio); +// } +// +// +// textio::STATUS wc(textio::TextIO *tio, textio::FileStats *fs) { +// if (!fs) +// return textio::INVALID_ARG_ERROR; +// +// size_t nchar = 0; +// size_t nwords = 0; +// size_t nlines = 0; +// size_t nblanklines = 0; +// +// size_t word_len = 0; +// +// FILE *fid = tio->fid; +// +// int c = '\0'; +// while ((c = fgetc(fid)) != EOF) { +// +// switch (c) { +// case '\n': +// nlines++; +// +// if (word_len == 0) +// nblanklines++; +// else { +// nwords++; +// word_len = 0; +// } +// break; +// case ';': +// case ':': +// case ',': +// case '!': +// case '?': +// case '(': +// case ')': +// case '\"': +// case '\t': +// case ' ': +// if (word_len == 0) +// break; +// +// nwords++; +// word_len = 0; +// +// break; +// default: +// nchar++; +// word_len++; +// } +// +// } +// +// if (ferror(fid)) { +// fs = nullptr; +// return tio->bseek() == 0 ? textio::FERROR : textio::FSEEK_ERROR; +// } +// +// if (feof(fid) == 0) { +// fs = nullptr; +// return tio->bseek() == 0 ? textio::FEOF_ERROR : textio::FSEEK_ERROR; +// } +// +// fs->nchar = nchar; +// fs->nwords = nwords; +// fs->nlines = nlines; +// fs->nblanklines = nblanklines; +// +// return tio->bseek() ? textio::SUCCESS : textio::FSEEK_ERROR; +// } +// +// +// +// textio::STATUS textio::getline(textio::TextIO *tio, textio::Array *buf) { +// buf->fill('\0'); +// +// FILE *fid = buf->fid; +// +// int c = 0; +// while ((c = fgetc(fid)) != EOF) { +// +// if (c == '\n') { +// buf->append('\0'); +// return textio::SUCCESS; +// +// buf->append(c) +// } +// +// if (ferror(fid)) +// return textio::FERROR; +// +// if (feof(fid) == 0) +// return textio::FEOF_ERROR; +// +// return textio::SUCCESS; +// } From 4cc4c5d38eada7ac3a90d53b1a8d3f8c87f95863 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Wed, 18 Mar 2026 19:51:04 -0400 Subject: [PATCH 43/58] worked on implementing read/write for header class --- include/grm.h | 29 +++++++++---- src/grm.cpp | 114 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 120 insertions(+), 23 deletions(-) diff --git a/include/grm.h b/include/grm.h index 0348267..09ac2be 100644 --- a/include/grm.h +++ b/include/grm.h @@ -70,7 +70,7 @@ // making the equation above read // // i * (n_samples + 1) - i * (i+1)/2 + j -#define MATRIX_IDX_TO_ARRAY(i, j, n) ((i) * (n + 1) - (i)*(i+1)/2 + j) +#define MATRIX_IDX_TO_ARRAY(i, j, n) ((i)*((n) + 1) - (i)*((i)+1)/2 + (j)) namespace grm { @@ -95,6 +95,7 @@ enum GrmType { EAC, // Expected Alternative Allele Count BOTH, // Both EHC AND EAC DS, // Dosage, i.e. Called Alternative Allele Count + UNSPECIFIED, }; // @title: Store genomic coordinates used in GRM calculation @@ -103,7 +104,7 @@ struct Coordinates { Coordinates(const char* contig, const size_t len) : contig(contig), len(len), - pos(std::make_unique(len)) {}; + pos(std::make_unique(len)) {}; Coordinates(Coordinates&) = delete; Coordinates& operator=(Coordinates&) = delete; @@ -134,7 +135,7 @@ STATUS read(io::FileIO* fio, Coordinates* coords); // Samples stores sample id strings and the number of samples // struct Samples { - Samples(): len(0), names(nullptr); + Samples(): len(0), names(nullptr) {}; Samples(size_t n_samples): len(n_samples), names(len == 0 ? nullptr : std::make_unique(len)) {}; @@ -173,12 +174,24 @@ STATUS read(io::FileIO* fio, Samples* samples); // Header struct Hdr { + Hdr() + : version(""), + grm_type(UNSPECIFIED), + coords(std::make_unique), + samples(std::make_unique) {}; + + Hdr(Hdr&) = delete; + Hdr& operator=(Hdr&) = delete; + + Hdr(Hdr&&); + Hdr& operator=(Hdr&&); + // Data Fields + std::string version; GrmType grm_type; - Coordinates* coords; - Samples* samples; + std::unique_ptr coords; + std::unique_ptr samples; - const std::string version = constants::version; }; // Header Storage Layout @@ -241,8 +254,8 @@ class Grm { // @param hdr: an instance of grm::Hdr with important meta data // @return grm::STATUS: // -STATUS write(io.FileIO *fio, const Hdr *hdr, const Grm *grmatrix) const; -STATUS read(io.FileIO *fio, const Hdr *hdr, Grm *grmatrix); +STATUS write(io::FileIO *fio, const Hdr *hdr, const Grm *grmatrix) const; +STATUS read(io::FileIO *fio, const Hdr *hdr, Grm *grmatrix); } diff --git a/src/grm.cpp b/src/grm.cpp index b9506db..f90e7e4 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -81,6 +81,8 @@ grm::Coordinates& grm::Coordinates::operator=(Coordinates&& other) { other.pos=nullptr; other.len = 0; other.contig = ""; + + return *this; } // remember that Coordinates* should be uninstantiated @@ -153,11 +155,11 @@ grm::STATUS read(io::FileIO* fio, Coordinates* coords) { std::unique_ptr buffer = std::make_unique(size_contig_name + 1); std::memset(buffer.get(), '\0', size_contig_name + 1); - nread = fread(buffer, sizeof(char), size_contig_name, fio->fid); + nread = fread(buffer.get(), sizeof(char), size_contig_name, fio->fid); if (nread != size_contig_name) return grm::ERROR_ON_READ; - tmpc.contig = std::string(buffer); + tmpc.contig = std::string(buffer, size_contig_name); // read in positions size_t npos = 0; @@ -189,7 +191,7 @@ grm::Samples::Samples(grm::Samples&& other) } -grp::Samples& grm::Samples::operator=(grm::Samples&& other) { +grm::Samples& grm::Samples::operator=(grm::Samples&& other) { if (this == &other) return *this; @@ -228,7 +230,7 @@ grm::STATUS write(io::FileIO* fio, const grm::Samples* samples) { size_t nchar_max = 0; size_t tmp = 0; for (size_t n = 0; n < nsamps; n++) - if ((tmp = samples->name[n].size()) > nchar_max) nchar_max = tmp; + if ((tmp = samples->names[n].size()) > nchar_max) nchar_max = tmp; if (nchar_max == 0) return grm::ERROR_INVALID_ARG; @@ -266,7 +268,7 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { // I create a temporary Sample class, because I don't want the // input samples instance to partial update upon an error - grm::Samples tmp_samps(); + grm::Samples tmp_samps {}; size_t nread; size_t n_samples = 0; @@ -275,7 +277,8 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { if (nread != 1) return grm::ERROR_ON_READ; - tmp_samps->len = n_samples; + tmp_samps.len = n_samples; + tmp_samps.names = srd::make_unique(n_samples); // Get the number of characters of the longest string size_t nchar_max = 0; @@ -284,11 +287,11 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { return grm::ERROR_ON_READ; - std::unique_ptr buffer = std::make_unique(nchar_max + 1); + std::unique_ptr buffer = std::make_unique(nchar_max+1); std::memset(buffer.get(), '\0', nchar_max + 1); size_t nchar = 0; - for (size_t n = 0; n < n_samples: n++) { + for (size_t n = 0; n < n_samples; n++) { nread = fread(&nchar, sizeof(size_t), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; @@ -297,10 +300,10 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { if (nread != nchar) return grm::ERROR_ON_READ; - tmp_samps->names[n] = std::string(buffer); + tmp_samps.names[n] = std::string(buffer, nchar); - nchar = 0; std::memset(buffer.get(), '\0', nchar); + nchar = 0; } *samples = std::move(tmp_samps); @@ -312,6 +315,40 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { // HDR CLASS //////////////////////////////////////////////////////////////////// +grm::Hdr::Hdr(Hdr&& other) + : version(other.version), + grm_type(other.grm_type), + coords(nullptr), + samples(nullptr) { + + other.version = ""; + other.grm_type = grm::UNSPECIFIED; + + *coords = std::move(*other.coords); + other.coords = nullptr; + + samples = std::move(*other.samples); + other.samples = nullptr; +} + +grm::Hdr& grm::Hdr::operator=(Hdr&& other) { + if (this == &other) + return *this; + + version = other.version; + other.version = ""; + + grm_type = other.grm_type; + other.grm_type = grm::UNSPECIFIED; + + *coords = std::move(*other.coords); + other.coords = nullptr; + + *samples = std::move(*other.samples); + other.samples = nullptr; + return *this; +} + grm::STATUS write(io::FileIO* fio, const Hdr* hdr) { if (!fio) @@ -323,10 +360,22 @@ grm::STATUS write(io::FileIO* fio, const Hdr* hdr) { if (!hdr) return grm::ERROR_NULLPTR_ARG; - fwrite(&hdr->version.size(), sizeof(size_t), 1, fio->fid); - fwrite(hdr->version.c_str(), sizeof(char), hdr->version.size(), fio->fid); + size_t nwritten = 0; + + size_t nchar = hdr->version.size(); + nwritten = fwrite(&nchar, sizeof(size_t), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + nwritten = fwrite(hdr->version.c_str(), + sizeof(char), nchar, fio->fid); + if (nwritten != nchar) + return grm::ERROR_ON_WRITE; + + nwritten = fwrite(&hdr->grm_type, sizeof(GrmType), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; - fwrite(&hdr->grm_type, sizeof(GrmType), 1, fio->fid); grm::STATUS status; if ((status = write(fio, hdr->coords)) != grm::SUCCESS) return status; @@ -347,7 +396,42 @@ grm::STATUS read(io::FileIO* fio, Hdr* hdr) { if (!hdr) return grm::ERROR_NULLPTR_ARG; + Hdr tmp_hdr {}; + + size_t nread = 0; + size_t nchar_version = 0; + + nread = fread(&nchar_version, sizeof(size_t), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + std::unique_ptr buffer = std::make_unique(nchar_version + 1); + std::memset(buffer.get(), '\0', nchar_version + 1); + + nread = fread(buffer.get(), sizeof(char), nchar_version, fio->fid); + if (nread != nchar_version) + return grm::ERROR_ON_READ; + + tmp_hdr.version = std::string(buffer, nchar_version); + + nread = fread(&tmp_hdr.grm_type, sizeof(grm::GrmType), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + grm::STATUS status; + status = read(fio, tmp_hdr.coords.get()); + if (status != grm::SUCCESS) + return status; + + status = read(fio, tmp_hdr.samples.get()); + if (status != grm::SUCCESS) + return status; + + *hdr = std::move(tmp_hdr); + return grm::SUCCESS; } + + //////////////////////////////////////////////////////////////////// // GRM CLASS //////////////////////////////////////////////////////////////////// @@ -417,7 +501,7 @@ grm::STATUS grm::Grm::set(const size_t i, const size_t j, const float val) { } -grm::STATUS grm::Grm::write(io.FileIO *fio, const Hdr *hdr) const { +grm::STATUS grm::Grm::write(io::FileIO *fio, const Hdr *hdr) const { std::unique_ptr fid = make_unique(fopen(filename, "wb")); @@ -437,6 +521,6 @@ grm::STATUS grm::Grm::write(io.FileIO *fio, const Hdr *hdr) const { } -grm::Grm grm::Grm::read(io.FileIO *fio) { +grm::Grm grm::Grm::read(io::FileIO *fio) { return grm::STATUS; } From ba444506a203f58757e7f54eda163d38bb70160d Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 19 Mar 2026 15:46:14 -0400 Subject: [PATCH 44/58] added grm read /write code, update for size_t -> uint64_t, versioning structures --- README.md | 71 +++++-------- include/constants.h | 7 +- include/grm.h | 53 ++++++---- include/utils.h | 62 +++++++++++ src/grm.cpp | 244 ++++++++++++++++++++++++++++---------------- 5 files changed, 284 insertions(+), 153 deletions(-) create mode 100644 include/utils.h diff --git a/README.md b/README.md index 6c7fd00..e605b63 100644 --- a/README.md +++ b/README.md @@ -231,61 +231,44 @@ The program is only available as source from this repository and requires The `.grm` file format is a binary data format consisting of meta data and a payload. -### Defined types -The `Array` type is a minimal dynamic data storage structure where -the length of the array is known and its address on the heap saved in -a pointer. -``` -template -struct Array { - uint32_t len; - T data[len]; -}; -``` - -The version structure is bit packed with fields specifying -the `grm` program version number. +Assume 64-bit machine, little-endian, e.g. ARM and x86-64. + -``` -struct version { - uint32_t major: 10; - uint32_t minor: 10; - uint32_t micro: 10; - uint32_t : 2; -}; -``` +### Meta data + +| offset | type | size | description | +| (bytes) | | (bytes) | | +| --------- | --------- | --------- | ----------------------------------------- | +| 0 | uint32_t | 4 | File signature (0x47524D00 = "GRM\0") | +| 4 | uint32_t | 4 | File version, utils::Version | +| 8 | uint32_t | 4 | Program version, utils::Version | +| 12 | varies | $n_{coords}$ | Genomic coordinates, grm::Coordinates | +| $n_{coords}$ + 12 | varies | $n_{samps}$ | Sample names, grm::Samples | +| $n_{samps} + n_{coords} +12$ | | | Total | -The date structure is bit packed with fields specifying when -the `grm` program was launched. -``` -struct date { - uint32_t year : 12; - uint32_t month : 4; - uint32_t day : 5; - uint32_t hour : 5; - uint32_t sec : 6; -}; -``` +**Genomic coordinates** -### Meta data +| offset | type | size | description | +| (bytes) | | (bytes) | | +| --------- | --------- | --------- | ----------------------------------------- | +| 0 | uint32_t | 4 | $l_{contig}=$ strlen(contig name) | +| 4 | char[] | $n_{contig} = l$ | Contig name chars, sizeof(char) = 1 byte | +| $n_{contig} + 4$ | uint32_t | 4 | Number of positions ($l_{pos}$) | +| $n_{contig} + 8$ | uint32_t | $n_{pos} = 4 l_{pos}$ | loci positions on contig | +| $n_{coords} =n_{contig} + n_{pos} + 8$ | | Total | -| offset | field | type | size (bytes) | description | -| --- | --- | --- | --- | --- | -| 0 | magic | uint32_t | 4 | File signature (0x47524D00 = "GRM\0") | -| 4 | version | struct version | 4 | Program version: major/minor/micro | -| 8 | date | struct date | 4 | Launch time: year/month/day/hour/sec | -| 12 | user | Array\ | 4 + len | Username of person who ran the program | -| varies | contig | Array\ | 4 + len | Chromosome or contig name | -| varies | markers | Array\ | 4 + 4×len | Marker positions used for GRM computation | -| varies | samples | Array\\> | 4 + Σ(4 + len_i) | Sample IDs in column order of the GRM | +**Sample names** +| offset | type | size | description | +| (bytes) | | (bytes) | | +| --------- | --------- | --------- | ----------------------------------------- | ### Payload -The payload is the upper triangular and diagonal components of the +Given $N$ samples, the payload is the upper triangular components of the $N \times N$ GRM stored as an array of $N(N+1)/2$ 32-bit floating point numbers in row-major order. While the genotype-based GRM will not produce fractional values, the expected count GRMs will, making `float32` an diff --git a/include/constants.h b/include/constants.h index 339a9f1..2b8049e 100644 --- a/include/constants.h +++ b/include/constants.h @@ -2,10 +2,13 @@ #ifndef HEADER_CONSTANTS_H #define HEADER_CONSTANTS_H -#include +#include "utils.h" + namespace constants { - std::string version = std::string("0.0.1"); + +constexpr utils::Version PROG_VERSION { 0, 0, 1 }; + } #endif diff --git a/include/grm.h b/include/grm.h index 09ac2be..f93b520 100644 --- a/include/grm.h +++ b/include/grm.h @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -41,6 +42,8 @@ #include #include "constants.h" +#include "utils.h" + // The algorithm for getting the array idx from matrix indexes is simply @@ -75,6 +78,15 @@ namespace grm { + +// Recall that the magic number is simply GRM\0 in hex, +// 0x47 = G, 0x52 = R, and so on. The magic number provides +// a simple means to determine file type when parsing. +constexpr uint32_t FILE_TYPE_SPEC = 0x47524D00; +constexpr utils::Version FILE_VERSION = { 0, 0, 0 }; +constexpr char[] FILE_SUFFIX = ".grm"; + + enum STATUS { SUCCESS, FAILED, @@ -87,6 +99,7 @@ enum STATUS { ERROR_FILE_NOT_OPEN, ERROR_NULLPTR_ARG, ERROR_INVALID_ARG, + ERROR_NOT_A_GRM_FILE, }; @@ -114,8 +127,8 @@ struct Coordinates { // Data Fields std::string contig; - size_t len; - std::unique_ptr pos; + uint64_t len; + std::unique_ptr pos; }; @@ -136,7 +149,7 @@ STATUS read(io::FileIO* fio, Coordinates* coords); // struct Samples { Samples(): len(0), names(nullptr) {}; - Samples(size_t n_samples): + Samples(uint64_t n_samples): len(n_samples), names(len == 0 ? nullptr : std::make_unique(len)) {}; @@ -147,7 +160,7 @@ struct Samples { Samples& operator=(Samples&& other); // Data Fields - size_t len; //number of samples + uint64_t len; //number of samples std::unique_ptr names; }; @@ -174,12 +187,7 @@ STATUS read(io::FileIO* fio, Samples* samples); // Header struct Hdr { - Hdr() - : version(""), - grm_type(UNSPECIFIED), - coords(std::make_unique), - samples(std::make_unique) {}; - + Hdr(); Hdr(Hdr&) = delete; Hdr& operator=(Hdr&) = delete; @@ -187,7 +195,9 @@ struct Hdr { Hdr& operator=(Hdr&&); // Data Fields - std::string version; + utils::Version prog_version; + utils::Version file_version; + GrmType grm_type; std::unique_ptr coords; std::unique_ptr samples; @@ -218,15 +228,16 @@ STATUS read(io::FileIO *fio, Hdr *hdr); // // @param n_samples of the GRM. // -class Grm { -public: +struct Grm { // - Grm(const size_t n_samples); + Grm(); + Grm(uint64_t n_samps); Grm(const Grm&)=delete; - Grm(Grm&&)=delete; Grm& operator=(const Grm&)=delete; - Grm& operator=(Grm&&)=delete; + + Grm(Grm&&); + Grm& operator=(Grm&&); // Unchecked indexes when setting and getting of matrix values float operator()(const size_t i, const size_t j) const; @@ -236,12 +247,12 @@ class Grm { STATUS set(const size_t i, const size_t j, const float val); STATUS get(const size_t i, const size_t j, float *val) const; - size_t size() const; + size_t size(); + + STATUS midx_to_arr(const size_t i, const size_t j, size_t* idx) const; -private: - const size_t n_samples_; - std::unique_ptr data_; - size_t midx_to_arr_(const size_t&, const size_t&) const; + uint64_t n_samples; + std::unique_ptr data; }; // @title: Write meta-data and computed grm elements to file diff --git a/include/utils.h b/include/utils.h new file mode 100644 index 0000000..bb4e698 --- /dev/null +++ b/include/utils.h @@ -0,0 +1,62 @@ +#ifndef HEADER_UTILS_H +#define HEADER_UTILS_H + +#include + +namespace utils { + +// Claude recommends that I define serialization explicitly so +// that the bit packed field order is not ABI dependent +struct Version { + uint8_t major; + uint16_t minor; + uint16_t micro; + + // pack as 8, 12, 12 for 32 bits in total + uint32_t pack() { + return (static_cast(major) << 24 + | static_cast(minor & 0x0FFF) << 12 + | static_cast(micro & 0x0FFF)); + } + + static Version unpack(uint32_t vnum) { + return Version { + static_cast(vnum >> 24), + static_cast(vnum >> 12 & 0x0FFF), + static_cast(vnum & 0x0FFF) + }; + } +}; + +// template +// struct Array { +// Array(size_t size_in): size(size_in), +// data(size > 0 ? new T[size] : nullptr) {}; +// +// ~Array() { if (data) delete[] data; }; +// +// size_t size; +// T *data; +// size_t len = 0; +// +// //unsafe referencing +// T operator[](size_t i) { return data[i]; }; +// T& operator[](size_t i) { return data[i]; }; +// +// STATUS append(T val) { +// if (len >= size-1) +// return END_OF_BUF_ERROR; +// +// data[len++] = val; +// return SUCCESS; +// } +// +// void fill(T val) { +// std::memset(data, val, size); +// len = 0; +// } +// }; + +} + +#endif diff --git a/src/grm.cpp b/src/grm.cpp index f90e7e4..a10e005 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -60,12 +60,8 @@ //////////////////////////////////////////////////////////////////// grm::Coordinates::Coordinates(Coordinates&& other) - : len(0), contig(""), pos(nullptr) { - len = other.len; - contig = other.contig; - pos = std::move(other.pos); + : len(other.len), contig(other.contig), pos(std::move(other.pos)) { - other.pos=nullptr; other.len = 0; other.contig = ""; } @@ -100,7 +96,7 @@ grm::STATUS write(io::FileIO* fio, const Coordinates* coords) { size_t nwritten = 0; // write contig name to file - size_t nchar = coords->contig.size(); + uint64_t nchar = coords->contig.size(); nwritten = fwrite(&nchar, sizeof(nchar), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; @@ -109,20 +105,20 @@ grm::STATUS write(io::FileIO* fio, const Coordinates* coords) { sizeof(char), nchar, fio->fid); - if (nwritten != nchar) + if (static_cast(nwritten) != nchar) return grm::ERROR_ON_WRITE; // write positions - size_t npos = coords->len; + uint64_t npos = coords->len; nwritten = fwrite(&npos, sizeof(npos), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; nwritten = fwrite(coords->pos.get(), - sizeof(size_t), + sizeof(uint64_t), npos, fio->fid); - if (nwritten != npos) + if (static_cast(nwritten) != npos) return grm::ERROR_ON_WRITE; return grm::SUCCESS; @@ -147,8 +143,8 @@ grm::STATUS read(io::FileIO* fio, Coordinates* coords) { size_t nread = 0; // read contig name - size_t size_contig_name = 0; - nread = fread(&size_contig_name, sizeof(size_t), 1, fio->fid); + uint64_t size_contig_name = 0; + nread = fread(&size_contig_name, sizeof(size_contig_name), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; @@ -156,22 +152,22 @@ grm::STATUS read(io::FileIO* fio, Coordinates* coords) { std::memset(buffer.get(), '\0', size_contig_name + 1); nread = fread(buffer.get(), sizeof(char), size_contig_name, fio->fid); - if (nread != size_contig_name) + if (static_cast(nread) != size_contig_name) return grm::ERROR_ON_READ; tmpc.contig = std::string(buffer, size_contig_name); // read in positions - size_t npos = 0; - nread = fread(&npos, sizeof(size_t), 1, fio->fid); + uint64_t npos = 0; + nread = fread(&npos, sizeof(npos), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; tmpc.len = npos; - tmpc.pos = std::make_unique(npos); - nread = fread(tmpc.pos.get(), sizeof(size_t), npos, fio->fid); - if (nread != npos) + tmpc.pos = std::make_unique(npos); + nread = fread(tmpc.pos.get(), sizeof(uint64_t), npos, fio->fid); + if (static_cast(nread) != npos) return grm::ERROR_ON_READ; *coords = std::move(tmpc); @@ -184,10 +180,8 @@ grm::STATUS read(io::FileIO* fio, Coordinates* coords) { //////////////////////////////////////////////////////////////////// grm::Samples::Samples(grm::Samples&& other) - : len(other.len), names(nullptr) { - names = std::move(other.names); + : len(other.len), names(std::move(other.names)) { other.len = 0; - other.names = nullptr; } @@ -199,7 +193,6 @@ grm::Samples& grm::Samples::operator=(grm::Samples&& other) { names = std::move(other.names); other.len = 0; - other.names = nullptr; return *this; } @@ -216,8 +209,8 @@ grm::STATUS write(io::FileIO* fio, const grm::Samples* samples) { return grm::ERROR_NULLPTR_ARG; size_t nwritten = 0; - size_t nsamps = samples->len; - nwritten = fwrite(&nsamps, sizeof(size_t), 1, fio->fid); + uint64_t nsamps = samples->len; + nwritten = fwrite(&nsamps, sizeof(nsamps), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; @@ -227,24 +220,24 @@ grm::STATUS write(io::FileIO* fio, const grm::Samples* samples) { // number of characters. Here I find that number and store in // the binary file. - size_t nchar_max = 0; - size_t tmp = 0; - for (size_t n = 0; n < nsamps; n++) + uint64_t nchar_max = 0; + uint64_t tmp = 0; + for (uint64_t n = 0; n < nsamps; n++) if ((tmp = samples->names[n].size()) > nchar_max) nchar_max = tmp; if (nchar_max == 0) return grm::ERROR_INVALID_ARG; - nwritten = fwrite(&nchar_max, sizeof(size_t), 1, fio->fid); + nwritten = fwrite(&nchar_max, sizeof(nchar_max), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; // Write each string to file; - size_t nchar = 0; - for (size_t n = 0; n < nsamps; n++) { + uint64_t nchar = 0; + for (uint64_t n = 0; n < nsamps; n++) { nchar = samples->names[n].size(); - nwritten = fwrite(&nchar, sizeof(size_t), 1, fio->fid); + nwritten = fwrite(&nchar, sizeof(nchar), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; @@ -266,14 +259,17 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { if (!fio->fid) return grm::ERROR_NULLPTR_ARG; + if (!samples) + return grm::ERROR_NULLPTR_ARG; + // I create a temporary Sample class, because I don't want the // input samples instance to partial update upon an error grm::Samples tmp_samps {}; size_t nread; - size_t n_samples = 0; + uint64_t n_samples = 0; - nread = fread(&n_samples, sizeof(size_t), 1, fio->fid); + nread = fread(&n_samples, sizeof(n_samples), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; @@ -297,7 +293,7 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { return grm::ERROR_ON_READ; nread = fread(buffer.get(), sizeof(char), nchar, fio->fid); - if (nread != nchar) + if (static_cast(nread) != nchar) return grm::ERROR_ON_READ; tmp_samps.names[n] = std::string(buffer, nchar); @@ -315,37 +311,41 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { // HDR CLASS //////////////////////////////////////////////////////////////////// +grm::Hdr::Hdr() + : prog_version(constants::PROG_VERSION), + file_version(grm::FILE_VERSION), + grm_type(UNSPECIFIED), coords(nullptr), samples(nullptr) {}; + + grm::Hdr::Hdr(Hdr&& other) - : version(other.version), + : prog_version(other.prog_version), + file_version(other.file_version), grm_type(other.grm_type), - coords(nullptr), - samples(nullptr) { + coords(nullptr), samples(nullptr) { - other.version = ""; + other.prog_version = constants::PROG_VERSION; + other.file_version = grm::FILE_VERSION; other.grm_type = grm::UNSPECIFIED; - *coords = std::move(*other.coords); - other.coords = nullptr; - - samples = std::move(*other.samples); - other.samples = nullptr; + coords = std::move(other.coords); + samples = std::move(other.samples); } grm::Hdr& grm::Hdr::operator=(Hdr&& other) { if (this == &other) return *this; - version = other.version; - other.version = ""; + prog_version = other.prog_version; + other.prog_version = constants::PROG_VERSION; + + file_version = other.file_version; + other.file_version = FILE_VERSION; grm_type = other.grm_type; other.grm_type = grm::UNSPECIFIED; - *coords = std::move(*other.coords); - other.coords = nullptr; - - *samples = std::move(*other.samples); - other.samples = nullptr; + coords = std::move(other.coords); + samples = std::move(other.samples); return *this; } @@ -399,9 +399,9 @@ grm::STATUS read(io::FileIO* fio, Hdr* hdr) { Hdr tmp_hdr {}; size_t nread = 0; - size_t nchar_version = 0; + uint64_t nchar_version = 0; - nread = fread(&nchar_version, sizeof(size_t), 1, fio->fid); + nread = fread(&nchar_version, sizeof(nchar_version), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; @@ -440,87 +440,159 @@ grm::STATUS read(io::FileIO* fio, Hdr* hdr) { // to store the upper triagonal and diagonal element values. // Consequently, the size of the array storing the data is n*(n +1)/2. // -grm::Grm::Grm(const size_t n_samples) - : n_samples_(n_samples) - data_(size() != 0 ? std::make_unique(size()) : nullptr) { - if (data_) - std::memset(data_.get(), 0, size()); +grm::Grm::Grm(): n_samples(0), data(nullptr) {}; + +grm::Grm::Grm(uint64_t n_samps) + : n_samples(n_samps) + data(size() != 0 ? std::make_unique(size()) : nullptr) { + + if (data) + std::memset(data.get(), 0, size()); +} + +grm::Grm::Grm(grm::Grm&& other) + : n_samples(other.n_samples), data(std::move(other.data)) { + other.n_samples = 0; +}; + +grm::Grm& grm::Grm::operator=(grm::Grm&& other) { + if (this == &other) + return *this; + + n_samples = other.n_samples; + other.n_samples = 0; + + data = std::move(other.data); + + return *this; } // The number of upper diagonal + diagonal elements of the GRM -size_t grm::Grm::size() const { return n_samples * (n_samples + 1) / 2; }; +uint64_t grm::Grm::size() const { return n_samples * (n_samples + 1) / 2; }; -grm::STATUS grm::Grm::midx_to_arr_(const size_t i, const size_t j, size_t *idx) const { +grm::STATUS grm::Grm::midx_to_arr(const uint64_t i, const uint64_t j, + uint64_t* idx) const { - if (i >= n_samples_ || j >= n_samples_) + if (i >= n_samples || j >= n_samples) return grm::ERROR_IDX_ARR_BOUNDS; // remember that by symmetry, the matrix is equal to its transpose if (i <= j) - *idx = MATRIX_IDX_TO_ARRAY(i, j, n_samples_); + *idx = MATRIX_IDX_TO_ARRAY(i, j, n_samples); else - *idx = MATRIX_IDX_TO_ARRAY(j, i, n_samples_); + *idx = MATRIX_IDX_TO_ARRAY(j, i, n_samples); return grm::SUCCESS; } -float grm::Grm::operator()(const size_t i, const size_t j) const { - return data_[MATRIX_IDX_TO_ARRAY(i, j, n_samples_)]; +float grm::Grm::operator()(const uint64_t i, const uint64_t j) const { + return data[MATRIX_IDX_TO_ARRAY(i, j, n_samples)]; } -float& grm::Grm::operator()(const size_t i, const size_t j) { - return data_[MATRIX_IDX_TO_ARRAY(i, j, n_samples_)]; +float& grm::Grm::operator()(const uint64_t i, const uint64_t j) { + return data[MATRIX_IDX_TO_ARRAY(i, j, n_samples)]; } -grm::STATUS grm::Grm::get(const size_t i, const size_t j, float *val) const { - size_t idx = 0; +grm::STATUS grm::Grm::get(const uint64_t i, const uint64_t j, float *val) const { + uint64_t idx = 0; grm::STATUS status = grm::FAILED; if ((status = midx_to_arr_(i, j, &idx)) != grm::SUCCESS) return status; - *val = data_[idx]; + *val = data[idx]; return status; } -grm::STATUS grm::Grm::set(const size_t i, const size_t j, const float val) { - size_t idx = 0; +grm::STATUS grm::Grm::set(const uint64_t i, const uint64_t j, const float val) { + uint64_t idx = 0; grm::STATUS status = grm::STATUS::FAILED; if ((status = midx_to_arr_(i, j, &idx)) != grm::STATUS::SUCCESS) return status; - data_[idx] = val; + data[idx] = val; return status; } -grm::STATUS grm::Grm::write(io::FileIO *fio, const Hdr *hdr) const { +grm::STATUS grm::write(io::FileIO *fio, + const grm::Hdr* hdr, const grm::Grm* grmatrix) const { + + if (!fio || !fio->fid || !hdr || !grm) + return grm::ERROR_NULLPTR_ARG; - std::unique_ptr fid = make_unique(fopen(filename, "wb")); + size_t nwritten = 0; - size_t size_written = fwrite(&dims_, sizeof(Dims), 1, fid.get()); - if (size_written < 1) - return grm::STATUS::ERROR_ON_WRITE; + // write file type specification + nwritten = fwrite(&FILE_TYPE_SPEC, + sizeof(FILE_TYPE_SPEC), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; + + grm::STATUS status = grm::UNSPECIFIED; + + // srite meta data stored in header; + if ((status = grm::write(fio, hdr)) != grm::SUCCESS) + return status; - size_written = fwrite(data_.get(), - sizeof(float), - size(), - fid.get()); + uint64_t ndata = static_cast(grmatrix->size()); + nwritten = fwrite(&ndata, sizeof(ndata), 1, fio->fid); + if (nwritten != 1) + return grm::ERROR_ON_WRITE; - if (size_written < size()) - return grm::STATUS::ERROR_ON_WRITE; + nwritten = fwrite(grmatrix->data.get(), + sizeof(grmatrix->data[0]), + ndata, + fio->fid); + if (static_cast(nwritten) != ndata) + return grm::ERROR_ON_WRITE; - return grm::STATUS::SUCCESS; + return grm::SUCCESS; } -grm::Grm grm::Grm::read(io::FileIO *fio) { - return grm::STATUS; +grm::STATUS grm::read(io::FileIO *fio, + grm::Hdr* hdr, grm::Grm* grmatrix) { + + if (!fio || !fio->fid || !hdr || !grmatrix) + return grm::ERROR_ON_READ; + + size_t nread = 0; + uint32_t ftype = 0; + + nread = fread(&ftype, sizeof(ftype), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + if (ftype != FILE_TYPE_SPEC) + return grm::ERROR_NOT_A_GRM_FILE; + + grm::STATUS status = grm::read(fio, hdr); + + uint64_t n_samples = hdr->Samples->len; + grm::Grm tmp_grm { n_samples }; + uint64_t ndata = 0; + nread = fread(&ndata, sizeof(ndata), 1, fio->fid); + if (nread != 1) + return grm::ERROR_ON_READ; + + if (ndata != n_samples*(n_samples + 1) / 2) + return grm::ERROR_ON_READ; + + nread = fread(tmp_grm.data.get(), + sizeof(tmp_grm.data[0]), ndata, fio->fid); + + if (nread != ndata) + return grm::ERROR_ON_READ; + + *grmatrix = std::move(tmp_grm); + + return grm::SUCCESS; } From 0f904434b972f8eb92405a2541f2f5d5720fe2a2 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:06:24 -0400 Subject: [PATCH 45/58] debug careless errors. --- include/grm.h | 19 ++++---- include/io.h | 4 +- src/grm.cpp | 127 ++++++++++++++++++++------------------------------ 3 files changed, 62 insertions(+), 88 deletions(-) diff --git a/include/grm.h b/include/grm.h index f93b520..b9aee0e 100644 --- a/include/grm.h +++ b/include/grm.h @@ -84,7 +84,7 @@ namespace grm { // a simple means to determine file type when parsing. constexpr uint32_t FILE_TYPE_SPEC = 0x47524D00; constexpr utils::Version FILE_VERSION = { 0, 0, 0 }; -constexpr char[] FILE_SUFFIX = ".grm"; +constexpr char FILE_SUFFIX[] = ".grm"; enum STATUS { @@ -119,8 +119,8 @@ struct Coordinates { len(len), pos(std::make_unique(len)) {}; - Coordinates(Coordinates&) = delete; - Coordinates& operator=(Coordinates&) = delete; + Coordinates(const Coordinates&) = delete; + Coordinates& operator=(const Coordinates&) = delete; Coordinates(Coordinates&& other); Coordinates& operator=(Coordinates&& other); @@ -188,8 +188,8 @@ STATUS read(io::FileIO* fio, Samples* samples); struct Hdr { Hdr(); - Hdr(Hdr&) = delete; - Hdr& operator=(Hdr&) = delete; + Hdr(const Hdr&) = delete; + Hdr& operator=(const Hdr&) = delete; Hdr(Hdr&&); Hdr& operator=(Hdr&&); @@ -244,12 +244,13 @@ struct Grm { float& operator()(const size_t i, const size_t j); // Checked indexes when setting and getting of matrix values - STATUS set(const size_t i, const size_t j, const float val); - STATUS get(const size_t i, const size_t j, float *val) const; + STATUS set(const uint64_t i, const uint64_t j, const float val); + STATUS get(const uint64_t i, const uint64_t j, float *val) const; - size_t size(); + uint64_t size() const; - STATUS midx_to_arr(const size_t i, const size_t j, size_t* idx) const; + STATUS midx_to_arr(const uint64_t i, + const uint64_t j, uint64_t* idx) const; uint64_t n_samples; std::unique_ptr data; diff --git a/include/io.h b/include/io.h index 9eed6c0..78513ca 100644 --- a/include/io.h +++ b/include/io.h @@ -1,11 +1,11 @@ +#ifndef HEADER_IO_H +#define HEADER_IO_H #include #include #include -#ifndef HEADER_TEXTIO_H -#define HEADER_TEXTIO_H namespace io { diff --git a/src/grm.cpp b/src/grm.cpp index a10e005..d7db829 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -60,7 +60,9 @@ //////////////////////////////////////////////////////////////////// grm::Coordinates::Coordinates(Coordinates&& other) - : len(other.len), contig(other.contig), pos(std::move(other.pos)) { + : len(other.len), + contig(std::move(other.contig)), + pos(std::move(other.pos)) { other.len = 0; other.contig = ""; @@ -71,10 +73,9 @@ grm::Coordinates& grm::Coordinates::operator=(Coordinates&& other) { return *this; len = other.len; - contig = other.contig; + contig = std::move(other.contig); pos = std::move(other.pos); - other.pos=nullptr; other.len = 0; other.contig = ""; @@ -82,15 +83,9 @@ grm::Coordinates& grm::Coordinates::operator=(Coordinates&& other) { } // remember that Coordinates* should be uninstantiated -grm::STATUS write(io::FileIO* fio, const Coordinates* coords) { +grm::STATUS grm::write(io::FileIO* fio, const Coordinates* coords) { - if (!fio) - return grm::ERROR_NULLPTR_ARG; - - if (!fio->fid) - return grm::ERROR_NULLPTR_ARG; - - if (!coords) + if (!fio || !fio->fid || !coords) return grm::ERROR_NULLPTR_ARG; size_t nwritten = 0; @@ -125,14 +120,9 @@ grm::STATUS write(io::FileIO* fio, const Coordinates* coords) { } -grm::STATUS read(io::FileIO* fio, Coordinates* coords) { - if (!fio) - return grm::ERROR_NULLPTR_ARG; +grm::STATUS grm::read(io::FileIO* fio, Coordinates* coords) { - if (!fio->fid) - return grm::ERROR_NULLPTR_ARG; - - if (!coords) + if (!fio || !fio->fid || !coords) return grm::ERROR_NULLPTR_ARG; // I create a temporary Coordinates class, because I don't want @@ -198,14 +188,9 @@ grm::Samples& grm::Samples::operator=(grm::Samples&& other) { } -grm::STATUS write(io::FileIO* fio, const grm::Samples* samples) { - if (!fio) - return grm::ERROR_NULLPTR_ARG; +grm::STATUS grm::write(io::FileIO* fio, const grm::Samples* samples) { - if (!fio->fid) - return grm::ERROR_NULLPTR_ARG; - - if (!samples) + if (!fio || !fio->fid || !samples) return grm::ERROR_NULLPTR_ARG; size_t nwritten = 0; @@ -252,14 +237,9 @@ grm::STATUS write(io::FileIO* fio, const grm::Samples* samples) { return grm::SUCCESS; } -grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { - if (!fio) - return grm::ERROR_NULLPTR_ARG; - - if (!fio->fid) - return grm::ERROR_NULLPTR_ARG; +grm::STATUS grm::read(io::FileIO* fio, grm::Samples* samples) { - if (!samples) + if (!fio || !fio->fid || !samples) return grm::ERROR_NULLPTR_ARG; // I create a temporary Sample class, because I don't want the @@ -274,7 +254,7 @@ grm::STATUS read(io::FileIO* fio, grm::Samples* samples) { return grm::ERROR_ON_READ; tmp_samps.len = n_samples; - tmp_samps.names = srd::make_unique(n_samples); + tmp_samps.names = std::make_unique(n_samples); // Get the number of characters of the longest string size_t nchar_max = 0; @@ -350,75 +330,60 @@ grm::Hdr& grm::Hdr::operator=(Hdr&& other) { } -grm::STATUS write(io::FileIO* fio, const Hdr* hdr) { - if (!fio) - return grm::ERROR_NULLPTR_ARG; - - if (!fio->fid) - return grm::ERROR_NULLPTR_ARG; +grm::STATUS grm::write(io::FileIO* fio, const Hdr* hdr) { - if (!hdr) + if (!fio || !fio->fid || !hdr) return grm::ERROR_NULLPTR_ARG; size_t nwritten = 0; - - size_t nchar = hdr->version.size(); - nwritten = fwrite(&nchar, sizeof(size_t), 1, fio->fid); + uint32_t tmp_version = hdr->prog_version.pack(); + nwritten = fwrite(&tmp_version, sizeof(tmp_version), 1, fid->fio); if (nwritten != 1) return grm::ERROR_ON_WRITE; - nwritten = fwrite(hdr->version.c_str(), - sizeof(char), nchar, fio->fid); - if (nwritten != nchar) + tmp_version = hdr->file_version.pack(); + nwritten = fwrite(&tmp_version, sizeof(tmp_version), 1, fid->fio); + if (nwritten != 1) return grm::ERROR_ON_WRITE; nwritten = fwrite(&hdr->grm_type, sizeof(GrmType), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; - grm::STATUS status; - if ((status = write(fio, hdr->coords)) != grm::SUCCESS) + grm::STATUS status = grm::FAILED; + if ((status = write(fio, hdr->coords.get())) != grm::SUCCESS) return status; - if ((status = write(fio, hdr->samples)) != grm::SUCCESS) + if ((status = write(fio, hdr->samples.get())) != grm::SUCCESS) return status; return grm::SUCCESS; } -grm::STATUS read(io::FileIO* fio, Hdr* hdr) { - if (!fio) - return grm::ERROR_NULLPTR_ARG; - - if (!fio->fid) - return grm::ERROR_NULLPTR_ARG; +grm::STATUS grm::read(io::FileIO* fio, Hdr* hdr) { - if (!hdr) + if (!fio || !fio->fid || !hdr) return grm::ERROR_NULLPTR_ARG; Hdr tmp_hdr {}; size_t nread = 0; - uint64_t nchar_version = 0; - - nread = fread(&nchar_version, sizeof(nchar_version), 1, fio->fid); + uint32_t tmp_version_num = 0; + nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fid->fio); if (nread != 1) return grm::ERROR_ON_READ; + tmp_hdr.prog_version = utils::Version::unpack(tmp_version_num); - std::unique_ptr buffer = std::make_unique(nchar_version + 1); - std::memset(buffer.get(), '\0', nchar_version + 1); - - nread = fread(buffer.get(), sizeof(char), nchar_version, fio->fid); - if (nread != nchar_version) + nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fid->fio); + if (nread != 1) return grm::ERROR_ON_READ; - - tmp_hdr.version = std::string(buffer, nchar_version); + tmp_hdr.file_version = utils::Version::unpack(tmp_version_num); nread = fread(&tmp_hdr.grm_type, sizeof(grm::GrmType), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; - grm::STATUS status; + grm::STATUS status = grm::FAILED; status = read(fio, tmp_hdr.coords.get()); if (status != grm::SUCCESS) return status; @@ -444,11 +409,11 @@ grm::STATUS read(io::FileIO* fio, Hdr* hdr) { grm::Grm::Grm(): n_samples(0), data(nullptr) {}; grm::Grm::Grm(uint64_t n_samps) - : n_samples(n_samps) + : n_samples(n_samps), data(size() != 0 ? std::make_unique(size()) : nullptr) { if (data) - std::memset(data.get(), 0, size()); + std::memset(data.get(), 0, size() * sizeof(float)); } grm::Grm::Grm(grm::Grm&& other) @@ -469,7 +434,9 @@ grm::Grm& grm::Grm::operator=(grm::Grm&& other) { } // The number of upper diagonal + diagonal elements of the GRM -uint64_t grm::Grm::size() const { return n_samples * (n_samples + 1) / 2; }; +uint64_t grm::Grm::size() const { + return n_samples * (n_samples + 1) / 2; +}; grm::STATUS grm::Grm::midx_to_arr(const uint64_t i, const uint64_t j, @@ -489,11 +456,15 @@ grm::STATUS grm::Grm::midx_to_arr(const uint64_t i, const uint64_t j, float grm::Grm::operator()(const uint64_t i, const uint64_t j) const { + if (i > j) + return data[MATRIX_IDX_TO_ARRAY(j, i, n_samples)]; return data[MATRIX_IDX_TO_ARRAY(i, j, n_samples)]; } float& grm::Grm::operator()(const uint64_t i, const uint64_t j) { + if (i > j) + return data[MATRIX_IDX_TO_ARRAY(j, i, n_samples)]; return data[MATRIX_IDX_TO_ARRAY(i, j, n_samples)]; } @@ -501,7 +472,7 @@ float& grm::Grm::operator()(const uint64_t i, const uint64_t j) { grm::STATUS grm::Grm::get(const uint64_t i, const uint64_t j, float *val) const { uint64_t idx = 0; grm::STATUS status = grm::FAILED; - if ((status = midx_to_arr_(i, j, &idx)) != grm::SUCCESS) + if ((status = midx_to_arr(i, j, &idx)) != grm::SUCCESS) return status; *val = data[idx]; @@ -512,8 +483,8 @@ grm::STATUS grm::Grm::get(const uint64_t i, const uint64_t j, float *val) const grm::STATUS grm::Grm::set(const uint64_t i, const uint64_t j, const float val) { uint64_t idx = 0; - grm::STATUS status = grm::STATUS::FAILED; - if ((status = midx_to_arr_(i, j, &idx)) != grm::STATUS::SUCCESS) + grm::STATUS status = grm::FAILED; + if ((status = midx_to_arr(i, j, &idx)) != grm::SUCCESS) return status; data[idx] = val; @@ -523,9 +494,9 @@ grm::STATUS grm::Grm::set(const uint64_t i, const uint64_t j, const float val) { grm::STATUS grm::write(io::FileIO *fio, - const grm::Hdr* hdr, const grm::Grm* grmatrix) const { + const grm::Hdr* hdr, const grm::Grm* grmatrix) { - if (!fio || !fio->fid || !hdr || !grm) + if (!fio || !fio->fid || !hdr || !grmatrix) return grm::ERROR_NULLPTR_ARG; size_t nwritten = 0; @@ -536,7 +507,7 @@ grm::STATUS grm::write(io::FileIO *fio, if (nwritten != 1) return grm::ERROR_ON_WRITE; - grm::STATUS status = grm::UNSPECIFIED; + grm::STATUS status = grm::FAILED; // srite meta data stored in header; if ((status = grm::write(fio, hdr)) != grm::SUCCESS) @@ -575,8 +546,10 @@ grm::STATUS grm::read(io::FileIO *fio, return grm::ERROR_NOT_A_GRM_FILE; grm::STATUS status = grm::read(fio, hdr); + if (status != grm::SUCCESS) + return status; - uint64_t n_samples = hdr->Samples->len; + uint64_t n_samples = hdr->samples->len; grm::Grm tmp_grm { n_samples }; uint64_t ndata = 0; nread = fread(&ndata, sizeof(ndata), 1, fio->fid); From 2f07e7eb5c3e8d74073344999f0b944212f52536 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 19 Mar 2026 22:24:20 -0400 Subject: [PATCH 46/58] debugging more careless errors. --- include/grm.h | 6 +++--- include/utils.h | 2 +- src/grm.cpp | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/grm.h b/include/grm.h index b9aee0e..71a80fe 100644 --- a/include/grm.h +++ b/include/grm.h @@ -240,8 +240,8 @@ struct Grm { Grm& operator=(Grm&&); // Unchecked indexes when setting and getting of matrix values - float operator()(const size_t i, const size_t j) const; - float& operator()(const size_t i, const size_t j); + float operator()(const uint64_t i, const uint64_t j) const; + float& operator()(const uint64_t i, const uint64_t j); // Checked indexes when setting and getting of matrix values STATUS set(const uint64_t i, const uint64_t j, const float val); @@ -266,7 +266,7 @@ struct Grm { // @param hdr: an instance of grm::Hdr with important meta data // @return grm::STATUS: // -STATUS write(io::FileIO *fio, const Hdr *hdr, const Grm *grmatrix) const; +STATUS write(io::FileIO *fio, const Hdr *hdr, const Grm *grmatrix); STATUS read(io::FileIO *fio, const Hdr *hdr, Grm *grmatrix); diff --git a/include/utils.h b/include/utils.h index bb4e698..adf574d 100644 --- a/include/utils.h +++ b/include/utils.h @@ -13,7 +13,7 @@ struct Version { uint16_t micro; // pack as 8, 12, 12 for 32 bits in total - uint32_t pack() { + uint32_t pack() const { return (static_cast(major) << 24 | static_cast(minor & 0x0FFF) << 12 | static_cast(micro & 0x0FFF)); diff --git a/src/grm.cpp b/src/grm.cpp index d7db829..100abf9 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -337,12 +337,12 @@ grm::STATUS grm::write(io::FileIO* fio, const Hdr* hdr) { size_t nwritten = 0; uint32_t tmp_version = hdr->prog_version.pack(); - nwritten = fwrite(&tmp_version, sizeof(tmp_version), 1, fid->fio); + nwritten = fwrite(&tmp_version, sizeof(tmp_version), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; tmp_version = hdr->file_version.pack(); - nwritten = fwrite(&tmp_version, sizeof(tmp_version), 1, fid->fio); + nwritten = fwrite(&tmp_version, sizeof(tmp_version), 1, fio->fid); if (nwritten != 1) return grm::ERROR_ON_WRITE; @@ -369,12 +369,12 @@ grm::STATUS grm::read(io::FileIO* fio, Hdr* hdr) { size_t nread = 0; uint32_t tmp_version_num = 0; - nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fid->fio); + nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; tmp_hdr.prog_version = utils::Version::unpack(tmp_version_num); - nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fid->fio); + nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; tmp_hdr.file_version = utils::Version::unpack(tmp_version_num); From b4ef2c14665a92aa76d5e6bca1edb2b8210fee5a Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 20 Mar 2026 15:17:40 -0400 Subject: [PATCH 47/58] debug grm io so that it compiles, and started writing unit tests. --- include/grm.h | 13 +++++----- include/io.h | 19 +++++++-------- src/grm.cpp | 60 +++++++--------------------------------------- tests/test_grm.cpp | 46 +++++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 68 deletions(-) create mode 100644 tests/test_grm.cpp diff --git a/include/grm.h b/include/grm.h index 71a80fe..c02b7b8 100644 --- a/include/grm.h +++ b/include/grm.h @@ -41,6 +41,7 @@ #include #include +#include "io.h" #include "constants.h" #include "utils.h" @@ -114,10 +115,10 @@ enum GrmType { // @title: Store genomic coordinates used in GRM calculation struct Coordinates { Coordinates(): contig(""), len(0), pos(nullptr) {}; - Coordinates(const char* contig, const size_t len) - : contig(contig), - len(len), - pos(std::make_unique(len)) {}; + Coordinates(char* contig_in, uint64_t len_in) + : contig(contig_in == nullptr ? "" : contig_in), + len(contig == "" ? 0 : len_in), + pos(std::make_unique(len)) {}; Coordinates(const Coordinates&) = delete; Coordinates& operator=(const Coordinates&) = delete; @@ -266,8 +267,8 @@ struct Grm { // @param hdr: an instance of grm::Hdr with important meta data // @return grm::STATUS: // -STATUS write(io::FileIO *fio, const Hdr *hdr, const Grm *grmatrix); -STATUS read(io::FileIO *fio, const Hdr *hdr, Grm *grmatrix); +STATUS write(io::FileIO* fio, const Hdr *hdr, const Grm *grmatrix); +STATUS read(io::FileIO* fio, Hdr* hdr, Grm* grmatrix); } diff --git a/include/io.h b/include/io.h index 78513ca..8b9eed8 100644 --- a/include/io.h +++ b/include/io.h @@ -5,9 +5,6 @@ #include #include - - - namespace io { enum STATUS { @@ -78,12 +75,12 @@ FileIO open(const char *filename, const char *mode) { // @title: File statistics // @description: This object is returned by any function meant to calculate // file character statistics. -struct FileStats { - size_t nchar = 0; - size_t nwords = 0; - size_t nlines = 0; - size_t nblanklines = 0; -} +// struct FileStats { +// size_t nchar = 0; +// size_t nwords = 0; +// size_t nlines = 0; +// size_t nblanklines = 0; +// } // @title: word count @@ -94,12 +91,12 @@ struct FileStats { // @param fs: the structure that the file statistics will be stored // @return a STATUS code that specifies whether the function was successful // or failed. -STATUS wc(TextIO *tio, FileStats *fs); +// STATUS wc(TextIO *tio, FileStats *fs); -STATUS getline(TextIO *tio, Array linebuf); +// STATUS getline(TextIO *tio, Array linebuf); } #endif diff --git a/src/grm.cpp b/src/grm.cpp index 100abf9..02cb008 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -9,51 +9,8 @@ // // -#include - -// int grm::details::num_lines_in_file(FILE *fid, size_t *num_lines) { -// // TODO: errno, need to reset? -// -// size_t line_num = 0; -// size_t word_len = 0; -// int c; -// while ((c = fgetc(fid)) != EOF) { -// -// if (c == '\n' && word_len != 0) { -// line_num++; -// word_len = 0; -// } else if (c != '\n') -// word_len++; -// } -// -// if (ferror(fid)) -// return fseek(fid, 0, SEEK_SET) == 0 ? -1 : -3; -// -// if (feof(fid) == 0) -// return fseek(fid, 0, SEEK_SET) == 0 ? -2 : -3; -// -// *num_lines = line_num; -// return fseek(fid, 0, SEEK_SET) == 0 ? 0 : -3; -// } - - -// int grm::details::get_size_t(FILE *fid, size_t *val) { -// -// std::string s { "" }; -// while (std::getline(fid, s)) -// if (s.size() < ) -// -// int c; -// for (int i = 0; (c = fgetc(fid)) != EOF && i < max_bitsize_size_t; i++) { -// if (c == '\n') -// break; -// s[i] = c; -// } -// s[i] = '\0'; -// -// return 0; -// } -// +#include "grm.h" + //////////////////////////////////////////////////////////////////// // COORDINATES CLASS @@ -145,7 +102,7 @@ grm::STATUS grm::read(io::FileIO* fio, Coordinates* coords) { if (static_cast(nread) != size_contig_name) return grm::ERROR_ON_READ; - tmpc.contig = std::string(buffer, size_contig_name); + tmpc.contig = std::string(buffer.get(), size_contig_name); // read in positions uint64_t npos = 0; @@ -266,9 +223,10 @@ grm::STATUS grm::read(io::FileIO* fio, grm::Samples* samples) { std::unique_ptr buffer = std::make_unique(nchar_max+1); std::memset(buffer.get(), '\0', nchar_max + 1); - size_t nchar = 0; - for (size_t n = 0; n < n_samples; n++) { - nread = fread(&nchar, sizeof(size_t), 1, fio->fid); + // read sample names + uint64_t nchar = 0; + for (uint64_t n = 0; n < n_samples; n++) { + nread = fread(&nchar, sizeof(nchar), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; @@ -276,7 +234,7 @@ grm::STATUS grm::read(io::FileIO* fio, grm::Samples* samples) { if (static_cast(nread) != nchar) return grm::ERROR_ON_READ; - tmp_samps.names[n] = std::string(buffer, nchar); + tmp_samps.names[n] = std::string(buffer.get(), nchar); std::memset(buffer.get(), '\0', nchar); nchar = 0; @@ -529,7 +487,7 @@ grm::STATUS grm::write(io::FileIO *fio, } -grm::STATUS grm::read(io::FileIO *fio, +grm::STATUS grm::read(io::FileIO* fio, grm::Hdr* hdr, grm::Grm* grmatrix) { if (!fio || !fio->fid || !hdr || !grmatrix) diff --git a/tests/test_grm.cpp b/tests/test_grm.cpp new file mode 100644 index 0000000..71a9e80 --- /dev/null +++ b/tests/test_grm.cpp @@ -0,0 +1,46 @@ + +#include +#include +#include +#include + + + +TEST(TestCoords, DefaultConstructor) { + // verify default values + grm::Coordinates coords {}; + + std::string contig = std::string(""); + + EXPECT_EQ(coords.contig.size(), 0); + EXPECT_EQ(coords.contig, contig); + + EXPECT_EQ(coords.len, 0); + EXPECT_EQ(coords.pos, nullptr); +} + + +TEST(TestCoords, ConstructorValidInput) { + char contig_in[] = "chr12"; + std::string contig { contig_in }; + uint64_t len_in = 1000; + + grm::Coordinates coords { contig_in, len }; + + EXPECT_EQ(coords.contig, contig); + EXPECT_EQ(coords.len, len_in); + EXPECT_NE(coords.pos, nullptr); +} + +// If input contig name is nullptr, resort to the default values +// of the default constructor. +TEST(TestCoords, ConstructorInvalidInput) { + uint64_t len_in = 10; + grm::Coordinates coords { nullptr, len_in }; + + EXPECT_EQ(coords.contig, std::string("")); + EXPECT_EQ(coords.len, 0); + EXPECT_EQ(coords.pos, nullptr); +} + + From 5935161284b3d336d3d282740a655d3d91682a05 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 20 Mar 2026 20:20:39 -0400 Subject: [PATCH 48/58] Aesthetic changes to comments. --- src/bcfio.cpp | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 76620ff..113f219 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -11,9 +11,9 @@ #include -// ***************************************************************************** -// class BcfHeader -// ***************************************************************************** +/////////////////////////////////////////////////////////////////// +// BcfHeader +/////////////////////////////////////////////////////////////////// // int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, const int bcf_dt_type, @@ -77,9 +77,9 @@ const std::unique_ptr bcfio::BcfHeader::sample_names() const { return samp_names; } -// ***************************************************************************** -// class BcfFloatRecord -// ***************************************************************************** +/////////////////////////////////////////////////////////////////// +// BcfFloatRecord +/////////////////////////////////////////////////////////////////// bcfio::BcfFloatRecord::~BcfFloatRecord() { if (rec_) htslib::bcf_destroy(rec_); @@ -122,9 +122,10 @@ int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { } -// ***************************************************************************** -// class BcfRead -// ***************************************************************************** +/////////////////////////////////////////////////////////////////// +// BcfRead +/////////////////////////////////////////////////////////////////// +/// bcfio::ReadBcf::ReadBcf(const char *bcfname) : fname_(bcfname), fid_(htslib::hts_open(bcfname, "r")), From 4deca00791c4b3e6affb73d6d94851a4df2f7f79 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 20 Mar 2026 20:48:03 -0400 Subject: [PATCH 49/58] debug, with invalid input I wasn't setting the Coordinates.pos unique_ptr to nullptr, fixed. minor bugs in test code and io.h have been resolve. --- include/grm.h | 2 +- include/io.h | 4 ++-- src/grm.cpp | 4 ++-- tests/test_grm.cpp | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/grm.h b/include/grm.h index c02b7b8..35a6222 100644 --- a/include/grm.h +++ b/include/grm.h @@ -118,7 +118,7 @@ struct Coordinates { Coordinates(char* contig_in, uint64_t len_in) : contig(contig_in == nullptr ? "" : contig_in), len(contig == "" ? 0 : len_in), - pos(std::make_unique(len)) {}; + pos(len == 0 ? nullptr : std::make_unique(len)) {}; Coordinates(const Coordinates&) = delete; Coordinates& operator=(const Coordinates&) = delete; diff --git a/include/io.h b/include/io.h index 8b9eed8..fd9d9bf 100644 --- a/include/io.h +++ b/include/io.h @@ -58,8 +58,8 @@ FileIO open(const char *filename, const char *mode) { if (!fid) return nullptr; - FileIO fio = FileIO(fid); - return std::move(fio); + // compiler implements elision + return FileIO(fid); } diff --git a/src/grm.cpp b/src/grm.cpp index 02cb008..13dd5a2 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -17,8 +17,8 @@ //////////////////////////////////////////////////////////////////// grm::Coordinates::Coordinates(Coordinates&& other) - : len(other.len), - contig(std::move(other.contig)), + : contig(std::move(other.contig)), + len(other.len), pos(std::move(other.pos)) { other.len = 0; diff --git a/tests/test_grm.cpp b/tests/test_grm.cpp index 71a9e80..e1ef979 100644 --- a/tests/test_grm.cpp +++ b/tests/test_grm.cpp @@ -25,7 +25,7 @@ TEST(TestCoords, ConstructorValidInput) { std::string contig { contig_in }; uint64_t len_in = 1000; - grm::Coordinates coords { contig_in, len }; + grm::Coordinates coords { contig_in, len_in }; EXPECT_EQ(coords.contig, contig); EXPECT_EQ(coords.len, len_in); From 0809300541a327a0eb2b691937a18fd3bb9ec0bb Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Fri, 20 Mar 2026 20:50:19 -0400 Subject: [PATCH 50/58] only header file, therefore delete the io.cpp. --- src/io.cpp | 120 ----------------------------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 src/io.cpp diff --git a/src/io.cpp b/src/io.cpp deleted file mode 100644 index 6a91f5b..0000000 --- a/src/io.cpp +++ /dev/null @@ -1,120 +0,0 @@ - -#include - - - - -// textio::TextIO::TextIO(FILE *fileid): fid(fileid) {}; -// -// -// -// -// std::unique_ptr textio::open(const char *filename, -// const char *mode) { -// -// FILE *fid = fopen(filename, mode); -// if (ferror(fid)) { -// fclose(fid); -// return nullptr; -// } -// -// std::unique_ptr tio = std::make_unique(fid); -// -// return std::move(tio); -// } -// -// -// textio::STATUS wc(textio::TextIO *tio, textio::FileStats *fs) { -// if (!fs) -// return textio::INVALID_ARG_ERROR; -// -// size_t nchar = 0; -// size_t nwords = 0; -// size_t nlines = 0; -// size_t nblanklines = 0; -// -// size_t word_len = 0; -// -// FILE *fid = tio->fid; -// -// int c = '\0'; -// while ((c = fgetc(fid)) != EOF) { -// -// switch (c) { -// case '\n': -// nlines++; -// -// if (word_len == 0) -// nblanklines++; -// else { -// nwords++; -// word_len = 0; -// } -// break; -// case ';': -// case ':': -// case ',': -// case '!': -// case '?': -// case '(': -// case ')': -// case '\"': -// case '\t': -// case ' ': -// if (word_len == 0) -// break; -// -// nwords++; -// word_len = 0; -// -// break; -// default: -// nchar++; -// word_len++; -// } -// -// } -// -// if (ferror(fid)) { -// fs = nullptr; -// return tio->bseek() == 0 ? textio::FERROR : textio::FSEEK_ERROR; -// } -// -// if (feof(fid) == 0) { -// fs = nullptr; -// return tio->bseek() == 0 ? textio::FEOF_ERROR : textio::FSEEK_ERROR; -// } -// -// fs->nchar = nchar; -// fs->nwords = nwords; -// fs->nlines = nlines; -// fs->nblanklines = nblanklines; -// -// return tio->bseek() ? textio::SUCCESS : textio::FSEEK_ERROR; -// } -// -// -// -// textio::STATUS textio::getline(textio::TextIO *tio, textio::Array *buf) { -// buf->fill('\0'); -// -// FILE *fid = buf->fid; -// -// int c = 0; -// while ((c = fgetc(fid)) != EOF) { -// -// if (c == '\n') { -// buf->append('\0'); -// return textio::SUCCESS; -// -// buf->append(c) -// } -// -// if (ferror(fid)) -// return textio::FERROR; -// -// if (feof(fid) == 0) -// return textio::FEOF_ERROR; -// -// return textio::SUCCESS; -// } From 5ea3fc83dbb1355e20e25bf2789e745e8d5a0f3d Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 21 Mar 2026 11:31:54 -0400 Subject: [PATCH 51/58] I fixed an error in the macro that transorms matrix row and col numbers to idx of the linear array. --- include/grm.h | 43 +++++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/include/grm.h b/include/grm.h index 35a6222..79d8866 100644 --- a/include/grm.h +++ b/include/grm.h @@ -49,32 +49,39 @@ // The algorithm for getting the array idx from matrix indexes is simply // -// i * n_samples - n_skipped_idxs + j +// idx = i * n_samples - n_skipped_idxs + j // -// where i is the matrix row index and j is the matrix column index. +// where i is the matrix row index and j is the matrix column index. The // interesting term is n_skipped_idxs, this is the number of elements // that referencing (i, j) skip when only storing upper triangle. For -// example, consider the following table with matrix to array indexes +// example, suppose that i = 3 and j = 2. Here three complete rows of +// the matrix has been traversed, therefore the number skipped is // -// i j num_skipped idx -// 0 0 0 0 -// 0 5 0 5 -// 1 0 0 1n - 0 -// 2 0 1 2n - 1 -// 3 0 3 3n - 3 -// 4 0 6 4n - 6 -// -// we see that number skipped is the number of lower triangular elements -// of a matrix constructed from i rows, (i-1) * i / 2. Here, we see an -// obvious problem, that when i = 0 we get a negative number, which doesn't -// make sense. This can be avoided by using the equivalent formulat +// n_skipped_idx >= (i-1) * i / 2 +// +// if j < i, then as we are only storing the upper triangle and +// by symmetry swap the values of i an j. If j > i, then the number +// skipped in row i is i, making the total skipped, // -// n_skipped_idxs = i * (i + 1) / 2 - i +// n_skipped_idx = i * (i - 1) / 2 + i // // making the equation above read // -// i * (n_samples + 1) - i * (i+1)/2 + j -#define MATRIX_IDX_TO_ARRAY(i, j, n) ((i)*((n) + 1) - (i)*((i)+1)/2 + (j)) +// idx = i * n_samples - i * (i - 1)/2 - i + j +// +// Then let's manually validate +// +// n = 3 +// i j idx_true idx +// 0 0 0 0 +// 0 1 1 1 +// 0 2 2 2 +// 1 0 --- transpose -> 0, 1 --- +// 1 1 3 1*3 - 1*(0)/2 - 1 + 1 = 3 +// 1 2 4 1*3 - 1*(0)/2 - 1 + 2 = 4 +// 2 2 5 2*3 - 2*(1)/2 - 2 + 2 = 6 - 1 = 5 +// +#define MATRIX_IDX_TO_ARRAY(i, j, n) ((i)*(n) - (i)*((i)-1)/2 + (j) - (i)) namespace grm { From 33a333c49c96427c05719babb4fc7ebe141ce142 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 21 Mar 2026 15:24:46 -0400 Subject: [PATCH 52/58] Fixed nullptr dereference by making the deefault constructor of grm::Hdr to call data member default constructors instead of setting to nullptr. --- src/grm.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/grm.cpp b/src/grm.cpp index 13dd5a2..3e47411 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -252,7 +252,9 @@ grm::STATUS grm::read(io::FileIO* fio, grm::Samples* samples) { grm::Hdr::Hdr() : prog_version(constants::PROG_VERSION), file_version(grm::FILE_VERSION), - grm_type(UNSPECIFIED), coords(nullptr), samples(nullptr) {}; + grm_type(UNSPECIFIED), + coords(std::make_unique()), + samples(std::make_unique()) {}; grm::Hdr::Hdr(Hdr&& other) @@ -326,16 +328,16 @@ grm::STATUS grm::read(io::FileIO* fio, Hdr* hdr) { Hdr tmp_hdr {}; size_t nread = 0; - uint32_t tmp_version_num = 0; - nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fio->fid); + uint32_t version = 0; + nread = fread(&version, sizeof(version), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; - tmp_hdr.prog_version = utils::Version::unpack(tmp_version_num); + tmp_hdr.prog_version = utils::Version::unpack(version); - nread = fread(&tmp_version_num, sizeof(tmp_version_num), 1, fio->fid); + nread = fread(&version, sizeof(version), 1, fio->fid); if (nread != 1) return grm::ERROR_ON_READ; - tmp_hdr.file_version = utils::Version::unpack(tmp_version_num); + tmp_hdr.file_version = utils::Version::unpack(version); nread = fread(&tmp_hdr.grm_type, sizeof(grm::GrmType), 1, fio->fid); if (nread != 1) From df85b14bc102a041d548c18ebb2e24d0f2b08ba9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 21 Mar 2026 19:34:52 +0000 Subject: [PATCH 53/58] Add grm unit tests exposing bug in grm::read null-arg error code Add 42 unit tests covering Coordinates, Samples, Hdr, Grm struct, MATRIX_IDX_TO_ARRAY macro, and full GRM file I/O. Tests can be built and run standalone via `make test_grm`. TestGrmFile.ReadNullArgs FAILS: grm::read(FileIO*, Hdr*, Grm*) at src/grm.cpp:496 returns ERROR_ON_READ for null pointer arguments, but the corresponding write function returns ERROR_NULLPTR_ARG. Also fixes: add missing `inline` on io::open() in include/io.h to prevent multiple-definition linker errors when included from multiple translation units. https://claude.ai/code/session_01FXkdgU8L4ws7LoCu16wG4s --- Makefile | 7 + include/io.h | 2 +- tests/test_grm.cpp | 712 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 720 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 67ea8d0..05518f0 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,13 @@ $(TEST_TARGET_PRG): $(TEST_DIR)/main.cpp $(TEST_OBJS) $(APP_OBJS) | $(TARGET) $(BUILD_DIR)/test_%.o: $(TEST_DIR)/test_%.cpp $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(OBJ_OUTPUT_OPTIONS) $< +TEST_GRM_PRG = $(BUILD_DIR)/test_grm +test_grm: $(TEST_GRM_PRG) + ./$(TEST_GRM_PRG) + +$(TEST_GRM_PRG): $(TEST_DIR)/main.cpp $(BUILD_DIR)/test_grm.o $(BUILD_DIR)/grm.o | $(BUILD_DIR) + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -lgtest + data: | $(TEST_DATA_DST) $(BUILD_DIR)/geno_test_data%: $(TEST_DIR)/geno_test_data% diff --git a/include/io.h b/include/io.h index fd9d9bf..0d1b7fd 100644 --- a/include/io.h +++ b/include/io.h @@ -50,7 +50,7 @@ struct FileIO { // @param filename: name and path of file to open // @param mode: a mode in the set of those in the C library function fopen // @return a pointer to opened file -FileIO open(const char *filename, const char *mode) { +inline FileIO open(const char *filename, const char *mode) { if (!mode || !filename) return nullptr; diff --git a/tests/test_grm.cpp b/tests/test_grm.cpp index e1ef979..8fd9e0f 100644 --- a/tests/test_grm.cpp +++ b/tests/test_grm.cpp @@ -1,10 +1,14 @@ #include #include +#include #include #include +//////////////////////////////////////////////////////////////////// +// COORDINATES TESTS +//////////////////////////////////////////////////////////////////// TEST(TestCoords, DefaultConstructor) { // verify default values @@ -44,3 +48,711 @@ TEST(TestCoords, ConstructorInvalidInput) { } +TEST(TestCoords, ConstructorZeroLength) { + char contig_in[] = "chr1"; + grm::Coordinates coords { contig_in, 0 }; + + EXPECT_EQ(coords.contig, std::string("chr1")); + EXPECT_EQ(coords.len, 0); + EXPECT_EQ(coords.pos, nullptr); +} + + +TEST(TestCoords, MoveConstructor) { + char contig_in[] = "chr7"; + uint64_t len_in = 5; + grm::Coordinates src { contig_in, len_in }; + + // set positions to known values + for (uint64_t i = 0; i < len_in; i++) + src.pos[i] = (i + 1) * 100; + + grm::Coordinates dst { std::move(src) }; + + // destination should have source's data + EXPECT_EQ(dst.contig, std::string("chr7")); + EXPECT_EQ(dst.len, len_in); + EXPECT_NE(dst.pos, nullptr); + for (uint64_t i = 0; i < len_in; i++) + EXPECT_EQ(dst.pos[i], (i + 1) * 100); + + // source should be in moved-from state + EXPECT_EQ(src.contig, std::string("")); + EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.pos, nullptr); +} + + +TEST(TestCoords, MoveAssignment) { + char contig_in[] = "chr3"; + uint64_t len_in = 3; + grm::Coordinates src { contig_in, len_in }; + src.pos[0] = 10; + src.pos[1] = 20; + src.pos[2] = 30; + + grm::Coordinates dst {}; + dst = std::move(src); + + EXPECT_EQ(dst.contig, std::string("chr3")); + EXPECT_EQ(dst.len, 3); + EXPECT_NE(dst.pos, nullptr); + EXPECT_EQ(dst.pos[0], 10); + EXPECT_EQ(dst.pos[1], 20); + EXPECT_EQ(dst.pos[2], 30); + + EXPECT_EQ(src.contig, std::string("")); + EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.pos, nullptr); +} + + +TEST(TestCoords, WriteReadRoundTrip) { + char contig_in[] = "chr22"; + uint64_t len_in = 4; + grm::Coordinates src { contig_in, len_in }; + src.pos[0] = 100; + src.pos[1] = 200; + src.pos[2] = 500; + src.pos[3] = 1000; + + // write to a temporary file + io::FileIO fio_w { tmpfile() }; + ASSERT_NE(fio_w.fid, nullptr); + + grm::STATUS status = grm::write(&fio_w, &src); + ASSERT_EQ(status, grm::SUCCESS); + + // rewind and read back + rewind(fio_w.fid); + grm::Coordinates dst {}; + status = grm::read(&fio_w, &dst); + ASSERT_EQ(status, grm::SUCCESS); + + EXPECT_EQ(dst.contig, std::string("chr22")); + EXPECT_EQ(dst.len, len_in); + ASSERT_NE(dst.pos, nullptr); + for (uint64_t i = 0; i < len_in; i++) + EXPECT_EQ(dst.pos[i], src.pos[i]); +} + + +TEST(TestCoords, WriteNullArgs) { + grm::Coordinates coords {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::write(nullptr, &coords), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::write(&fio, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestCoords, ReadNullArgs) { + grm::Coordinates coords {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::read(nullptr, &coords), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::read(&fio, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +//////////////////////////////////////////////////////////////////// +// SAMPLES TESTS +//////////////////////////////////////////////////////////////////// + +TEST(TestSamples, DefaultConstructor) { + grm::Samples samps {}; + + EXPECT_EQ(samps.len, 0); + EXPECT_EQ(samps.names, nullptr); +} + + +TEST(TestSamples, ConstructorValidInput) { + uint64_t n = 5; + grm::Samples samps { n }; + + EXPECT_EQ(samps.len, n); + EXPECT_NE(samps.names, nullptr); +} + + +TEST(TestSamples, ConstructorZero) { + grm::Samples samps { 0 }; + + EXPECT_EQ(samps.len, 0); + EXPECT_EQ(samps.names, nullptr); +} + + +TEST(TestSamples, MoveConstructor) { + grm::Samples src { 3 }; + src.names[0] = "sample_A"; + src.names[1] = "sample_B"; + src.names[2] = "sample_C"; + + grm::Samples dst { std::move(src) }; + + EXPECT_EQ(dst.len, 3); + ASSERT_NE(dst.names, nullptr); + EXPECT_EQ(dst.names[0], "sample_A"); + EXPECT_EQ(dst.names[1], "sample_B"); + EXPECT_EQ(dst.names[2], "sample_C"); + + EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.names, nullptr); +} + + +TEST(TestSamples, MoveAssignment) { + grm::Samples src { 2 }; + src.names[0] = "id_1"; + src.names[1] = "id_2"; + + grm::Samples dst {}; + dst = std::move(src); + + EXPECT_EQ(dst.len, 2); + ASSERT_NE(dst.names, nullptr); + EXPECT_EQ(dst.names[0], "id_1"); + EXPECT_EQ(dst.names[1], "id_2"); + + EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.names, nullptr); +} + + +TEST(TestSamples, WriteReadRoundTrip) { + grm::Samples src { 3 }; + src.names[0] = "alpha"; + src.names[1] = "beta"; + src.names[2] = "gamma"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + grm::STATUS status = grm::write(&fio, &src); + ASSERT_EQ(status, grm::SUCCESS); + + rewind(fio.fid); + + grm::Samples dst {}; + status = grm::read(&fio, &dst); + ASSERT_EQ(status, grm::SUCCESS); + + EXPECT_EQ(dst.len, 3); + ASSERT_NE(dst.names, nullptr); + EXPECT_EQ(dst.names[0], "alpha"); + EXPECT_EQ(dst.names[1], "beta"); + EXPECT_EQ(dst.names[2], "gamma"); +} + + +TEST(TestSamples, WriteReadVaryingLengthNames) { + grm::Samples src { 3 }; + src.names[0] = "a"; + src.names[1] = "longer_sample_name"; + src.names[2] = "xy"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + rewind(fio.fid); + + grm::Samples dst {}; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.len, 3); + ASSERT_NE(dst.names, nullptr); + EXPECT_EQ(dst.names[0], "a"); + EXPECT_EQ(dst.names[1], "longer_sample_name"); + EXPECT_EQ(dst.names[2], "xy"); +} + + +TEST(TestSamples, WriteNullArgs) { + grm::Samples samps {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::write(nullptr, &samps), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::write(&fio, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestSamples, ReadNullArgs) { + grm::Samples samps {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::read(nullptr, &samps), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::read(&fio, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +//////////////////////////////////////////////////////////////////// +// HDR TESTS +//////////////////////////////////////////////////////////////////// + +TEST(TestHdr, DefaultConstructor) { + grm::Hdr hdr {}; + + EXPECT_EQ(hdr.prog_version.major, constants::PROG_VERSION.major); + EXPECT_EQ(hdr.prog_version.minor, constants::PROG_VERSION.minor); + EXPECT_EQ(hdr.prog_version.micro, constants::PROG_VERSION.micro); + + EXPECT_EQ(hdr.file_version.major, grm::FILE_VERSION.major); + EXPECT_EQ(hdr.file_version.minor, grm::FILE_VERSION.minor); + EXPECT_EQ(hdr.file_version.micro, grm::FILE_VERSION.micro); + + EXPECT_EQ(hdr.grm_type, grm::UNSPECIFIED); + EXPECT_NE(hdr.coords, nullptr); + EXPECT_NE(hdr.samples, nullptr); +} + + +TEST(TestHdr, MoveConstructor) { + grm::Hdr src {}; + src.grm_type = grm::EHC; + src.coords->contig = "chr1"; + + utils::Version saved_prog = src.prog_version; + utils::Version saved_file = src.file_version; + + grm::Hdr dst { std::move(src) }; + + EXPECT_EQ(dst.grm_type, grm::EHC); + EXPECT_EQ(dst.prog_version.major, saved_prog.major); + EXPECT_EQ(dst.file_version.major, saved_file.major); + EXPECT_NE(dst.coords, nullptr); + EXPECT_EQ(dst.coords->contig, "chr1"); + + // source should be reset + EXPECT_EQ(src.grm_type, grm::UNSPECIFIED); + EXPECT_EQ(src.coords, nullptr); + EXPECT_EQ(src.samples, nullptr); +} + + +TEST(TestHdr, MoveAssignment) { + grm::Hdr src {}; + src.grm_type = grm::DS; + + grm::Hdr dst {}; + dst = std::move(src); + + EXPECT_EQ(dst.grm_type, grm::DS); + EXPECT_NE(dst.coords, nullptr); + EXPECT_NE(dst.samples, nullptr); + + EXPECT_EQ(src.grm_type, grm::UNSPECIFIED); + EXPECT_EQ(src.coords, nullptr); + EXPECT_EQ(src.samples, nullptr); +} + + +TEST(TestHdr, WriteReadRoundTrip) { + grm::Hdr src {}; + src.grm_type = grm::EAC; + + // set up coordinates + char contig[] = "chr5"; + *src.coords = grm::Coordinates { contig, 3 }; + src.coords->pos[0] = 100; + src.coords->pos[1] = 200; + src.coords->pos[2] = 300; + + // set up samples + *src.samples = grm::Samples { 2 }; + src.samples->names[0] = "samp1"; + src.samples->names[1] = "samp2"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + + rewind(fio.fid); + + grm::Hdr dst {}; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.prog_version.major, src.prog_version.major); + EXPECT_EQ(dst.prog_version.minor, src.prog_version.minor); + EXPECT_EQ(dst.prog_version.micro, src.prog_version.micro); + EXPECT_EQ(dst.file_version.major, src.file_version.major); + EXPECT_EQ(dst.grm_type, grm::EAC); + + ASSERT_NE(dst.coords, nullptr); + EXPECT_EQ(dst.coords->contig, "chr5"); + EXPECT_EQ(dst.coords->len, 3); + EXPECT_EQ(dst.coords->pos[0], 100); + EXPECT_EQ(dst.coords->pos[1], 200); + EXPECT_EQ(dst.coords->pos[2], 300); + + ASSERT_NE(dst.samples, nullptr); + EXPECT_EQ(dst.samples->len, 2); + EXPECT_EQ(dst.samples->names[0], "samp1"); + EXPECT_EQ(dst.samples->names[1], "samp2"); +} + + +TEST(TestHdr, WriteNullArgs) { + grm::Hdr hdr {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::write(nullptr, &hdr), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::write(&fio, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestHdr, ReadNullArgs) { + grm::Hdr hdr {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::read(nullptr, &hdr), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::read(&fio, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +//////////////////////////////////////////////////////////////////// +// GRM STRUCT TESTS +//////////////////////////////////////////////////////////////////// + +TEST(TestGrm, DefaultConstructor) { + grm::Grm g {}; + + EXPECT_EQ(g.n_samples, 0); + EXPECT_EQ(g.data, nullptr); + EXPECT_EQ(g.size(), 0); +} + + +TEST(TestGrm, ConstructorValidInput) { + grm::Grm g { 4 }; + + EXPECT_EQ(g.n_samples, 4); + EXPECT_NE(g.data, nullptr); + EXPECT_EQ(g.size(), 4 * 5 / 2); // n*(n+1)/2 = 10 + + // data should be zero-initialized + for (uint64_t i = 0; i < g.size(); i++) + EXPECT_FLOAT_EQ(g.data[i], 0.0f); +} + + +TEST(TestGrm, ConstructorZero) { + grm::Grm g { 0 }; + + EXPECT_EQ(g.n_samples, 0); + EXPECT_EQ(g.data, nullptr); + EXPECT_EQ(g.size(), 0); +} + + +TEST(TestGrm, Size) { + EXPECT_EQ(grm::Grm(0).size(), 0); + EXPECT_EQ(grm::Grm(1).size(), 1); + EXPECT_EQ(grm::Grm(2).size(), 3); + EXPECT_EQ(grm::Grm(3).size(), 6); + EXPECT_EQ(grm::Grm(4).size(), 10); + EXPECT_EQ(grm::Grm(5).size(), 15); +} + + +TEST(TestGrm, MidxToArr) { + // Verify the manual example from grm.h comments for n=3 + grm::Grm g { 3 }; + uint64_t idx = 0; + + // Upper triangle and diagonal + EXPECT_EQ(g.midx_to_arr(0, 0, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 0); + + EXPECT_EQ(g.midx_to_arr(0, 1, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 1); + + EXPECT_EQ(g.midx_to_arr(0, 2, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 2); + + EXPECT_EQ(g.midx_to_arr(1, 1, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 3); + + EXPECT_EQ(g.midx_to_arr(1, 2, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 4); + + EXPECT_EQ(g.midx_to_arr(2, 2, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 5); + + // Lower triangle should map to same idx by symmetry + EXPECT_EQ(g.midx_to_arr(1, 0, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 1); // same as (0,1) + + EXPECT_EQ(g.midx_to_arr(2, 0, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 2); // same as (0,2) + + EXPECT_EQ(g.midx_to_arr(2, 1, &idx), grm::SUCCESS); + EXPECT_EQ(idx, 4); // same as (1,2) +} + + +TEST(TestGrm, MidxToArrBoundsCheck) { + grm::Grm g { 3 }; + uint64_t idx = 0; + + EXPECT_EQ(g.midx_to_arr(3, 0, &idx), grm::ERROR_IDX_ARR_BOUNDS); + EXPECT_EQ(g.midx_to_arr(0, 3, &idx), grm::ERROR_IDX_ARR_BOUNDS); + EXPECT_EQ(g.midx_to_arr(3, 3, &idx), grm::ERROR_IDX_ARR_BOUNDS); +} + + +TEST(TestGrm, OperatorParensSetAndGet) { + grm::Grm g { 3 }; + + // Set via operator() + g(0, 0) = 1.0f; + g(0, 1) = 2.0f; + g(0, 2) = 3.0f; + g(1, 1) = 4.0f; + g(1, 2) = 5.0f; + g(2, 2) = 6.0f; + + // Read back via operator() const + const grm::Grm& cg = g; + EXPECT_FLOAT_EQ(cg(0, 0), 1.0f); + EXPECT_FLOAT_EQ(cg(0, 1), 2.0f); + EXPECT_FLOAT_EQ(cg(0, 2), 3.0f); + EXPECT_FLOAT_EQ(cg(1, 1), 4.0f); + EXPECT_FLOAT_EQ(cg(1, 2), 5.0f); + EXPECT_FLOAT_EQ(cg(2, 2), 6.0f); +} + + +TEST(TestGrm, OperatorParensSymmetry) { + grm::Grm g { 3 }; + + g(0, 1) = 7.5f; + g(2, 0) = 3.3f; + + const grm::Grm& cg = g; + + // (i,j) should equal (j,i) due to symmetry + EXPECT_FLOAT_EQ(cg(0, 1), cg(1, 0)); + EXPECT_FLOAT_EQ(cg(0, 2), cg(2, 0)); + EXPECT_FLOAT_EQ(cg(0, 1), 7.5f); + EXPECT_FLOAT_EQ(cg(0, 2), 3.3f); +} + + +TEST(TestGrm, SetAndGet) { + grm::Grm g { 3 }; + + EXPECT_EQ(g.set(0, 0, 1.1f), grm::SUCCESS); + EXPECT_EQ(g.set(1, 2, 2.2f), grm::SUCCESS); + + float val = 0.0f; + EXPECT_EQ(g.get(0, 0, &val), grm::SUCCESS); + EXPECT_FLOAT_EQ(val, 1.1f); + + EXPECT_EQ(g.get(1, 2, &val), grm::SUCCESS); + EXPECT_FLOAT_EQ(val, 2.2f); + + // symmetry + EXPECT_EQ(g.get(2, 1, &val), grm::SUCCESS); + EXPECT_FLOAT_EQ(val, 2.2f); +} + + +TEST(TestGrm, SetGetBoundsCheck) { + grm::Grm g { 3 }; + + EXPECT_EQ(g.set(3, 0, 1.0f), grm::ERROR_IDX_ARR_BOUNDS); + EXPECT_EQ(g.set(0, 3, 1.0f), grm::ERROR_IDX_ARR_BOUNDS); + + float val = 0.0f; + EXPECT_EQ(g.get(3, 0, &val), grm::ERROR_IDX_ARR_BOUNDS); + EXPECT_EQ(g.get(0, 3, &val), grm::ERROR_IDX_ARR_BOUNDS); +} + + +TEST(TestGrm, MoveConstructor) { + grm::Grm src { 3 }; + src(0, 0) = 1.0f; + src(1, 2) = 5.0f; + + grm::Grm dst { std::move(src) }; + + EXPECT_EQ(dst.n_samples, 3); + EXPECT_NE(dst.data, nullptr); + EXPECT_FLOAT_EQ(dst(0, 0), 1.0f); + EXPECT_FLOAT_EQ(dst(1, 2), 5.0f); + + EXPECT_EQ(src.n_samples, 0); + EXPECT_EQ(src.data, nullptr); +} + + +TEST(TestGrm, MoveAssignment) { + grm::Grm src { 2 }; + src(0, 0) = 1.0f; + src(0, 1) = 2.0f; + src(1, 1) = 3.0f; + + grm::Grm dst {}; + dst = std::move(src); + + EXPECT_EQ(dst.n_samples, 2); + EXPECT_FLOAT_EQ(dst(0, 0), 1.0f); + EXPECT_FLOAT_EQ(dst(0, 1), 2.0f); + EXPECT_FLOAT_EQ(dst(1, 1), 3.0f); + + EXPECT_EQ(src.n_samples, 0); + EXPECT_EQ(src.data, nullptr); +} + + +//////////////////////////////////////////////////////////////////// +// MACRO TEST +//////////////////////////////////////////////////////////////////// + +TEST(TestMatrixMacro, ManualValidation) { + // Validate MATRIX_IDX_TO_ARRAY against the worked example + // in grm.h for a 3x3 matrix + uint64_t n = 3; + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 0, n), 0); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 1, n), 1); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 2, n), 2); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(1, 1, n), 3); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(1, 2, n), 4); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(2, 2, n), 5); +} + + +TEST(TestMatrixMacro, LargerMatrix) { + // 4x4 matrix: upper triangle has 10 elements (0..9) + uint64_t n = 4; + uint64_t expected = 0; + for (uint64_t i = 0; i < n; i++) + for (uint64_t j = i; j < n; j++) + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(i, j, n), expected++); +} + + +//////////////////////////////////////////////////////////////////// +// FULL GRM FILE WRITE/READ TESTS +//////////////////////////////////////////////////////////////////// + +// Helper to build a complete Hdr + Grm for file I/O tests +static void build_test_data(grm::Hdr* hdr, grm::Grm* g) { + hdr->grm_type = grm::EHC; + + char contig[] = "chr1"; + *hdr->coords = grm::Coordinates { contig, 2 }; + hdr->coords->pos[0] = 50; + hdr->coords->pos[1] = 150; + + *hdr->samples = grm::Samples { 3 }; + hdr->samples->names[0] = "s1"; + hdr->samples->names[1] = "s2"; + hdr->samples->names[2] = "s3"; + + *g = grm::Grm { 3 }; + g->set(0, 0, 1.0f); + g->set(0, 1, 0.5f); + g->set(0, 2, 0.2f); + g->set(1, 1, 1.0f); + g->set(1, 2, 0.3f); + g->set(2, 2, 1.0f); +} + + +TEST(TestGrmFile, WriteReadRoundTrip) { + grm::Hdr hdr_w {}; + grm::Grm grm_w {}; + build_test_data(&hdr_w, &grm_w); + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + ASSERT_EQ(grm::write(&fio, &hdr_w, &grm_w), grm::SUCCESS); + + rewind(fio.fid); + + grm::Hdr hdr_r {}; + grm::Grm grm_r {}; + ASSERT_EQ(grm::read(&fio, &hdr_r, &grm_r), grm::SUCCESS); + + // verify header + EXPECT_EQ(hdr_r.grm_type, grm::EHC); + EXPECT_EQ(hdr_r.coords->contig, "chr1"); + EXPECT_EQ(hdr_r.coords->len, 2); + EXPECT_EQ(hdr_r.samples->len, 3); + EXPECT_EQ(hdr_r.samples->names[0], "s1"); + EXPECT_EQ(hdr_r.samples->names[1], "s2"); + EXPECT_EQ(hdr_r.samples->names[2], "s3"); + + // verify grm data + EXPECT_EQ(grm_r.n_samples, 3); + float val = 0.0f; + grm_r.get(0, 0, &val); EXPECT_FLOAT_EQ(val, 1.0f); + grm_r.get(0, 1, &val); EXPECT_FLOAT_EQ(val, 0.5f); + grm_r.get(0, 2, &val); EXPECT_FLOAT_EQ(val, 0.2f); + grm_r.get(1, 1, &val); EXPECT_FLOAT_EQ(val, 1.0f); + grm_r.get(1, 2, &val); EXPECT_FLOAT_EQ(val, 0.3f); + grm_r.get(2, 2, &val); EXPECT_FLOAT_EQ(val, 1.0f); +} + + +TEST(TestGrmFile, ReadBadMagicNumber) { + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + // write a wrong magic number + uint32_t bad_magic = 0x12345678; + fwrite(&bad_magic, sizeof(bad_magic), 1, fio.fid); + rewind(fio.fid); + + grm::Hdr hdr {}; + grm::Grm g {}; + EXPECT_EQ(grm::read(&fio, &hdr, &g), grm::ERROR_NOT_A_GRM_FILE); +} + + +TEST(TestGrmFile, WriteNullArgs) { + grm::Hdr hdr {}; + grm::Grm g { 2 }; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::write(nullptr, &hdr, &g), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::write(&fio, static_cast(nullptr), &g), + grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::write(&fio, &hdr, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} + + +// The read function for grm::read(fio, hdr, grm) should return +// ERROR_NULLPTR_ARG for null pointer arguments (consistent with write). +TEST(TestGrmFile, ReadNullArgs) { + grm::Hdr hdr {}; + grm::Grm g {}; + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::read(nullptr, &hdr, &g), grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::read(&fio, static_cast(nullptr), &g), + grm::ERROR_NULLPTR_ARG); + EXPECT_EQ(grm::read(&fio, &hdr, static_cast(nullptr)), + grm::ERROR_NULLPTR_ARG); +} From f2f7d0135fe0e2d6268abb9dad411d4ecd0fc24d Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 21 Mar 2026 17:07:58 -0400 Subject: [PATCH 54/58] debug so that nullptr input returns the correct error. --- src/grm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/grm.cpp b/src/grm.cpp index 3e47411..f0b83b3 100644 --- a/src/grm.cpp +++ b/src/grm.cpp @@ -493,7 +493,7 @@ grm::STATUS grm::read(io::FileIO* fio, grm::Hdr* hdr, grm::Grm* grmatrix) { if (!fio || !fio->fid || !hdr || !grmatrix) - return grm::ERROR_ON_READ; + return grm::ERROR_NULLPTR_ARG; size_t nread = 0; uint32_t ftype = 0; From 48592fa24fc910d8f2a15972d0fc24d5b4fc2be5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 21 Mar 2026 21:28:09 +0000 Subject: [PATCH 55/58] Add 24 more grm unit tests covering edge cases and error paths New tests cover: - Coordinates: single position, null fid, read overwrites existing - Samples: single sample, empty names (ERROR_INVALID_ARG), null fid, read overwrites existing - Hdr: all GrmType enum values round-trip, version field fidelity, null fid - Grm: 1x1 matrix, set via lower/get via upper triangle, 5x5 matrix indexing, move-assign replaces non-empty - GRM file: empty file read, null fid, magic number verification, single-sample round-trip, truncated file, two sequential writes then reads All 66 tests pass. https://claude.ai/code/session_01FXkdgU8L4ws7LoCu16wG4s --- tests/test_grm.cpp | 450 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 450 insertions(+) diff --git a/tests/test_grm.cpp b/tests/test_grm.cpp index 8fd9e0f..b106ef2 100644 --- a/tests/test_grm.cpp +++ b/tests/test_grm.cpp @@ -159,6 +159,68 @@ TEST(TestCoords, ReadNullArgs) { } +TEST(TestCoords, WriteReadSinglePosition) { + char contig_in[] = "chrX"; + grm::Coordinates src { contig_in, 1 }; + src.pos[0] = 42; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + rewind(fio.fid); + + grm::Coordinates dst {}; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.contig, "chrX"); + EXPECT_EQ(dst.len, 1); + ASSERT_NE(dst.pos, nullptr); + EXPECT_EQ(dst.pos[0], 42); +} + + +// A FileIO with a null fid should be treated like a null arg +TEST(TestCoords, WriteNullFid) { + grm::Coordinates coords {}; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::write(&fio, &coords), grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestCoords, ReadNullFid) { + grm::Coordinates coords {}; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::read(&fio, &coords), grm::ERROR_NULLPTR_ARG); +} + + +// Overwriting a previously populated Coordinates with read +TEST(TestCoords, ReadOverwritesExisting) { + char contig_in[] = "chr9"; + grm::Coordinates src { contig_in, 2 }; + src.pos[0] = 10; + src.pos[1] = 20; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + rewind(fio.fid); + + // dst starts with different data + char other_contig[] = "chr1"; + grm::Coordinates dst { other_contig, 5 }; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.contig, "chr9"); + EXPECT_EQ(dst.len, 2); + EXPECT_EQ(dst.pos[0], 10); + EXPECT_EQ(dst.pos[1], 20); +} + + //////////////////////////////////////////////////////////////////// // SAMPLES TESTS //////////////////////////////////////////////////////////////////// @@ -296,6 +358,81 @@ TEST(TestSamples, ReadNullArgs) { } +TEST(TestSamples, WriteReadSingleSample) { + grm::Samples src { 1 }; + src.names[0] = "only_sample"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + rewind(fio.fid); + + grm::Samples dst {}; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.len, 1); + ASSERT_NE(dst.names, nullptr); + EXPECT_EQ(dst.names[0], "only_sample"); +} + + +// Writing samples where all names are empty strings should fail +// because nchar_max would be 0, triggering ERROR_INVALID_ARG +TEST(TestSamples, WriteEmptyNamesReturnsError) { + grm::Samples samps { 2 }; + // names are default-constructed empty strings + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + EXPECT_EQ(grm::write(&fio, &samps), grm::ERROR_INVALID_ARG); +} + + +TEST(TestSamples, WriteNullFid) { + grm::Samples samps { 1 }; + samps.names[0] = "s1"; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::write(&fio, &samps), grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestSamples, ReadNullFid) { + grm::Samples samps {}; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::read(&fio, &samps), grm::ERROR_NULLPTR_ARG); +} + + +// Overwriting a previously populated Samples with read +TEST(TestSamples, ReadOverwritesExisting) { + grm::Samples src { 2 }; + src.names[0] = "new_a"; + src.names[1] = "new_b"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + rewind(fio.fid); + + // dst starts with different data + grm::Samples dst { 3 }; + dst.names[0] = "old_x"; + dst.names[1] = "old_y"; + dst.names[2] = "old_z"; + + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.len, 2); + ASSERT_NE(dst.names, nullptr); + EXPECT_EQ(dst.names[0], "new_a"); + EXPECT_EQ(dst.names[1], "new_b"); +} + + //////////////////////////////////////////////////////////////////// // HDR TESTS //////////////////////////////////////////////////////////////////// @@ -425,6 +562,76 @@ TEST(TestHdr, ReadNullArgs) { } +// Verify that every GrmType enum value survives a write/read round-trip +TEST(TestHdr, WriteReadAllGrmTypes) { + grm::GrmType types[] = { + grm::EHC, grm::EAC, grm::BOTH, grm::DS, grm::UNSPECIFIED + }; + + for (grm::GrmType t : types) { + grm::Hdr src {}; + src.grm_type = t; + + // need valid samples for write to succeed + *src.samples = grm::Samples { 1 }; + src.samples->names[0] = "s"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + + rewind(fio.fid); + + grm::Hdr dst {}; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + EXPECT_EQ(dst.grm_type, t); + } +} + + +// Verify version fields survive pack/unpack through file I/O +TEST(TestHdr, VersionRoundTrip) { + grm::Hdr src {}; + // prog_version and file_version are set by defaults; + // verify they round-trip exactly + + *src.samples = grm::Samples { 1 }; + src.samples->names[0] = "s"; + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + ASSERT_EQ(grm::write(&fio, &src), grm::SUCCESS); + rewind(fio.fid); + + grm::Hdr dst {}; + ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); + + EXPECT_EQ(dst.prog_version.major, constants::PROG_VERSION.major); + EXPECT_EQ(dst.prog_version.minor, constants::PROG_VERSION.minor); + EXPECT_EQ(dst.prog_version.micro, constants::PROG_VERSION.micro); + + EXPECT_EQ(dst.file_version.major, grm::FILE_VERSION.major); + EXPECT_EQ(dst.file_version.minor, grm::FILE_VERSION.minor); + EXPECT_EQ(dst.file_version.micro, grm::FILE_VERSION.micro); +} + + +TEST(TestHdr, WriteNullFid) { + grm::Hdr hdr {}; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::write(&fio, &hdr), grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestHdr, ReadNullFid) { + grm::Hdr hdr {}; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::read(&fio, &hdr), grm::ERROR_NULLPTR_ARG); +} + + //////////////////////////////////////////////////////////////////// // GRM STRUCT TESTS //////////////////////////////////////////////////////////////////// @@ -621,6 +828,90 @@ TEST(TestGrm, MoveAssignment) { } +TEST(TestGrm, SingleSampleMatrix) { + grm::Grm g { 1 }; + + EXPECT_EQ(g.n_samples, 1); + EXPECT_EQ(g.size(), 1); + EXPECT_NE(g.data, nullptr); + + g(0, 0) = 2.5f; + EXPECT_FLOAT_EQ(g(0, 0), 2.5f); + + float val = 0.0f; + EXPECT_EQ(g.get(0, 0, &val), grm::SUCCESS); + EXPECT_FLOAT_EQ(val, 2.5f); + + EXPECT_EQ(g.set(0, 0, 3.0f), grm::SUCCESS); + EXPECT_EQ(g.get(0, 0, &val), grm::SUCCESS); + EXPECT_FLOAT_EQ(val, 3.0f); +} + + +// Set values via the lower triangle (i > j), confirm they are +// accessible via the upper triangle (i < j) +TEST(TestGrm, SetViaLowerTriangleGetViaUpper) { + grm::Grm g { 4 }; + + // set off-diagonal values using lower triangle indices + g.set(1, 0, 1.1f); + g.set(2, 0, 2.2f); + g.set(2, 1, 3.3f); + g.set(3, 0, 4.4f); + g.set(3, 1, 5.5f); + g.set(3, 2, 6.6f); + + // get via upper triangle + float val = 0.0f; + g.get(0, 1, &val); EXPECT_FLOAT_EQ(val, 1.1f); + g.get(0, 2, &val); EXPECT_FLOAT_EQ(val, 2.2f); + g.get(1, 2, &val); EXPECT_FLOAT_EQ(val, 3.3f); + g.get(0, 3, &val); EXPECT_FLOAT_EQ(val, 4.4f); + g.get(1, 3, &val); EXPECT_FLOAT_EQ(val, 5.5f); + g.get(2, 3, &val); EXPECT_FLOAT_EQ(val, 6.6f); +} + + +// Validate indexing for a 5x5 matrix +TEST(TestGrm, LargerMatrixIndexing) { + uint64_t n = 5; + grm::Grm g { n }; + + // fill every element with a unique value + float counter = 1.0f; + for (uint64_t i = 0; i < n; i++) + for (uint64_t j = i; j < n; j++) + g.set(i, j, counter++); + + // verify all values + counter = 1.0f; + float val = 0.0f; + for (uint64_t i = 0; i < n; i++) { + for (uint64_t j = i; j < n; j++) { + g.get(i, j, &val); + EXPECT_FLOAT_EQ(val, counter++); + } + } +} + + +// Verify that move assignment replaces a non-empty Grm +TEST(TestGrm, MoveAssignmentReplacesExisting) { + grm::Grm dst { 2 }; + dst(0, 0) = 99.0f; + + grm::Grm src { 4 }; + src(0, 0) = 1.0f; + src(3, 3) = 42.0f; + + dst = std::move(src); + + EXPECT_EQ(dst.n_samples, 4); + EXPECT_FLOAT_EQ(dst(0, 0), 1.0f); + EXPECT_FLOAT_EQ(dst(3, 3), 42.0f); +} + + //////////////////////////////////////////////////////////////////// // MACRO TEST //////////////////////////////////////////////////////////////////// @@ -756,3 +1047,162 @@ TEST(TestGrmFile, ReadNullArgs) { EXPECT_EQ(grm::read(&fio, &hdr, static_cast(nullptr)), grm::ERROR_NULLPTR_ARG); } + + +// Reading from an empty file should fail +TEST(TestGrmFile, ReadEmptyFile) { + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + grm::Hdr hdr {}; + grm::Grm g {}; + EXPECT_EQ(grm::read(&fio, &hdr, &g), grm::ERROR_ON_READ); +} + + +TEST(TestGrmFile, WriteNullFid) { + grm::Hdr hdr {}; + grm::Grm g { 1 }; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::write(&fio, &hdr, &g), grm::ERROR_NULLPTR_ARG); +} + + +TEST(TestGrmFile, ReadNullFid) { + grm::Hdr hdr {}; + grm::Grm g {}; + io::FileIO fio { nullptr }; + + EXPECT_EQ(grm::read(&fio, &hdr, &g), grm::ERROR_NULLPTR_ARG); +} + + +// Verify the magic number (FILE_TYPE_SPEC) is the first 4 bytes written +TEST(TestGrmFile, MagicNumberWrittenFirst) { + grm::Hdr hdr {}; + grm::Grm g {}; + build_test_data(&hdr, &g); + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + ASSERT_EQ(grm::write(&fio, &hdr, &g), grm::SUCCESS); + + rewind(fio.fid); + + uint32_t magic = 0; + size_t nread = fread(&magic, sizeof(magic), 1, fio.fid); + ASSERT_EQ(nread, 1); + EXPECT_EQ(magic, grm::FILE_TYPE_SPEC); +} + + +// Single sample GRM file round-trip +TEST(TestGrmFile, WriteReadSingleSample) { + grm::Hdr hdr {}; + hdr.grm_type = grm::DS; + + char contig[] = "chr1"; + *hdr.coords = grm::Coordinates { contig, 1 }; + hdr.coords->pos[0] = 500; + + *hdr.samples = grm::Samples { 1 }; + hdr.samples->names[0] = "lone_sample"; + + grm::Grm g { 1 }; + g.set(0, 0, 0.75f); + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + ASSERT_EQ(grm::write(&fio, &hdr, &g), grm::SUCCESS); + + rewind(fio.fid); + + grm::Hdr hdr_r {}; + grm::Grm g_r {}; + ASSERT_EQ(grm::read(&fio, &hdr_r, &g_r), grm::SUCCESS); + + EXPECT_EQ(hdr_r.grm_type, grm::DS); + EXPECT_EQ(hdr_r.samples->len, 1); + EXPECT_EQ(hdr_r.samples->names[0], "lone_sample"); + + EXPECT_EQ(g_r.n_samples, 1); + float val = 0.0f; + g_r.get(0, 0, &val); + EXPECT_FLOAT_EQ(val, 0.75f); +} + + +// Read from a file that has the correct magic number but is +// truncated after it (no header data follows) +TEST(TestGrmFile, ReadTruncatedAfterMagic) { + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + fwrite(&grm::FILE_TYPE_SPEC, sizeof(grm::FILE_TYPE_SPEC), 1, fio.fid); + rewind(fio.fid); + + grm::Hdr hdr {}; + grm::Grm g {}; + EXPECT_EQ(grm::read(&fio, &hdr, &g), grm::ERROR_ON_READ); +} + + +// Write then read two independent GRM files sequentially to the same +// file, verifying both are recovered correctly +TEST(TestGrmFile, WriteReadTwoSequential) { + grm::Hdr hdr1 {}; + hdr1.grm_type = grm::EHC; + char c1[] = "chr1"; + *hdr1.coords = grm::Coordinates { c1, 1 }; + hdr1.coords->pos[0] = 10; + *hdr1.samples = grm::Samples { 2 }; + hdr1.samples->names[0] = "a"; + hdr1.samples->names[1] = "b"; + + grm::Grm g1 { 2 }; + g1.set(0, 0, 1.0f); + g1.set(0, 1, 0.5f); + g1.set(1, 1, 1.0f); + + grm::Hdr hdr2 {}; + hdr2.grm_type = grm::EAC; + char c2[] = "chr2"; + *hdr2.coords = grm::Coordinates { c2, 1 }; + hdr2.coords->pos[0] = 20; + *hdr2.samples = grm::Samples { 2 }; + hdr2.samples->names[0] = "x"; + hdr2.samples->names[1] = "y"; + + grm::Grm g2 { 2 }; + g2.set(0, 0, 2.0f); + g2.set(0, 1, 0.8f); + g2.set(1, 1, 2.0f); + + io::FileIO fio { tmpfile() }; + ASSERT_NE(fio.fid, nullptr); + + ASSERT_EQ(grm::write(&fio, &hdr1, &g1), grm::SUCCESS); + ASSERT_EQ(grm::write(&fio, &hdr2, &g2), grm::SUCCESS); + + rewind(fio.fid); + + // read first + grm::Hdr r1 {}; + grm::Grm rg1 {}; + ASSERT_EQ(grm::read(&fio, &r1, &rg1), grm::SUCCESS); + EXPECT_EQ(r1.grm_type, grm::EHC); + EXPECT_EQ(r1.samples->names[0], "a"); + float val = 0.0f; + rg1.get(0, 1, &val); + EXPECT_FLOAT_EQ(val, 0.5f); + + // read second + grm::Hdr r2 {}; + grm::Grm rg2 {}; + ASSERT_EQ(grm::read(&fio, &r2, &rg2), grm::SUCCESS); + EXPECT_EQ(r2.grm_type, grm::EAC); + EXPECT_EQ(r2.samples->names[0], "x"); + rg2.get(0, 1, &val); + EXPECT_FLOAT_EQ(val, 0.8f); +} From ea482dd893439ef9853cede2b4909c5dca79bb47 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 21 Mar 2026 22:33:53 -0400 Subject: [PATCH 56/58] more debugging. --- .gitignore | 1 + Makefile | 4 ++-- tests/test_grm.cpp | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index cd8cd0a..d3f64c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build/ +tmp_* *.swp *.swo *~ diff --git a/Makefile b/Makefile index 05518f0..e10d716 100644 --- a/Makefile +++ b/Makefile @@ -108,8 +108,8 @@ TEST_GRM_PRG = $(BUILD_DIR)/test_grm test_grm: $(TEST_GRM_PRG) ./$(TEST_GRM_PRG) -$(TEST_GRM_PRG): $(TEST_DIR)/main.cpp $(BUILD_DIR)/test_grm.o $(BUILD_DIR)/grm.o | $(BUILD_DIR) - $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -lgtest +$(TEST_GRM_PRG): $(BUILD_DIR)/test_grm.o $(BUILD_DIR)/grm.o | $(BUILD_DIR) + $(CXX) $(CXXFLAGS) $(CXXLDFLAGS) $(CXXLIBFLAGS) -o $@ $^ -lgtest -lgtest_main data: | $(TEST_DATA_DST) diff --git a/tests/test_grm.cpp b/tests/test_grm.cpp index b106ef2..eaf2ed5 100644 --- a/tests/test_grm.cpp +++ b/tests/test_grm.cpp @@ -16,10 +16,10 @@ TEST(TestCoords, DefaultConstructor) { std::string contig = std::string(""); - EXPECT_EQ(coords.contig.size(), 0); + EXPECT_EQ(coords.contig.size(), static_cast(0)); EXPECT_EQ(coords.contig, contig); - EXPECT_EQ(coords.len, 0); + EXPECT_EQ(coords.len, static_cast(0)); EXPECT_EQ(coords.pos, nullptr); } @@ -43,7 +43,7 @@ TEST(TestCoords, ConstructorInvalidInput) { grm::Coordinates coords { nullptr, len_in }; EXPECT_EQ(coords.contig, std::string("")); - EXPECT_EQ(coords.len, 0); + EXPECT_EQ(coords.len, static_cast(0)); EXPECT_EQ(coords.pos, nullptr); } @@ -53,7 +53,7 @@ TEST(TestCoords, ConstructorZeroLength) { grm::Coordinates coords { contig_in, 0 }; EXPECT_EQ(coords.contig, std::string("chr1")); - EXPECT_EQ(coords.len, 0); + EXPECT_EQ(coords.len, static_cast(0)); EXPECT_EQ(coords.pos, nullptr); } @@ -78,7 +78,7 @@ TEST(TestCoords, MoveConstructor) { // source should be in moved-from state EXPECT_EQ(src.contig, std::string("")); - EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.len, static_cast(0)); EXPECT_EQ(src.pos, nullptr); } @@ -95,14 +95,14 @@ TEST(TestCoords, MoveAssignment) { dst = std::move(src); EXPECT_EQ(dst.contig, std::string("chr3")); - EXPECT_EQ(dst.len, 3); + EXPECT_EQ(dst.len, static_cast(3)); EXPECT_NE(dst.pos, nullptr); - EXPECT_EQ(dst.pos[0], 10); - EXPECT_EQ(dst.pos[1], 20); - EXPECT_EQ(dst.pos[2], 30); + EXPECT_EQ(dst.pos[0], static_cast(10)); + EXPECT_EQ(dst.pos[1], static_cast(20)); + EXPECT_EQ(dst.pos[2], static_cast(30)); EXPECT_EQ(src.contig, std::string("")); - EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.len, static_cast(0)); EXPECT_EQ(src.pos, nullptr); } @@ -174,9 +174,9 @@ TEST(TestCoords, WriteReadSinglePosition) { ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); EXPECT_EQ(dst.contig, "chrX"); - EXPECT_EQ(dst.len, 1); + EXPECT_EQ(dst.len, static_cast(1)); ASSERT_NE(dst.pos, nullptr); - EXPECT_EQ(dst.pos[0], 42); + EXPECT_EQ(dst.pos[0], static_cast(42)); } From c09cc2a495a84d04e51b5ea6d16c17462c89b4c0 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Sat, 21 Mar 2026 23:04:42 -0400 Subject: [PATCH 57/58] Claude introduced the same bug many many times. The bug is the comparison of a unit64_t with an int, I've tried to find all cases and fix by static_cast. --- tests/test_grm.cpp | 120 ++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/tests/test_grm.cpp b/tests/test_grm.cpp index eaf2ed5..7917529 100644 --- a/tests/test_grm.cpp +++ b/tests/test_grm.cpp @@ -215,9 +215,9 @@ TEST(TestCoords, ReadOverwritesExisting) { ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); EXPECT_EQ(dst.contig, "chr9"); - EXPECT_EQ(dst.len, 2); - EXPECT_EQ(dst.pos[0], 10); - EXPECT_EQ(dst.pos[1], 20); + EXPECT_EQ(dst.len, static_cast(2)); + EXPECT_EQ(dst.pos[0], static_cast(10)); + EXPECT_EQ(dst.pos[1], static_cast(20)); } @@ -228,7 +228,7 @@ TEST(TestCoords, ReadOverwritesExisting) { TEST(TestSamples, DefaultConstructor) { grm::Samples samps {}; - EXPECT_EQ(samps.len, 0); + EXPECT_EQ(samps.len, static_cast(0)); EXPECT_EQ(samps.names, nullptr); } @@ -245,7 +245,7 @@ TEST(TestSamples, ConstructorValidInput) { TEST(TestSamples, ConstructorZero) { grm::Samples samps { 0 }; - EXPECT_EQ(samps.len, 0); + EXPECT_EQ(samps.len, static_cast(0)); EXPECT_EQ(samps.names, nullptr); } @@ -258,13 +258,13 @@ TEST(TestSamples, MoveConstructor) { grm::Samples dst { std::move(src) }; - EXPECT_EQ(dst.len, 3); + EXPECT_EQ(dst.len, static_cast(3)); ASSERT_NE(dst.names, nullptr); EXPECT_EQ(dst.names[0], "sample_A"); EXPECT_EQ(dst.names[1], "sample_B"); EXPECT_EQ(dst.names[2], "sample_C"); - EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.len, static_cast(0)); EXPECT_EQ(src.names, nullptr); } @@ -277,12 +277,12 @@ TEST(TestSamples, MoveAssignment) { grm::Samples dst {}; dst = std::move(src); - EXPECT_EQ(dst.len, 2); + EXPECT_EQ(dst.len, static_cast(2)); ASSERT_NE(dst.names, nullptr); EXPECT_EQ(dst.names[0], "id_1"); EXPECT_EQ(dst.names[1], "id_2"); - EXPECT_EQ(src.len, 0); + EXPECT_EQ(src.len, static_cast(0)); EXPECT_EQ(src.names, nullptr); } @@ -305,7 +305,7 @@ TEST(TestSamples, WriteReadRoundTrip) { status = grm::read(&fio, &dst); ASSERT_EQ(status, grm::SUCCESS); - EXPECT_EQ(dst.len, 3); + EXPECT_EQ(dst.len, static_cast(3)); ASSERT_NE(dst.names, nullptr); EXPECT_EQ(dst.names[0], "alpha"); EXPECT_EQ(dst.names[1], "beta"); @@ -328,7 +328,7 @@ TEST(TestSamples, WriteReadVaryingLengthNames) { grm::Samples dst {}; ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); - EXPECT_EQ(dst.len, 3); + EXPECT_EQ(dst.len, static_cast(3)); ASSERT_NE(dst.names, nullptr); EXPECT_EQ(dst.names[0], "a"); EXPECT_EQ(dst.names[1], "longer_sample_name"); @@ -371,7 +371,7 @@ TEST(TestSamples, WriteReadSingleSample) { grm::Samples dst {}; ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); - EXPECT_EQ(dst.len, 1); + EXPECT_EQ(dst.len, static_cast(1)); ASSERT_NE(dst.names, nullptr); EXPECT_EQ(dst.names[0], "only_sample"); } @@ -426,7 +426,7 @@ TEST(TestSamples, ReadOverwritesExisting) { ASSERT_EQ(grm::read(&fio, &dst), grm::SUCCESS); - EXPECT_EQ(dst.len, 2); + EXPECT_EQ(dst.len, static_cast(2)); ASSERT_NE(dst.names, nullptr); EXPECT_EQ(dst.names[0], "new_a"); EXPECT_EQ(dst.names[1], "new_b"); @@ -528,13 +528,13 @@ TEST(TestHdr, WriteReadRoundTrip) { ASSERT_NE(dst.coords, nullptr); EXPECT_EQ(dst.coords->contig, "chr5"); - EXPECT_EQ(dst.coords->len, 3); - EXPECT_EQ(dst.coords->pos[0], 100); - EXPECT_EQ(dst.coords->pos[1], 200); - EXPECT_EQ(dst.coords->pos[2], 300); + EXPECT_EQ(dst.coords->len, static_cast(3)); + EXPECT_EQ(dst.coords->pos[0], static_cast(100)); + EXPECT_EQ(dst.coords->pos[1], static_cast(200)); + EXPECT_EQ(dst.coords->pos[2], static_cast(300)); ASSERT_NE(dst.samples, nullptr); - EXPECT_EQ(dst.samples->len, 2); + EXPECT_EQ(dst.samples->len, static_cast(2)); EXPECT_EQ(dst.samples->names[0], "samp1"); EXPECT_EQ(dst.samples->names[1], "samp2"); } @@ -639,18 +639,18 @@ TEST(TestHdr, ReadNullFid) { TEST(TestGrm, DefaultConstructor) { grm::Grm g {}; - EXPECT_EQ(g.n_samples, 0); + EXPECT_EQ(g.n_samples, static_cast(0)); EXPECT_EQ(g.data, nullptr); - EXPECT_EQ(g.size(), 0); + EXPECT_EQ(g.size(), static_cast(0)); } TEST(TestGrm, ConstructorValidInput) { grm::Grm g { 4 }; - EXPECT_EQ(g.n_samples, 4); + EXPECT_EQ(g.n_samples, static_cast(4)); EXPECT_NE(g.data, nullptr); - EXPECT_EQ(g.size(), 4 * 5 / 2); // n*(n+1)/2 = 10 + EXPECT_EQ(g.size(), static_cast(4 * 5 / 2)); // n*(n+1)/2 = 10 // data should be zero-initialized for (uint64_t i = 0; i < g.size(); i++) @@ -661,19 +661,19 @@ TEST(TestGrm, ConstructorValidInput) { TEST(TestGrm, ConstructorZero) { grm::Grm g { 0 }; - EXPECT_EQ(g.n_samples, 0); + EXPECT_EQ(g.n_samples, static_cast(0)); EXPECT_EQ(g.data, nullptr); - EXPECT_EQ(g.size(), 0); + EXPECT_EQ(g.size(), static_cast(0)); } TEST(TestGrm, Size) { - EXPECT_EQ(grm::Grm(0).size(), 0); - EXPECT_EQ(grm::Grm(1).size(), 1); - EXPECT_EQ(grm::Grm(2).size(), 3); - EXPECT_EQ(grm::Grm(3).size(), 6); - EXPECT_EQ(grm::Grm(4).size(), 10); - EXPECT_EQ(grm::Grm(5).size(), 15); + EXPECT_EQ(grm::Grm(0).size(), static_cast(0)); + EXPECT_EQ(grm::Grm(1).size(), static_cast(1)); + EXPECT_EQ(grm::Grm(2).size(), static_cast(3)); + EXPECT_EQ(grm::Grm(3).size(), static_cast(6)); + EXPECT_EQ(grm::Grm(4).size(), static_cast(10)); + EXPECT_EQ(grm::Grm(5).size(), static_cast(15)); } @@ -684,32 +684,32 @@ TEST(TestGrm, MidxToArr) { // Upper triangle and diagonal EXPECT_EQ(g.midx_to_arr(0, 0, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 0); + EXPECT_EQ(idx, static_cast(0)); EXPECT_EQ(g.midx_to_arr(0, 1, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 1); + EXPECT_EQ(idx, static_cast(1)); EXPECT_EQ(g.midx_to_arr(0, 2, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 2); + EXPECT_EQ(idx, static_cast(2)); EXPECT_EQ(g.midx_to_arr(1, 1, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 3); + EXPECT_EQ(idx, static_cast(3)); EXPECT_EQ(g.midx_to_arr(1, 2, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 4); + EXPECT_EQ(idx, static_cast(4)); EXPECT_EQ(g.midx_to_arr(2, 2, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 5); + EXPECT_EQ(idx, static_cast(5)); // Lower triangle should map to same idx by symmetry EXPECT_EQ(g.midx_to_arr(1, 0, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 1); // same as (0,1) + EXPECT_EQ(idx, static_cast(1)); // same as (0,1) EXPECT_EQ(g.midx_to_arr(2, 0, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 2); // same as (0,2) + EXPECT_EQ(idx, static_cast(2)); // same as (0,2) EXPECT_EQ(g.midx_to_arr(2, 1, &idx), grm::SUCCESS); - EXPECT_EQ(idx, 4); // same as (1,2) + EXPECT_EQ(idx, static_cast(4)); // same as (1,2) } @@ -799,12 +799,12 @@ TEST(TestGrm, MoveConstructor) { grm::Grm dst { std::move(src) }; - EXPECT_EQ(dst.n_samples, 3); + EXPECT_EQ(dst.n_samples, static_cast(3)); EXPECT_NE(dst.data, nullptr); EXPECT_FLOAT_EQ(dst(0, 0), 1.0f); EXPECT_FLOAT_EQ(dst(1, 2), 5.0f); - EXPECT_EQ(src.n_samples, 0); + EXPECT_EQ(src.n_samples, static_cast(0)); EXPECT_EQ(src.data, nullptr); } @@ -818,12 +818,12 @@ TEST(TestGrm, MoveAssignment) { grm::Grm dst {}; dst = std::move(src); - EXPECT_EQ(dst.n_samples, 2); + EXPECT_EQ(dst.n_samples, static_cast(2)); EXPECT_FLOAT_EQ(dst(0, 0), 1.0f); EXPECT_FLOAT_EQ(dst(0, 1), 2.0f); EXPECT_FLOAT_EQ(dst(1, 1), 3.0f); - EXPECT_EQ(src.n_samples, 0); + EXPECT_EQ(src.n_samples, static_cast(0)); EXPECT_EQ(src.data, nullptr); } @@ -831,8 +831,8 @@ TEST(TestGrm, MoveAssignment) { TEST(TestGrm, SingleSampleMatrix) { grm::Grm g { 1 }; - EXPECT_EQ(g.n_samples, 1); - EXPECT_EQ(g.size(), 1); + EXPECT_EQ(g.n_samples, static_cast(1)); + EXPECT_EQ(g.size(), static_cast(1)); EXPECT_NE(g.data, nullptr); g(0, 0) = 2.5f; @@ -906,7 +906,7 @@ TEST(TestGrm, MoveAssignmentReplacesExisting) { dst = std::move(src); - EXPECT_EQ(dst.n_samples, 4); + EXPECT_EQ(dst.n_samples, static_cast(4)); EXPECT_FLOAT_EQ(dst(0, 0), 1.0f); EXPECT_FLOAT_EQ(dst(3, 3), 42.0f); } @@ -920,12 +920,12 @@ TEST(TestMatrixMacro, ManualValidation) { // Validate MATRIX_IDX_TO_ARRAY against the worked example // in grm.h for a 3x3 matrix uint64_t n = 3; - EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 0, n), 0); - EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 1, n), 1); - EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 2, n), 2); - EXPECT_EQ(MATRIX_IDX_TO_ARRAY(1, 1, n), 3); - EXPECT_EQ(MATRIX_IDX_TO_ARRAY(1, 2, n), 4); - EXPECT_EQ(MATRIX_IDX_TO_ARRAY(2, 2, n), 5); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 0, n), static_cast(0)); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 1, n), static_cast(1)); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(0, 2, n), static_cast(2)); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(1, 1, n), static_cast(3)); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(1, 2, n), static_cast(4)); + EXPECT_EQ(MATRIX_IDX_TO_ARRAY(2, 2, n), static_cast(5)); } @@ -949,8 +949,8 @@ static void build_test_data(grm::Hdr* hdr, grm::Grm* g) { char contig[] = "chr1"; *hdr->coords = grm::Coordinates { contig, 2 }; - hdr->coords->pos[0] = 50; - hdr->coords->pos[1] = 150; + hdr->coords->pos[0] = static_cast(50); + hdr->coords->pos[1] = static_cast(150); *hdr->samples = grm::Samples { 3 }; hdr->samples->names[0] = "s1"; @@ -986,14 +986,14 @@ TEST(TestGrmFile, WriteReadRoundTrip) { // verify header EXPECT_EQ(hdr_r.grm_type, grm::EHC); EXPECT_EQ(hdr_r.coords->contig, "chr1"); - EXPECT_EQ(hdr_r.coords->len, 2); - EXPECT_EQ(hdr_r.samples->len, 3); + EXPECT_EQ(hdr_r.coords->len, static_cast(2)); + EXPECT_EQ(hdr_r.samples->len, static_cast(3)); EXPECT_EQ(hdr_r.samples->names[0], "s1"); EXPECT_EQ(hdr_r.samples->names[1], "s2"); EXPECT_EQ(hdr_r.samples->names[2], "s3"); // verify grm data - EXPECT_EQ(grm_r.n_samples, 3); + EXPECT_EQ(grm_r.n_samples, static_cast(3)); float val = 0.0f; grm_r.get(0, 0, &val); EXPECT_FLOAT_EQ(val, 1.0f); grm_r.get(0, 1, &val); EXPECT_FLOAT_EQ(val, 0.5f); @@ -1092,7 +1092,7 @@ TEST(TestGrmFile, MagicNumberWrittenFirst) { uint32_t magic = 0; size_t nread = fread(&magic, sizeof(magic), 1, fio.fid); - ASSERT_EQ(nread, 1); + ASSERT_EQ(nread, static_cast(1)); EXPECT_EQ(magic, grm::FILE_TYPE_SPEC); } @@ -1123,10 +1123,10 @@ TEST(TestGrmFile, WriteReadSingleSample) { ASSERT_EQ(grm::read(&fio, &hdr_r, &g_r), grm::SUCCESS); EXPECT_EQ(hdr_r.grm_type, grm::DS); - EXPECT_EQ(hdr_r.samples->len, 1); + EXPECT_EQ(hdr_r.samples->len, static_cast(1)); EXPECT_EQ(hdr_r.samples->names[0], "lone_sample"); - EXPECT_EQ(g_r.n_samples, 1); + EXPECT_EQ(g_r.n_samples, static_cast(1)); float val = 0.0f; g_r.get(0, 0, &val); EXPECT_FLOAT_EQ(val, 0.75f); From 1edf2a9fbaf57864ac7f7655358d351eff424e09 Mon Sep 17 00:00:00 2001 From: Robert Vogel <12845765+robert-vogel@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:53:10 -0400 Subject: [PATCH 58/58] Changed aesthetics in comments for source and header files. Added unit tests for all expected haplotype counts. The ground truth values are stored in a csv hd_0i.csv where hd represents haplotype dose and i the sample number. --- README.md | 4 +- include/bcfio.h | 153 +++++++++------- src/bcfio.cpp | 36 ++-- tests/hd_01.csv | 11 ++ tests/hd_02.csv | 11 ++ tests/hd_03.csv | 11 ++ tests/hd_04.csv | 11 ++ tests/hd_05.csv | 11 ++ tests/hd_06.csv | 11 ++ tests/hd_07.csv | 11 ++ tests/hd_08.csv | 11 ++ tests/test_bcfio.cpp | 397 ++++++++++++++++++++++++------------------ tests/test_matrix.cpp | 86 --------- 13 files changed, 441 insertions(+), 323 deletions(-) create mode 100644 tests/hd_01.csv create mode 100644 tests/hd_02.csv create mode 100644 tests/hd_03.csv create mode 100644 tests/hd_04.csv create mode 100644 tests/hd_05.csv create mode 100644 tests/hd_06.csv create mode 100644 tests/hd_07.csv create mode 100644 tests/hd_08.csv delete mode 100644 tests/test_matrix.cpp diff --git a/README.md b/README.md index e605b63..205d532 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# `grm` a tool for computing genetic relationship matrices +# `hwas` a tool for haplotype wide association analyses - 🏗️ **Under construction** 🏗️ + 🏗️ **Under construction** 🏗️ ## Table of Contents diff --git a/include/bcfio.h b/include/bcfio.h index fc38538..9ab670c 100644 --- a/include/bcfio.h +++ b/include/bcfio.h @@ -33,35 +33,40 @@ namespace bcfio { // @title The meta data on a BCF attribute -// @description BCF, VCF, and VCF.GZ files hold metadata in the header that -// specify the type and format of data in records. I call each unique -// piece of data in a record a record attribute, e.g. an INFO column or -// FORMAT column of a record are attributes of that record. HTSLIB encodes -// attribute information in an unsigned 64 bit integer, and to access any -// value one needs to correctly implement bit shifting and masking. This -// struct contains bit-fields representing each value stored in the -// uint64_t. -// @bitfield number: the number of distinct values required to specify a sample -// record at loci i. For example, a SNP genotype is specified by a single -// string, e.g. 0/1, while the posterior genotype (0/0, 0/1, 1/1) -// probabilities requires three numbers. -// @bitfield vl_type: Specifies whether a variable is fixed length (BCF_VL_FIXED, -// in htslib/vcf.h line 68), variable length, etc. -// @bitfield type: the type of variable: binary flag (BCF_HT_FLAG), integer, -// real number, string, and 64 bit integers. Note that HT is header type. +// @description BCF, VCF, and VCF.GZ files hold metadata in the +// header that specify the type and format of data in records. +// I call each unique piece of data in a record a record attribute, +// e.g. an INFO column or FORMAT column of a record are attributes +// of that record. HTSLIB encodes attribute information in an +// unsigned 64 bit integer, and to access any value one needs to +// correctly implement bit shifting and masking. This struct contains +// bit-fields representing each value stored in the uint64_t. +// @bitfield number: the number of distinct values required to +// specify a sample +// record at loci i. For example, a SNP genotype is specified by a +// single +// string, e.g. 0/1, while the posterior genotype (0/0, 0/1, 1/1) +// probabilities requires three numbers. +// @bitfield vl_type: Specifies whether a variable is fixed length +// (BCF_VL_FIXED, in htslib/vcf.h line 68), variable length, etc. +// @bitfield type: the type of variable: binary flag (BCF_HT_FLAG), +// integer, real number, string, and 64 bit integers. Note that HT +// is header type. // @bitfield coltype: struct BcfHdrAttr { uint64_t number : 20, vl_type : 4, type : 4, coltype : 4; }; // @title: Interface and manager of htslib bcf_hdr_t -// @description: The bcf header C-struct requires manual allocation and release -// of memory. This class manages applies RAII, reducing the chance of a -// memory leak. +// @description: The bcf header C-struct requires manual allocation +// and release of memory. This class manages applies RAII, reducing +// the chance of a memory leak. class BcfHeader { public: - - BcfHeader(htslib::htsFile *fid): - hdr_(fid ? htslib::bcf_hdr_read(fid) : nullptr) {}; + BcfHeader() + : hdr_(nullptr) {}; + + BcfHeader(htslib::htsFile *fid) + : hdr_(fid ? htslib::bcf_hdr_read(fid) : nullptr) {}; ~BcfHeader() { if (hdr_) htslib::bcf_hdr_destroy(hdr_); }; @@ -74,8 +79,8 @@ class BcfHeader { // @title: "get_*" member functions for attribute retrieval // @description: // @param id: the id of the formatted data field to retrieve - // @param ptr: the pointer to memory for which the BcfHdrAttr data will be - // copied into memory. + // @param ptr: the pointer to memory for which the BcfHdrAttr + // data will be copied into memory. // @return 0 for success < 0 for fail int get_format_attr(const char *id, BcfHdrAttr *ptr) const; int get_info_attr(const char *id, BcfHdrAttr *ptr) const; @@ -83,15 +88,16 @@ class BcfHeader { int subset_samples(const char *filename); // @title: The number of values stored in format id - // @description: Each bcf format field is able to hold unique number of - // values per sample. This function provides a simple interface to - // the bcf file to retrieve that number. + // @description: Each bcf format field is able to hold unique + // number of values per sample. This function provides a simple + // interface to the bcf file to retrieve that number. // @param id: the format field id - // @return if an error occured that value returned is < 0, otherwise the - // number of values of fmt field id recorded per sample is returned. + // @return if an error occured that value returned is < 0, + // otherwise the number of values of fmt field id recorded per + // sample is returned. int32_t k_fmt(const char *id) const; - size_t n_samples() const { return hdr_->n[BCF_DT_SAMPLE]; }; + uint32_t n_samples() const { return hdr_->n[BCF_DT_SAMPLE]; }; // TODO: what unit test should I do for this? const htslib::bcf_hdr_t *hts_hdr() const { return hdr_; }; @@ -101,8 +107,8 @@ class BcfHeader { // BcfHdrAttr attr_ {}; // @title: - // @description decoder based upon htslib/vcf.h line 100 in the typedef - // struct bcf_idinfo_t. + // @description decoder based upon htslib/vcf.h line 100 in the + // typedef struct bcf_idinfo_t. // @param name: // @param bcf_dt_type // @param ptr @@ -115,11 +121,11 @@ class BcfHeader { // @title: Interface and manage htslib bcf1_t // @description: The htslib bcf1_t data structure requires manual memory -// management, knowledge of several bit-packed values, knowledge of -// several functions for querying data. This class simplifies -// memory management using C++ RAII idiom and provides a simplified, -// albeit non-comprehensive, interface for loading and querying data -// stored in the bcf1_t struct. +// management, knowledge of several bit-packed values, knowledge of +// several functions for querying data. This class simplifies +// memory management using C++ RAII idiom and provides a simplified, +// albeit non-comprehensive, interface for loading and querying data +// stored in the bcf1_t struct. class BcfFloatRecord { public: @@ -133,16 +139,21 @@ class BcfFloatRecord { std::optional get(const size_t row_idx, const size_t col_idx) const; // @title: Load sample data at the current locus - // @description: Sample data of the specified format at the current locus - // is not made available by reading a locus's record and storing in - // the bcf1_t type. Instead, we need to supply a pointer variable - // and format id to make that id's smaple data available. This - // function help simplify this process. + // @description: Sample data at the current locus, is not made + // available by reading a locus's record and storing in the + // bcf1_t type. Instead, we need to supply a pointer variable + // and format id to make that id's smaple data available. This + // function help simplify this process. // @param hdr: instance of the bcf header to retreive meta data - // @param tag: the C-string id representing the data we want to query. + // @param tag: the C-string id representing the data we want to + // query. // @return 0 upon success and != 0 for failure int load_data(BcfHeader *hdr, const char *tag); - size_t size() const { return static_cast(ndst_); }; + + // the total amount of values per record, n_samples * k_founders + uint64_t size() const { return static_cast(ndst_); }; + uint64_t ncols() const { return col_num_; }; + uint64_t nrows() const { return row_num_; }; htslib::bcf1_t *cur_rec() const { return rec_; }; @@ -152,13 +163,17 @@ class BcfFloatRecord { htslib::bcf1_t *rec_; // These attributes store htslib access points to record data + // + // ndst_: The number of values in memory, the length of dst + // *dst_: an array of length ndst_ with float values of the + // current record int ndst_ = 0; float *dst_ = nullptr; - // data that dst_ point to are stored in row major order, with columns - // being k_fmt and rows being n_samples. - size_t col_num_ = 0; - size_t row_num_ = 0; + // data that dst_ point to are stored in row major order, with + // columns being k_fmt and rows being n_samples. + uint64_t col_num_ = 0; + uint64_t row_num_ = 0; }; @@ -167,28 +182,33 @@ class BcfFloatRecord { // and organizes the bcf file header and any one record for easy // and memory safe parsing. // @param bcfname: the path and filename to the bcf file to be read. -// @param sample_fname: the path and filename of the text file listing the -// samples id's of records to be retreived. If this is not included -// all sample records are retrieved. +// @param sample_fname: the path and filename of the text file listing +// samples id's of records to be retreived. If this is not included +// all sample records are retrieved. class ReadBcf { public: - // TODO: Review C++ idioms the rule of three and five - ReadBcf(const char *bcfname); + ReadBcf(); + ReadBcf(const char* filename, htslib::htsFile* fid); - ReadBcf()=delete; ReadBcf(const ReadBcf&)=delete; - ReadBcf(const ReadBcf&&)=delete; + ReadBcf& operator=(const ReadBcf&)=delete; + + ReadBcf(ReadBcf&&)=delete; + ReadBcf& operator=(ReadBcf&&)=delete; ~ReadBcf(); + bool isopen() const; + // @title: The number of values stored in format id - // @description: Each bcf format field is able to hold unique number of - // values per sample. This function provides a simple interface to - // the bcf file to retrieve that number. + // @description: Each bcf format field is able to hold unique + // number of values per sample. This function provides a + // simple interface to the bcf file to retrieve that number. // @param id: the format field id - // @return if an error occured that value returned is < 0, otherwise the - // number of values of fmt field id recorded per sample is returned. + // @return if an error occured that value returned is < 0, + // otherwise the number of values of fmt field id recorded per + // sample is returned. int32_t k_fmt(const char *id) const { return hdr_.k_fmt(id); }; // See htslib/vcf.h line 649 @@ -212,6 +232,19 @@ class ReadBcf BcfHeader hdr_; }; + +// mode according to htslib: quoting from htslib/hts.h line 608 +// +// @example +// [rw]b .. compressed BCF, BAM, FAI +// [rw]bu .. uncompressed BCF +// [rw]z .. compressed VCF +// [rw] .. uncompressed VCF +// +// End quote +// +ReadBcf open(const char* filename, const char* mode); + } #endif diff --git a/src/bcfio.cpp b/src/bcfio.cpp index 113f219..93294ce 100644 --- a/src/bcfio.cpp +++ b/src/bcfio.cpp @@ -11,6 +11,7 @@ #include + /////////////////////////////////////////////////////////////////// // BcfHeader /////////////////////////////////////////////////////////////////// @@ -19,8 +20,8 @@ int bcfio::BcfHeader::decode_hts_idinfo_(const char *name, const int bcf_dt_type, bcfio::BcfHdrAttr *ptr) const { - // BCF_DT_ID is the C macro for the ID dictionary index defined by htslib - // see htslib/vcf.h line 86 + // BCF_DT_ID is the C macro for the ID dictionary index defined + // by htslib see htslib/vcf.h line 86 int idx = htslib::bcf_hdr_id2int(hdr_, BCF_DT_ID, name); if (idx < 0) @@ -90,9 +91,10 @@ bcfio::BcfFloatRecord::~BcfFloatRecord() { std::optional bcfio::BcfFloatRecord::get(const size_t row_idx, const size_t col_idx) const { - if ((row_idx * col_idx + col_idx) >= size()) return std::nullopt; + size_t idx = row_idx * col_num_ + col_idx; + if (idx >= size()) return std::nullopt; - return *(dst_ + row_idx * col_idx + col_idx); + return *(dst_ + idx); } int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { @@ -115,7 +117,7 @@ int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { return k; } - col_num_ = static_cast(k); + col_num_ = static_cast(k); row_num_ = hdr->n_samples(); return 0; @@ -126,16 +128,22 @@ int bcfio::BcfFloatRecord::load_data(bcfio::BcfHeader *hdr, const char *id) { // BcfRead /////////////////////////////////////////////////////////////////// /// -bcfio::ReadBcf::ReadBcf(const char *bcfname) - : fname_(bcfname), - fid_(htslib::hts_open(bcfname, "r")), - hdr_(fid_) {}; +bcfio::ReadBcf::ReadBcf() + : fname_(""), fid_(nullptr), hdr_() {}; + +bcfio::ReadBcf::ReadBcf(const char *filename, htslib::htsFile* fid) + : fname_(filename), + fid_(fid), + hdr_(fid_) {}; bcfio::ReadBcf::~ReadBcf() { if (fid_) htslib::hts_close(fid_); } +bool bcfio::ReadBcf::isopen() const { + return fid_ != nullptr; +} // TODO: subset samples by those in sample_fname file int bcfio::ReadBcf::set_samples(const char *sample_fname) { @@ -150,7 +158,6 @@ int bcfio::ReadBcf::set_samples(const char *sample_fname) { return hdr_.subset_samples(sample_fname); }; - // title: load next record int bcfio::ReadBcf::next_record(bcfio::BcfFloatRecord *ptr, const char *id) { int status = htslib::bcf_read(fid_, hdr_.hts_hdr(), ptr->cur_rec()); @@ -163,3 +170,12 @@ int bcfio::ReadBcf::next_record(bcfio::BcfFloatRecord *ptr, const char *id) { return ptr->load_data(&hdr_, id); } + +bcfio::ReadBcf bcfio::open(const char* filename, const char* mode) { + htslib::htsFile* fid = htslib::hts_open(filename, mode); + if (!fid) + return bcfio::ReadBcf(); + + return bcfio::ReadBcf(filename, fid); +} + diff --git a/tests/hd_01.csv b/tests/hd_01.csv new file mode 100644 index 0000000..8fd1fc2 --- /dev/null +++ b/tests/hd_01.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0.001,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_02.csv b/tests/hd_02.csv new file mode 100644 index 0000000..9e05481 --- /dev/null +++ b/tests/hd_02.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_03.csv b/tests/hd_03.csv new file mode 100644 index 0000000..9e05481 --- /dev/null +++ b/tests/hd_03.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_04.csv b/tests/hd_04.csv new file mode 100644 index 0000000..1305763 --- /dev/null +++ b/tests/hd_04.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_05.csv b/tests/hd_05.csv new file mode 100644 index 0000000..9e05481 --- /dev/null +++ b/tests/hd_05.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_06.csv b/tests/hd_06.csv new file mode 100644 index 0000000..9e05481 --- /dev/null +++ b/tests/hd_06.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_07.csv b/tests/hd_07.csv new file mode 100644 index 0000000..9e05481 --- /dev/null +++ b/tests/hd_07.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0.001,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.069,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.907,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/hd_08.csv b/tests/hd_08.csv new file mode 100644 index 0000000..4bf0c43 --- /dev/null +++ b/tests/hd_08.csv @@ -0,0 +1,11 @@ +1.004,0,0.002,0.001,0,0.991,0.001,0.002 +0,0,0,0.989,0.005,0.005,1,0 +0.998,0,0.001,0,0,0,1.001,0 +0.84,0,0,0,0,0,1.159,0 +0.005,0.535,0.193,0.068,0.597,0.004,0.001,0.597 +0.001,0,0.001,1.002,0.02,0.013,0,0.962 +0.001,0.001,0,1.004,0.024,0.97,0,0.001 +0,0.955,0.908,0.125,0.003,0.004,0,0.005 +0.002,0.001,0.001,0.991,0.988,0.007,0.002,0.008 +0.249,0.001,0,1,0.748,0,0,0.001 +0.592,0,1,0,0,0,0,0.407 diff --git a/tests/test_bcfio.cpp b/tests/test_bcfio.cpp index af56356..1818ac8 100644 --- a/tests/test_bcfio.cpp +++ b/tests/test_bcfio.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace htslib { extern "C" { @@ -17,16 +18,17 @@ extern "C" { char VCF_NAME[] { "build/geno_test_data.vcf" }; char VCFGZ_NAME[] { "build/geno_test_data.vcf.gz" }; char BCF_NAME[] { "build/geno_test_data.bcf" }; -size_t K_FOUNDERS = 8; -size_t N_SAMPS = 11; +uint8_t K_FOUNDERS = 8; +uint8_t N_SAMPS = 11; -// ************************************************************************ + +/////////////////////////////////////////////////////////////////////////// // Test bcfio::BcfHeader -// ************************************************************************ +/////////////////////////////////////////////////////////////////////////// TEST(TestBcfHeader, ConstructorVcfHdr) { - htslib::htsFile *fid = htslib::hts_open(VCF_NAME, "r"); + htslib::htsFile* fid = htslib::hts_open(VCF_NAME, "r"); bcfio::BcfHeader hdr { fid }; EXPECT_FALSE(hdr.isnull()); @@ -35,15 +37,20 @@ TEST(TestBcfHeader, ConstructorVcfHdr) { int status = hdr.get_format_attr("HD", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.number, K_FOUNDERS); - EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); - EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.number, static_cast(K_FOUNDERS)); + + // This is a bit tricky. htslib/vcf.h encodes BCF_VL_FIXED + // and BCF_HT_REAL as integer macros. The values are bit packed + // in a uint64_t type. Consequently I need to cast the htslib + // macros to unsigned int + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_FIXED)); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); if (fid) htslib::hts_close(fid); } TEST(TestBcfHeader, ConstructorVcfGzHdr) { - htslib::htsFile *fid = htslib::hts_open(VCFGZ_NAME, "r"); + htslib::htsFile* fid = htslib::hts_open(VCFGZ_NAME, "r"); bcfio::BcfHeader hdr { fid }; EXPECT_FALSE(hdr.isnull()); @@ -52,9 +59,9 @@ TEST(TestBcfHeader, ConstructorVcfGzHdr) { int status = hdr.get_format_attr("HD", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.number, K_FOUNDERS); - EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); - EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.number, static_cast(K_FOUNDERS)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_FIXED)); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); if (fid) htslib::hts_close(fid); } @@ -69,9 +76,9 @@ TEST(TestBcfHeader, ConstructorBcfHdr) { int status = hdr.get_format_attr("HD", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.number, K_FOUNDERS); - EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); - EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.number, static_cast(K_FOUNDERS)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_FIXED)); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); if (fid) htslib::hts_close(fid); } @@ -87,9 +94,9 @@ TEST(TestBcfHeader, BcfHdrFmtGt) { int status = hdr.get_format_attr("GT", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.number, 1); - EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); - EXPECT_EQ(attr.type, BCF_HT_STR); + EXPECT_EQ(attr.number, static_cast(1)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_FIXED)); + EXPECT_EQ(attr.type, static_cast(BCF_HT_STR)); if (fid) htslib::hts_close(fid); } @@ -105,9 +112,9 @@ TEST(TestBcfHeader, BcfHdrFmtGp) { int status = hdr.get_format_attr("GP", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.number, 3); - EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); - EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.number, static_cast(3)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_FIXED)); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); if (fid) htslib::hts_close(fid); } @@ -122,9 +129,9 @@ TEST(TestBcfHeader, BcfHdrFmtDs) { int status = hdr.get_format_attr("DS", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.number, 1); - EXPECT_EQ(attr.vl_type, BCF_VL_FIXED); - EXPECT_EQ(attr.type, BCF_HT_REAL); + EXPECT_EQ(attr.number, static_cast(1)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_FIXED)); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); if (fid) htslib::hts_close(fid); } @@ -173,8 +180,8 @@ TEST(TestBcfHeader, BcfHdrInfoEaf) { int status = hdr.get_info_attr("EAF", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.type, BCF_HT_REAL); - EXPECT_EQ(attr.vl_type, BCF_VL_VAR); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_VAR)); if (fid) htslib::hts_close(fid); } @@ -190,8 +197,8 @@ TEST(TestBcfHeader, BcfHdrInfoErc) { int status = hdr.get_info_attr("ERC", &attr); EXPECT_EQ(status, 0); - EXPECT_EQ(attr.type, BCF_HT_REAL); - EXPECT_EQ(attr.vl_type, BCF_VL_VAR); + EXPECT_EQ(attr.type, static_cast(BCF_HT_REAL)); + EXPECT_EQ(attr.vl_type, static_cast(BCF_VL_VAR)); if (fid) htslib::hts_close(fid); } @@ -240,8 +247,8 @@ TEST(TestBcfHeader, Nsamples) { htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); bcfio::BcfHeader hdr { fid }; - // DS is alt allele dosage, which is more clearly defined as the expected - // count of alt alleles under the trained HMM + // DS is alt allele dosage, which is more clearly defined as the + // expected count of alt alleles under the trained HMM EXPECT_EQ(hdr.n_samples(), N_SAMPS); } @@ -251,72 +258,76 @@ TEST(TestBcfHeader, VcfSampNames) { const std::unique_ptr s = hdr.sample_names(); - char samp_name[] = "S01"; - - for (size_t i = 0; i < hdr.n_samples(); i++) { - snprintf(samp_name, 4, "S%02zu", i+1); + char samp_name[] = "S011"; + for (uint8_t i = 0; i < hdr.n_samples(); i++) { + snprintf(samp_name, 5, "S%02u", i+1); EXPECT_STREQ(s[i].c_str(), samp_name); } } -TEST(TestReadBcf, VcfGzSampNames) { - htslib::htsFile *fid = htslib::hts_open(VCFGZ_NAME, "r"); - bcfio::BcfHeader hdr { fid }; - - const std::unique_ptr s = hdr.sample_names(); +/////////////////////////////////////////////////////////////////////////// +// Test bcfio::BcfFloatRecord +/////////////////////////////////////////////////////////////////////////// - char samp_name[] = "S01"; +TEST(TestBcfFloatRecord, Constructor) { + bcfio::BcfFloatRecord brec {}; - for (size_t i = 0; i < hdr.n_samples(); i++) { - snprintf(samp_name, 4, "S%02zu", i+1); - EXPECT_STREQ(s[i].c_str(), samp_name); - } + EXPECT_EQ(brec.size(), static_cast(0)); + EXPECT_EQ(brec.get(1, 3), std::nullopt); } - -TEST(TestReadBcf, BcfSampNames) { +TEST(TestBcfFloatRecord, Load) { htslib::htsFile *fid = htslib::hts_open(BCF_NAME, "r"); bcfio::BcfHeader hdr { fid }; - const std::unique_ptr s = hdr.sample_names(); - - char samp_name[] = "S01"; - - for (size_t i = 0; i < hdr.n_samples(); i++) { - snprintf(samp_name, 4, "S%02zu", i+1); - EXPECT_STREQ(s[i].c_str(), samp_name); - } } +/////////////////////////////////////////////////////////////////////////// +// Test bcfio::ReadBcf +/////////////////////////////////////////////////////////////////////////// -// ************************************************************************ -// Test bcfio::BcfFloatRecord -// ************************************************************************ -TEST(TestBcfFloatRecord, Constructor) { - bcfio::BcfFloatRecord brec {}; +TEST(TestReadBcf, DefaultConstructor) { + bcfio::ReadBcf bcf {}; - EXPECT_EQ(brec.size(), static_cast(0)); - EXPECT_EQ(brec.get(1, 3), std::nullopt); + EXPECT_FALSE(bcf.isopen()); } +TEST(TestReadBcf, Constructor) { + htslib::htsFile* fid = htslib::hts_open(VCF_NAME, "r"); + bcfio::ReadBcf bcf { VCF_NAME, fid }; + EXPECT_TRUE(bcf.isopen()); -// ************************************************************************ -// Test bcfio::ReadBcf -// ************************************************************************ - -TEST(TestReadBcf, Constructor) { - bcfio::ReadBcf bcf { VCF_NAME }; EXPECT_EQ(bcf.n_samples(), N_SAMPS); EXPECT_EQ(bcf.k_fmt("HD"), K_FOUNDERS); } +TEST(TestReadBcf, OpenFailure) { + bcfio::ReadBcf bcf = bcfio::open("", "r"); + + EXPECT_FALSE(bcf.isopen()); +} + +TEST(TestReadBcf, OpenVCFSuccess) { + bcfio::ReadBcf bcf = bcfio::open(VCF_NAME, "r"); + EXPECT_TRUE(bcf.isopen()); +} + +TEST(TestReadBcf, OpenVCFGZSuccess) { + bcfio::ReadBcf bcf = bcfio::open(VCFGZ_NAME, "r"); + EXPECT_TRUE(bcf.isopen()); +} + +TEST(TestReadBcf, OpenBCFSuccess) { + bcfio::ReadBcf bcf = bcfio::open(BCF_NAME, "r"); + EXPECT_TRUE(bcf.isopen()); +} TEST(TestReadBcf, Kfmt) { - bcfio::ReadBcf bcf { VCF_NAME }; + bcfio::ReadBcf bcf = bcfio::open( VCF_NAME, "r"); // DS is alt allele dosage, which is more clearly defined as the expected // count of alt alleles under the trained HMM @@ -331,102 +342,158 @@ TEST(TestReadBcf, Kfmt) { EXPECT_TRUE(bcf.k_fmt(nullptr) < 0); } +TEST(TestReadBcf, VcfGzSampNames) { + bcfio::ReadBcf bcf = bcfio::open( VCFGZ_NAME, "r"); + + const std::unique_ptr s = bcf.sample_names(); + + char samp_name[] = "S011"; + + for (uint8_t i = 0; i < bcf.n_samples(); i++) { + snprintf(samp_name, 5, "S%02u", i+1); + EXPECT_STREQ(s[i].c_str(), samp_name); + } +} + + +TEST(TestReadBcf, BcfSampNames) { + bcfio::ReadBcf bcf = bcfio::open( BCF_NAME, "r"); + + const std::unique_ptr s = bcf.sample_names(); + + char samp_name[] = "S011"; + + for (uint8_t i = 0; i < bcf.n_samples(); i++) { + snprintf(samp_name, 5, "S%02u", i+1); + EXPECT_STREQ(s[i].c_str(), samp_name); + } +} + + +TEST(TestReadBcf, LoadRecord) { + + bcfio::ReadBcf bcf = bcfio::open(VCF_NAME, "r"); + bcfio::BcfFloatRecord rec {}; + + bcf.next_record(&rec, "HD"); + + int32_t k_founders = bcf.k_fmt("HD"); + EXPECT_FALSE(k_founders <= 0); + EXPECT_EQ(rec.size(), bcf.n_samples() * static_cast(k_founders)); + EXPECT_EQ(bcf.n_samples(), rec.nrows()); + EXPECT_EQ(static_cast(k_founders), rec.ncols()); + EXPECT_TRUE(rec.is_snp()); + + // EXPECT_EQ(record.chrom(), "chr12"); + // EXPECT_EQ(record.pos(), 788); + // EXPECT_EQ(record.id(), "."); + // EXPECT_EQ(record.ref(), 'A'); + // EXPECT_EQ(record.alt(), 'G'); + // EXkECT_EQ(record.qual(), "."); + // EXPECT_EQ(record.filter(), "PASS"); + // EXPECT_EQ(record.info(), "EAF=0.00228;INFO_SCORE=1;HWE=1;ERC=0.01949;EAC=7.94153;PAF=0.00245;REF_PANEL=0"); + // EXPECT_EQ(record.format(), "GT:GP:DS:HD"); +} + + + +struct Buff { + Buff(const uint32_t size_in) + : size(size_in), array(new char[size]) { reset(); }; + ~Buff() { if (array) delete[] array; }; + + void reset() { + std::memset(array, '\0', size); + } + + uint32_t size; + char* array; +}; + +struct FloatArray { + FloatArray(const uint32_t size_in) + : size(size_in), array(new float[size]) { reset(); }; + ~FloatArray() { if (array) delete[] array; }; + + void reset() { + for (uint32_t i = 0; i < size; i++) + array[i] = static_cast(0); + } + uint32_t size; + float *array; +}; -// TEST(TestHaplotypeVCFParser, LoadRecord) { -// -// HaplotypeVcfParser vcf { VCF_NAME }; -// -// HaplotypeDataRecord record { vcf.n_samples(), vcf.k_haps() }; -// -// bool record_loaded { false }; -// record_loaded = vcf.load_record(record); -// -// EXPECT_TRUE(record_loaded); -// -// EXPECT_EQ(record.chrom(), "chr12"); -// EXPECT_EQ(record.pos(), 788); -// EXPECT_EQ(record.id(), "."); -// EXPECT_EQ(record.ref(), 'A'); -// EXPECT_EQ(record.alt(), 'G'); -// EXPECT_EQ(record.qual(), "."); -// EXPECT_EQ(record.filter(), "PASS"); -// EXPECT_EQ(record.info(), "EAF=0.00228;INFO_SCORE=1;HWE=1;ERC=0.01949;EAC=7.94153;PAF=0.00245;REF_PANEL=0"); -// EXPECT_EQ(record.format(), "GT:GP:DS:HD"); -// -// EXPECT_EQ(record(0,0), 1.004); -// EXPECT_EQ(record(0,1), 0); -// EXPECT_EQ(record(0,2),0.002); -// EXPECT_EQ(record(0,3),0.001); -// EXPECT_EQ(record(0,4),0); -// EXPECT_EQ(record(0,5),0.991); -// EXPECT_EQ(record(0,6), 0.001); -// EXPECT_EQ(record(0,7), 0.002); -// -// EXPECT_EQ(record(1,0), 0.001); -// EXPECT_EQ(record(1,1), 0); -// EXPECT_EQ(record(1,2),0); -// EXPECT_EQ(record(1,3),0.989); -// EXPECT_EQ(record(1,4),0.005); -// EXPECT_EQ(record(1,5),0.005); -// EXPECT_EQ(record(1,6), 1); -// EXPECT_EQ(record(1,7), 0); -// -// EXPECT_EQ(record(2,0), 0.998); -// EXPECT_EQ(record(2,1), 0); -// EXPECT_EQ(record(2,2),0.001); -// EXPECT_EQ(record(2,3),0); -// EXPECT_EQ(record(2,4),0); -// EXPECT_EQ(record(2,5),0); -// EXPECT_EQ(record(2,6), 1); -// EXPECT_EQ(record(2,7), 0); -// -// EXPECT_EQ(record(3,0), 0.84); -// EXPECT_EQ(record(3,1), 0); -// EXPECT_EQ(record(3,2),0); -// EXPECT_EQ(record(3,3),0); -// EXPECT_EQ(record(3,4),0); -// EXPECT_EQ(record(3,5),0); -// EXPECT_EQ(record(3,6), 1.159); -// EXPECT_EQ(record(3,7), 0); -// -// EXPECT_EQ(record(10,0), 0.592); -// EXPECT_EQ(record(10,1), 0); -// EXPECT_EQ(record(10,2),1); -// EXPECT_EQ(record(10,3),0); -// EXPECT_EQ(record(10,4),0); -// EXPECT_EQ(record(10,5),0); -// EXPECT_EQ(record(10,6),0); -// EXPECT_EQ(record(10,7),0.407); -// -// // load second record -// record_loaded = vcf.load_record(record); -// EXPECT_EQ(record.chrom(), "chr12"); -// EXPECT_EQ(record.pos(), 1321); -// EXPECT_EQ(record.id(), "."); -// EXPECT_EQ(record.ref(), 'A'); -// EXPECT_EQ(record.alt(), 'C'); -// EXPECT_EQ(record.qual(), "."); -// EXPECT_EQ(record.filter(), "PASS"); -// EXPECT_EQ(record.info(), "EAF=0.01487;INFO_SCORE=0.17212;HWE=1;ERC=1.33325;EAC=116.998;PAF=0.01127;REF_PANEL=0"); -// EXPECT_EQ(record.format(), "GT:GP:DS:HD"); -// -// EXPECT_EQ(record(0,0),1.004); -// EXPECT_EQ(record(0,1), 0); -// EXPECT_EQ(record(0,2),0.002); -// EXPECT_EQ(record(0,3),0.001); -// EXPECT_EQ(record(0,4),0); -// EXPECT_EQ(record(0,5),0.991); -// EXPECT_EQ(record(0,6), 0.001); -// EXPECT_EQ(record(0,7), 0.002); -// -// EXPECT_EQ(record(10,0), 0.592); -// EXPECT_EQ(record(10,1), 0); -// EXPECT_EQ(record(10,2),1); -// EXPECT_EQ(record(10,3),0); -// EXPECT_EQ(record(10,4),0); -// EXPECT_EQ(record(10,5),0); -// EXPECT_EQ(record(10,6),0); -// EXPECT_EQ(record(10,7),0.407); -// -// } + +int get_sample_truth_vals(FILE* fid, + FloatArray* data, + Buff* buff) { + + size_t buff_idx = 0; + size_t data_idx = 0; + int c; + while ((c = fgetc(fid)) != EOF) { + + if (c == ',' || c == '\n') { + if (buff_idx >= buff->size-1) + return -1; + buff->array[buff_idx] = '\0'; + + if (data_idx >= data->size) + return -1; + + data->array[data_idx++] = atof(buff->array); + buff_idx = 0; + buff->reset(); + + if (c == '\n') break; + + continue; + } + + buff->array[buff_idx++] = c; + } + + return 0; +} + + + +TEST(TestReadBcf, HDRecordValue) { + + bcfio::ReadBcf bcf = bcfio::open(VCF_NAME, "r"); + bcfio::BcfFloatRecord rec {}; + + Buff buff_fname { 100 }; + Buff buff_data { 1000 }; + FloatArray data { static_cast(bcf.k_fmt("HD")) }; + + FILE* fid; + // iterate positions + size_t pos = 1; + int status; + while (bcf.next_record(&rec, "HD") == 0) { + + snprintf(buff_fname.array, + buff_fname.size, + "tests/hd_%02zu.csv", pos++); + + fid = fopen(buff_fname.array, "r"); + + EXPECT_EQ(rec.nrows(), bcf.n_samples()); + EXPECT_EQ(rec.ncols(), static_cast(bcf.k_fmt("HD"))); + + // loop over samples + for (uint64_t i = 0; i < rec.nrows(); i++) { + + status = get_sample_truth_vals(fid, &data, &buff_data); + if (status != 0) + printf("\n\nERROR\n\n"); + + // loop over haplotypes + for (uint64_t j = 0; j < rec.ncols(); j++) + EXPECT_EQ(data.array[j], rec.get(i, j).value()); + } + fclose(fid); + } +} diff --git a/tests/test_matrix.cpp b/tests/test_matrix.cpp deleted file mode 100644 index c95575a..0000000 --- a/tests/test_matrix.cpp +++ /dev/null @@ -1,86 +0,0 @@ - -#include -#include -#include - - -TEST(TestGrm, Init) { - size_t n_row { 3 }; - size_t m_col { 2 }; - Grm a { n_row, m_col }; - std::array dims { a.dims() }; - EXPECT_EQ(dims[0], n_row); - EXPECT_EQ(dims[1], m_col); - - for (size_t i = 0; i < n_row; i++) - for (size_t j = 0; j < m_col; j++) - EXPECT_EQ(a(i, j), 0); - - n_row = 0; - - EXPECT_THROW({ - size_t n_row = 0; - size_t m_col = 2; - Grm b(n_row, m_col); - }, - std::runtime_error); - - EXPECT_ANY_THROW({ - size_t n_row = -1; - size_t m_col = 2; - Grm b(n_row, m_col); - }); - - EXPECT_THROW({Grm b(1, 0);}, std::runtime_error); - EXPECT_ANY_THROW({Grm b(1, -1);}); -} - - - -TEST(TestGrm, Vals) { - size_t n_row { 3 }; - size_t m_col { 5 }; - - Grm a { n_row, m_col }; - - std::array dims { a.dims() }; - - double x = { 1 }; - for (size_t i = 0; i < dims[0]; i++) - for (size_t j = 0; j < dims[1]; j++) - a(i, j) = x++; - - x = 1; - for (size_t i = 0; i < dims[0]; i++) - for (size_t j = 0; j < dims[1]; j++) - EXPECT_FLOAT_EQ(a(i, j), x++); -} - - -TEST(TestGrm, OutOfBounds) { - size_t n_row { 3 }; - size_t m_col { 5 }; - - Grm a { n_row, m_col }; - - EXPECT_THROW({ a(4, 3); }, std::runtime_error); - EXPECT_THROW({ a(3, 5); }, std::runtime_error); - EXPECT_THROW({ a(3, 2); }, std::runtime_error); - EXPECT_THROW({ a(2, 5); }, std::runtime_error); - EXPECT_THROW({ a(-2, 4); }, std::runtime_error); - -} - - -TEST(TestGrm, DimAndSize) { - size_t n_row { 3 }; - size_t m_col { 5 }; - - Grm a { n_row, m_col }; - - std::array dims { a.dims() }; - EXPECT_EQ(dims[0], n_row); - EXPECT_EQ(dims[1], m_col); - - EXPECT_EQ(a.size(), n_row * m_col); -}