diff --git a/homework_1/Makefile b/homework_1/Makefile index 4333f8d..66f5790 100644 --- a/homework_1/Makefile +++ b/homework_1/Makefile @@ -27,8 +27,8 @@ SRC_DIR_LIST := src gtest # Include directories list(space seperated). Makefile-relative path. INC_DIR_LIST := inc \ src \ + /usr/include/hdf5/serial/ \ gtest \ - /usr/include/hdf5/serial/ # Libs/MATLAB/R2019b/include/ \ # Exclude files list(space seperated). Filenames only. @@ -179,7 +179,7 @@ hpc-clean: rm hpc-results/post # -# ================ Local via docker build rules ================= +# ================ Local (and/or) via docker build rules ================= # # examples: # make IMAGE=hpcimage v0 @@ -190,12 +190,6 @@ local_v0: CXXFLAGS := $(DEB_CXXFLAGS) -DCODE_VERSION=0 local_v0: TARGET := local_v0 local_v0: $(BUILD_DIR)/$(TARGET) cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) - -local_v0_opt: CFLAGS := $(REL_CFLAGS) -DCODE_VERSION=0 -local_v0_opt: CXXFLAGS := $(REL_CXXFLAGS) -DCODE_VERSION=0 -local_v0_opt: TARGET := local_v0_opt -local_v0_opt: $(BUILD_DIR)/$(TARGET) - cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) local_v1: CFLAGS := $(DEB_CFLAGS) -DCODE_VERSION=1 local_v1: CXXFLAGS := $(DEB_CXXFLAGS) -DCODE_VERSION=1 @@ -203,6 +197,21 @@ local_v1: TARGET := local_v1 local_v1: $(BUILD_DIR)/$(TARGET) cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) +local_v1_omp: CFLAGS := $(DEB_CFLAGS) -fopenmp -DCODE_VERSION=1 -DOMP +local_v1_omp: CXXFLAGS := $(DEB_CXXFLAGS) -fopenmp -DCODE_VERSION=1 -DOMP +local_v1_omp: LDFLAGS += -fopenmp +local_v1_omp: TARGET := local_v1_omp +local_v1_omp: $(BUILD_DIR)/$(TARGET) + cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) + + +local_v1_pth: CFLAGS := $(DEB_CFLAGS) -DCODE_VERSION=1 -DPTHREADS +local_v1_pth: CXXFLAGS := $(DEB_CXXFLAGS) -DCODE_VERSION=1 -DPTHREADS +local_v1_pth: TARGET := local_v1_pth +local_v1_pth: $(BUILD_DIR)/$(TARGET) + cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) + + v0: DOCKER := $(DOCKER_CMD) v0: CFLAGS := $(REL_CFLAGS) -DCODE_VERSION=0 v0: CXXFLAGS := $(REL_CXXFLAGS) -DCODE_VERSION=0 @@ -215,7 +224,7 @@ v1_cilk: CXX := /usr/local/OpenCilk-9.0.1-Linux/bin/clang++ v1_cilk: CFLAGS := $(REL_CFLAGS) -fcilkplus -DCODE_VERSION=1 -DCILK v1_cilk: CXXFLAGS := $(REL_CXXFLAGS) -fcilkplus -DCODE_VERSION=1 -DCILK v1_cilk: LDFLAGS += -fcilkplus -v1_cilk: TARGET := knnsearch_cilkv1 +v1_cilk: TARGET := knnsearch_v1_cilk v1_cilk: $(BUILD_DIR)/$(TARGET) cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) @@ -223,10 +232,17 @@ v1_omp: DOCKER := $(DOCKER_CMD) v1_omp: CFLAGS := $(REL_CFLAGS) -fopenmp -DCODE_VERSION=1 -DOMP v1_omp: CXXFLAGS := $(REL_CXXFLAGS) -fopenmp -DCODE_VERSION=1 -DOMP v1_omp: LDFLAGS += -fopenmp -v1_omp: TARGET := knnsearch_ompv1 +v1_omp: TARGET := knnsearch_v1_omp v1_omp: $(BUILD_DIR)/$(TARGET) cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) +v1_pth: DOCKER := $(DOCKER_CMD) +v1_pth: CFLAGS := $(REL_CFLAGS) -DCODE_VERSION=1 -DPTHREADS +v1_pth: CXXFLAGS := $(REL_CXXFLAGS) -DCODE_VERSION=1 -DPTHREADS +v1_pth: TARGET := knnsearch_v1_pth +v1_pth: $(BUILD_DIR)/$(TARGET) + cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) + v1: DOCKER := $(DOCKER_CMD) v1: CFLAGS := $(REL_CFLAGS) -DCODE_VERSION=1 v1: CXXFLAGS := $(REL_CXXFLAGS) -DCODE_VERSION=1 @@ -240,7 +256,12 @@ tests: TARGET := tests tests: $(BUILD_DIR)/$(TARGET) cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) - +tests_rel: CFLAGS := $(REL_CFLAGS) -DCODE_VERSION=0 -DTESTING +tests_rel: CXXFLAGS := $(REL_CXXFLAGS) -DCODE_VERSION=0 -DTESTING +tests_rel: TARGET := tests +tests_rel: $(BUILD_DIR)/$(TARGET) + cp $(BUILD_DIR)/$(TARGET) out/$(TARGET) + # # ========= Inside CSAL Image build rules =========== # diff --git a/homework_1/src/tests.cpp b/homework_1/gtest/tests.cpp similarity index 55% rename from homework_1/src/tests.cpp rename to homework_1/gtest/tests.cpp index 6147ee6..caa91cc 100644 --- a/homework_1/src/tests.cpp +++ b/homework_1/gtest/tests.cpp @@ -19,6 +19,8 @@ using matrix_t = mtx::Matrix; +extern void loadMtx(MatrixDst& Corpus, MatrixDst& Query); +extern void storeMtx(MatrixIdx& Idx, MatrixDst& Dst); // ===================================== // C1, Q1 @@ -140,11 +142,44 @@ TEST(Tv0_UT, pdist2_test2) { } +TEST(Tv0_UT, pdist2_test3) { + + mtx::Matrix D2_exp(16, 16, { + 0, 0.7433, 0.6868, 0.8846, 0.6342, 0.4561, 0.5118, 0.6341, 0.5461, 0.7322, 0.6974, 0.4330, 0.7028, 0.6303, 0.6826, 0.4179, + 0.7433, 0, 0.3400, 0.4555, 0.4207, 0.9736, 0.9690, 0.7386, 1.1055, 0.5462, 0.5345, 0.6576, 0.8677, 1.0291, 0.5393, 0.8106, + 0.6868, 0.3400, 0, 0.5380, 0.6268, 0.9512, 1.0234, 0.8403, 0.9843, 0.8187, 0.3091, 0.7829, 0.5759, 0.9411, 0.7239, 0.9186, + 0.8846, 0.4555, 0.5380, 0, 0.6796, 1.1672, 1.0460, 1.1016, 1.1139, 0.7542, 0.6480, 0.9304, 1.0568, 1.3482, 0.8316, 0.9750, + 0.6342, 0.4207, 0.6268, 0.6796, 0, 0.9267, 0.8772, 0.4847, 0.9317, 0.4093, 0.8351, 0.4215, 0.9736, 0.9007, 0.5999, 0.5291, + 0.4561, 0.9736, 0.9512, 1.1672, 0.9267, 0, 0.3903, 0.7795, 0.9308, 0.8429, 0.8436, 0.5672, 0.9284, 0.7064, 0.6435, 0.5975, + 0.5118, 0.9690, 1.0234, 1.0460, 0.8772, 0.3903, 0, 0.8920, 0.9253, 0.7060, 0.9427, 0.5728, 1.1515, 0.9907, 0.6471, 0.4811, + 0.6341, 0.7386, 0.8403, 1.1016, 0.4847, 0.7795, 0.8920, 0, 0.9824, 0.6416, 0.9844, 0.3398, 0.9355, 0.5428, 0.6536, 0.5309, + 0.5461, 1.1055, 0.9843, 1.1139, 0.9317, 0.9308, 0.9253, 0.9824, 0, 1.1517, 1.0541, 0.8746, 0.8506, 0.8777, 1.2036, 0.7607, + 0.7322, 0.5462, 0.8187, 0.7542, 0.4093, 0.8429, 0.7060, 0.6416, 1.1517, 0, 0.9106, 0.4245, 1.2071, 1.0738, 0.3745, 0.5170, + 0.6974, 0.5345, 0.3091, 0.6480, 0.8351, 0.8436, 0.9427, 0.9844, 1.0541, 0.9106, 0, 0.8647, 0.5941, 0.9954, 0.7148, 0.9876, + 0.4330, 0.6576, 0.7829, 0.9304, 0.4215, 0.5672, 0.5728, 0.3398, 0.8746, 0.4245, 0.8647, 0, 0.9590, 0.6782, 0.4586, 0.2525, + 0.7028, 0.8677, 0.5759, 1.0568, 0.9736, 0.9284, 1.1515, 0.9355, 0.8506, 1.2071, 0.5941, 0.9590, 0, 0.6838, 1.0517, 1.0675, + 0.6303, 1.0291, 0.9411, 1.3482, 0.9007, 0.7064, 0.9907, 0.5428, 0.8777, 1.0738, 0.9954, 0.6782, 0.6838, 0, 0.9482, 0.7937, + 0.6826, 0.5393, 0.7239, 0.8316, 0.5999, 0.6435, 0.6471, 0.6536, 1.2036, 0.3745, 0.7148, 0.4586, 1.0517, 0.9482, 0, 0.6345, + 0.4179, 0.8106, 0.9186, 0.9750, 0.5291, 0.5975, 0.4811, 0.5309, 0.7607, 0.5170, 0.9876, 0.2525, 1.0675, 0.7937, 0.6345, 0 + }); + + mtx::Matrix D (16,16); + + v0::pdist2(C2, C2, D); + + for (size_t i = 0 ; i< D.rows() ; ++i) + for (size_t j = 0 ; j D(i, j), true); + EXPECT_EQ (D2_exp.get(i ,j) - 0.01 < D(i, j), true); + } +} + + /* * ========================================== * v0::knn */ -TEST(Tv0_UT, knn_test1) { +TEST(Tv0_UT, knn_v0_test1) { size_t k = 3; mtx::Matrix Idx_exp(5, k, { 5, 8, 9, @@ -177,7 +212,7 @@ TEST(Tv0_UT, knn_test1) { } -TEST(Tv0_UT, knn_test2) { +TEST(Tv0_UT, knn_v0_test2) { size_t k = 3; mtx::Matrix Idx_exp(8, k, { 14, 13, 1, @@ -220,7 +255,46 @@ TEST(Tv0_UT, knn_test2) { * ========================================== * v1::knn */ -TEST(Tv1_UT, knn_test1) { +TEST(Tv1_UT, knn_v1_1slice) { + size_t k = 3; + mtx::Matrix Idx_exp(8, k, { + 14, 13, 1, + 15, 10, 12, + 14, 8, 12, + 4, 1, 3, + 8, 12, 5, + 4, 3, 2, + 10, 2, 4, + 1, 11, 9 + }); + + mtx::Matrix Dst_exp(8, k, { + 0.1939, 0.5768, 0.6020, + 0.2708, 0.3686, 0.3740, + 0.2684, 0.3103, 0.5277, + 0.4709, 0.6050, 0.6492, + 0.4397, 0.6842, 0.7074, + 0.3402, 0.4360, 0.4475, + 0.3708, 0.4037, 0.4417, + 0.6352, 0.6653, 0.6758 + }); + + mtx::Matrix Idx(8, k); + mtx::Matrix Dst(8, k); + + v1::knnsearch(C2, Q2, 1, k, k, Idx, Dst); + + + for (size_t i = 0 ; i< Idx.rows() ; ++i) + for (size_t j = 0 ; j Dst(i, j), true); + EXPECT_EQ (Dst_exp.get(i ,j) - 0.01 < Dst(i, j), true); + } + +} + +TEST(Tv1_UT, knn_v1_2slice) { size_t k = 3; mtx::Matrix Idx_exp(8, k, { 14, 13, 1, @@ -247,7 +321,7 @@ TEST(Tv1_UT, knn_test1) { mtx::Matrix Idx(8, k); mtx::Matrix Dst(8, k); - v1::knnsearch(C2, Q2, 0, k, k, Idx, Dst); + v1::knnsearch(C2, Q2, 2, k, k, Idx, Dst); for (size_t i = 0 ; i< Idx.rows() ; ++i) @@ -260,7 +334,7 @@ TEST(Tv1_UT, knn_test1) { } // all-to-all -TEST(Tv1_UT, knn_test2) { +TEST(Tv1_UT, knn_v1_4slice) { size_t k = 3; mtx::Matrix Idx_exp(16, k, { 1, 16, 12, @@ -303,7 +377,7 @@ TEST(Tv1_UT, knn_test2) { mtx::Matrix Idx(16, k); mtx::Matrix Dst(16, k); - v1::knnsearch(C2, C2, 0, k, k, Idx, Dst); + v1::knnsearch(C2, C2, 4, k, k, Idx, Dst); for (size_t i = 0 ; i< Idx.rows() ; ++i) @@ -315,3 +389,130 @@ TEST(Tv1_UT, knn_test2) { } + + +/* + * ============== Live hdf5 tests =============== + * + * In order to run these test we need the followin hdf5 files in ./mtx directory: + * + * - fasion-mnist-784-euclidean.hdf5 + * - mnist-784-euclidean.hdf5 + * - sift-128-euclidean.hdf5 + * - gist-960-euclidean.hdf5 + * + */ + +TEST(Tlive_UT, knn_v0_sift_test) { + // Instantiate matrixes + MatrixDst Corpus; + MatrixDst Query; + MatrixIdx Idx; + MatrixDst Dst; + + // setup environment + session.corpusMtxFile = "mtx/sift-128-euclidean.hdf5"; + session.corpusDataSet = "/test"; + session.queryMtx = false; + session.k = 100; + size_t m = session.k; + session.timing = true; + session.outMtxFile = "test/knn_v0.hdf5"; + + + loadMtx(Corpus, Query); + + // Prepare output memory (There is no Query, so from Corpus + Idx.resize(Corpus.rows(), session.k); + Dst.resize(Corpus.rows(), session.k); + + v0::knnsearch(Corpus, Corpus, 0, session.k, m, Idx, Dst); + storeMtx(Idx, Dst); + EXPECT_EQ(true, true); +} + + +TEST(Tlive_UT, knn_v1_sift_test_1slice) { + // Instantiate matrixes + MatrixDst Corpus; + MatrixDst Query; + MatrixIdx Idx; + MatrixDst Dst; + + // setup environment + session.corpusMtxFile = "mtx/sift-128-euclidean.hdf5"; + session.corpusDataSet = "/test"; + session.queryMtx = false; + session.k = 100; + size_t m = session.k; + session.timing = true; + session.outMtxFile = "test/knn_v1ser.hdf5"; + + + loadMtx(Corpus, Query); + + // Prepare output memory (There is no Query, so from Corpus + Idx.resize(Corpus.rows(), session.k); + Dst.resize(Corpus.rows(), session.k); + + v1::knnsearch(Corpus, Corpus, 0, session.k, m, Idx, Dst); + storeMtx(Idx, Dst); + EXPECT_EQ(true, true); +} + +TEST(Tlive_UT, knn_v1_sift_test_2slice) { + // Instantiate matrixes + MatrixDst Corpus; + MatrixDst Query; + MatrixIdx Idx; + MatrixDst Dst; + + // setup environment + session.corpusMtxFile = "mtx/sift-128-euclidean.hdf5"; + session.corpusDataSet = "/test"; + session.queryMtx = false; + session.k = 100; + size_t m = session.k; + session.timing = true; + session.outMtxFile = "test/knn_v1ser.hdf5"; + + + loadMtx(Corpus, Query); + + // Prepare output memory (There is no Query, so from Corpus + Idx.resize(Corpus.rows(), session.k); + Dst.resize(Corpus.rows(), session.k); + + v1::knnsearch(Corpus, Corpus, 2, session.k, m, Idx, Dst); + storeMtx(Idx, Dst); + EXPECT_EQ(true, true); +} + +TEST(Tlive_UT, knn_v1_sift_test_4slice) { + // Instantiate matrixes + MatrixDst Corpus; + MatrixDst Query; + MatrixIdx Idx; + MatrixDst Dst; + + // setup environment + session.corpusMtxFile = "mtx/sift-128-euclidean.hdf5"; + session.corpusDataSet = "/test"; + session.queryMtx = false; + session.k = 100; + size_t m = session.k; + session.timing = true; + session.outMtxFile = "test/knn_v1ser.hdf5"; + + + loadMtx(Corpus, Query); + + // Prepare output memory (There is no Query, so from Corpus + Idx.resize(Corpus.rows(), session.k); + Dst.resize(Corpus.rows(), session.k); + + v1::knnsearch(Corpus, Corpus, 4, session.k, m, Idx, Dst); + storeMtx(Idx, Dst); + EXPECT_EQ(true, true); +} + diff --git a/homework_1/inc/config.h b/homework_1/inc/config.h index 573ac32..34d3845 100644 --- a/homework_1/inc/config.h +++ b/homework_1/inc/config.h @@ -55,7 +55,9 @@ struct session_t { std::string outMtxFile {"out.hdf5"}; //!< output matrix file name in HDF5 format std::string outMtxIdxDataSet {"/Idx"}; //!< Index output dataset name in HDF5 matrix file std::string outMtxDstDataSet {"/Dst"}; //!< Distance output dataset name in HDF5 matrix file - std::size_t max_threads {}; //!< Maximum threads to use + std::size_t max_threads {0}; //!< Maximum threads to use + std::size_t slices {0}; //!< Slices/threads to use + std::size_t accuracy {100}; //!< The neighbor finding accuracy bool timing {false}; //!< Enable timing prints of the program bool verbose {false}; //!< Flag to enable verbose output to stdout }; diff --git a/homework_1/inc/matrix.hpp b/homework_1/inc/matrix.hpp index 809006c..33b3266 100644 --- a/homework_1/inc/matrix.hpp +++ b/homework_1/inc/matrix.hpp @@ -134,6 +134,9 @@ struct Matrix { Matrix& operator=(Matrix&& m) noexcept { moves(std::move(m)); return *this; } Matrix(const Matrix& m) = delete; //!< No copy ctor Matrix& operator=(const Matrix& m) = delete; //!< No copy + //Matrix(const Matrix& m); + //Matrix& operator=(const Matrix& m) { copy(m); } + //! @} //! \name Data exposure @@ -233,6 +236,11 @@ struct Matrix { // a basic serial iterator support DataType* data() noexcept { return data_; } + DataType* begin() noexcept { return data_; } + const DataType* begin() const noexcept { return data_; } + DataType* end() noexcept { return data_ + capacity(rows_, cols_); } + const DataType* end() const noexcept { return data_ + capacity(rows_, cols_); } + // IndexType begin_idx() noexcept { return 0; } // IndexType end_idx() noexcept { return capacity(rows_, cols_); } @@ -265,17 +273,19 @@ struct Matrix { std::swap(rows_, src.rows_); std::swap(cols_, src.cols_); } + private: //! move helper void moves(Matrix&& src) noexcept { - data_ = std::move(src.vector_storage_); - data_ = std::move(src.raw_storage_); - data_ = std::move(src.data_); - data_ = std::move(src.use_vector_); - rows_ = std::move(src.rows_); - cols_ = std::move(src.cols_); + vector_storage_ = std::move(src.vector_storage_); + raw_storage_ = std::move(src.raw_storage_); + data_ = std::move(src.data_); + use_vector_ = std::move(src.use_vector_); + rows_ = std::move(src.rows_); + cols_ = std::move(src.cols_); } + // Storage std::vector vector_storage_; //!< Internal storage (if used). DataType* raw_storage_; //!< External storage (if used). @@ -528,125 +538,6 @@ private: }; -template struct Matrix_view { }; - -/*! - * @struct Matrix_view - * @tparam MatrixType - */ -template