hoo2/PDS
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

A V0 implementation tested against matlab

Browse Source 
Supports:
 - HDF5 file load and store
 - Precision timing of the process to stdout
 - logging (verbose mode) to stdout
 - Command line arguments and help
This commit is contained in:
hoo2
2024-11-18 00:29:04 +02:00
parent 7c88260900
commit f591127e16
11 changed files with 1357 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
homework_1/inc/config.h
+75
View File
@@ -0,0 +1,75 @@
/*!
* \file config,h
* \brief Build configuration file.
*
* \author
* Christos Choutouridis AEM:8997
* <cchoutou@ece.auth.gr>
*/
#ifndef CONFIG_H_
#define CONFIG_H_
#include <iostream>
#include <string>
#include <matrix.hpp>
// HDF5 supported types
enum class HDF5_type {
SCHAR, CHAR, SHORT, USHORT, INT, UINT, LONG, ULONG, LLONG, ULLONG, FLOAT, DOUBLE
};
/*
* Defines for different version of the exercise
*/
#define V0 0
#define V1 1
// Fail-safe version selection
#if !defined CODE_VERSION
#define CODE_VERSION V1
#endif
// matrix alias template dispatcher based on pre-define flag from compiler (see Makefile)
#if CODE_VERSION == V0
#define NAMESPACE_VERSION using namespace v0
using MatrixDst = mtx::Matrix<double>;
using MatrixIdx = mtx::Matrix<uint32_t>;
static constexpr HDF5_type DstHDF5Type = HDF5_type::DOUBLE;
static constexpr HDF5_type IdxHDF5Type = HDF5_type::INT;
#elif CODE_VERSION == V1
#define NAMESPACE_VERSION using namespace v1
using MatrixDst = mtx::Matrix<double>;
using MatrixIdx = mtx::Matrix<uint32_t>;
static constexpr HDF5_type DstHDF5Type = HDF5_type::DOUBLE;
static constexpr HDF5_type IdxHDF5Type = HDF5_type::INT;
#endif
//! enumerator for output handling
enum class StdOutputMode{ STD, FILE };
/*!
* Session option for each invocation of the executable
*/
struct session_t {
std::string corpusMtxFile {}; //!< corpus matrix file name in HDF5 format
std::string corpusDataSet {}; //!< corpus dataset name in HDF5 matrix file
std::string queryMtxFile {}; //!< optional query matrix file name in HDF5 format
std::string queryDataSet {}; //!< optional query dataset name in HDF5 matrix file
bool queryMtx {false}; //!< Flag to indicate that there is a separate query matrix
size_t k {1}; //!< The number of nearest neighbors to find
std::string outMtxFile {"out.hdf5"}; //!< output matrix file name in HDF5 format
std::string outMtxIdxDataSet {"/Idx"}; //!< Index output dataset name in HDF5 matrix file
std::string outMtxDstDataSet {"/Dst"}; //!< Distance output dataset name in HDF5 matrix file
std::size_t max_threads {}; //!< Maximum threads to use
bool timing {false}; //!< Enable timing prints of the program
bool verbose {false}; //!< Flag to enable verbose output to stdout
};
extern session_t session;
#endif /* CONFIG_H_ */
homework_1/inc/utils.hpp
+224
View File
@@ -0,0 +1,224 @@
/**
* \file utils.hpp
* \brief Utilities header
*
* \author
* Christos Choutouridis AEM:8997
* <cchoutou@ece.auth.gr>
*/
#ifndef UTILS_HPP_
#define UTILS_HPP_
#include <iostream>
#include <chrono>
#include <unistd.h>
#include <hdf5.h>
#include <matrix.hpp>
#include <config.h>
/*!
* A Logger for entire program.
*/
struct Log {
struct Endl {} endl; //!< a tag object to to use it as a new line request.
//! We provide logging via << operator
template<typename T>
Log& operator<< (T&& t) {
if (session.verbose) {
if (line_) {
std::cout << "[Log]: " << t;
line_ = false;
}
else
std::cout << t;
}
return *this;
}
// overload for special end line handling
Log& operator<< (Endl e) { (void)e;
if (session.verbose) {
std::cout << '\n';
line_ = true;
}
return *this;
}
private:
bool line_ {true};
};
extern Log logger;
/*!
* A small timing utility based on chrono.
*/
struct Timing{
using Tpoint = std::chrono::steady_clock::time_point;
using microseconds = std::chrono::microseconds;
using milliseconds = std::chrono::milliseconds;
using seconds = std::chrono::seconds;
//! tool to mark the starting point
Tpoint start () noexcept { return start_ = std::chrono::steady_clock::now(); }
//! tool to mark the ending point
Tpoint stop () noexcept { return stop_ = std::chrono::steady_clock::now(); }
auto dt () noexcept {
return std::chrono::duration_cast<std::chrono::microseconds>(stop_ - start_).count();
}
//! tool to print the time interval
void print_dt (const char* what) noexcept {
if (session.timing) {
auto t = stop_ - start_;
if (std::chrono::duration_cast<microseconds>(t).count() < 10000)
std::cout << "[Timing]: " << what << ": " << std::to_string(std::chrono::duration_cast<microseconds>(t).count()) << " [usec]\n";
else if (std::chrono::duration_cast<milliseconds>(t).count() < 10000)
std::cout << "[Timing]: " << what << ": " << std::to_string(std::chrono::duration_cast<milliseconds>(t).count()) << " [msec]\n";
else
std::cout << "[Timing]: " << what << ": " << std::to_string(std::chrono::duration_cast<seconds>(t).count()) << " [sec]\n";
}
}
private:
Tpoint start_;
Tpoint stop_;
};
struct Mtx {
template<typename MatrixType, HDF5_type Type>
static void load(const std::string& filename, const std::string& dataset, MatrixType& matrix) {
hid_t file_id{}, dataset_id{}, dataspace_id{};
herr_t read_st;
do {
// Open file
logger << "Load HDF5 file: " << filename << " Dataset: " << dataset << "...";
if ((file_id = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
break;
// Open dataset
if ((dataset_id = H5Dopen2(file_id, dataset.c_str(), H5P_DEFAULT)) < 0)
break;
// Get dataspace and allocate memory for read buffer
if ((dataspace_id = H5Dget_space(dataset_id)) < 0)
break;
hsize_t dims[2];
H5Sget_simple_extent_dims(dataspace_id, dims, NULL);
matrix.resize(dims[0], dims[1]);
// Read the dataset
// ToDo: Come up with a better way to do this
if constexpr (Type == HDF5_type::DOUBLE) {
if ((read_st = H5Dread(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) < 0)
break;
}
else if (Type == HDF5_type::FLOAT) {
if ((read_st = H5Dread(dataset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) < 0)
break;
}
else if (Type == HDF5_type::UINT) {
if ((read_st = H5Dread(dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) < 0)
break;
}
else if (Type == HDF5_type::INT) {
if ((read_st = H5Dread(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) < 0)
break;
}
// Done
H5Dclose(dataset_id);
H5Sclose(dataspace_id);
H5Fclose(file_id);
logger << " Done" << logger.endl;
return;
} while (0);
// Error: close everything (if possible) and return false
H5Dclose(dataset_id);
H5Sclose(dataspace_id);
H5Fclose(file_id);
throw std::runtime_error("Cannot store to " + filename + " dataset:" + dataset + '\n');
}
template<typename MatrixType, HDF5_type Type>
static void store(const std::string& filename, const std::string& dataset, MatrixType& matrix) {
hid_t file_id{}, dataset_id{}, dataspace_id{};
herr_t write_st;
do {
// Try to open the file in read-write mode
logger << "Store HDF5 file: " << filename << " Dataset: " << dataset << "...";
if (access(session.outMtxFile.c_str(), F_OK) == 0){
if ((file_id = H5Fopen(filename.c_str(), H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
break;
}
else {
if ((file_id = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
break;
}
// Create the dataspace for the dataset
hsize_t dims[] = { matrix.rows(), matrix.columns() };
if ((dataspace_id = H5Screate_simple(2, dims, NULL)) < 0)
break;
// ToDo: Come up with a better way to do this
if constexpr (Type == HDF5_type::DOUBLE) {
// Create the dataset with default properties
if ((dataset_id = H5Dcreate2(
file_id, dataset.c_str(), H5T_NATIVE_DOUBLE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
break;
// Write the data to the dataset
if ((write_st = H5Dwrite(dataset_id, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) <0 )
break;
}
else if (Type == HDF5_type::FLOAT) {
// Create the dataset with default properties
if ((dataset_id = H5Dcreate2(
file_id, dataset.c_str(), H5T_NATIVE_FLOAT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
break;
// Write the data to the dataset
if ((write_st = H5Dwrite(dataset_id, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) <0 )
break;
}
else if (Type == HDF5_type::UINT) {
// Create the dataset with default properties
if ((dataset_id = H5Dcreate2(
file_id, dataset.c_str(), H5T_NATIVE_UINT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
break;
// Write the data to the dataset
if ((write_st = H5Dwrite(dataset_id, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) <0 )
break;
}
else if (Type == HDF5_type::INT) {
// Create the dataset with default properties
if ((dataset_id = H5Dcreate2(
file_id, dataset.c_str(), H5T_NATIVE_INT, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
break;
// Write the data to the dataset
if ((write_st = H5Dwrite(dataset_id, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, matrix.data())) <0 )
break;
}
// Close the dataset, dataspace, and file
H5Dclose(dataset_id);
H5Sclose(dataspace_id);
H5Fclose(file_id);
logger << " Done" << logger.endl;
return;
} while (0);
// Error: close everything (if possible) and return false
H5Dclose(dataset_id);
H5Sclose(dataspace_id);
H5Fclose(file_id);
throw std::runtime_error("Cannot store " + filename + " with dataset:" + dataset +'\n');
}
};
#endif /* UTILS_HPP_ */
homework_1/inc/v0.hpp
+119
View File
@@ -0,0 +1,119 @@
/**
* \file v0.hpp
* \brief
*
* \author
* Christos Choutouridis AEM:8997
* <cchoutou@ece.auth.gr>
*/
#ifndef V0_HPP_
#define V0_HPP_
#include <cblas.h>
#include <cmath>
#include <vector>
#include <algorithm>
#include <matrix.hpp>
#include <config.h>
namespace v0 {
/*!
* Function to compute squared Euclidean distances
*
* \fn void pdist2(const double*, const double*, double*, int, int, int)
* \param X m x d matrix (Column major)
* \param Y n x d matrix (Column major)
* \param D2 m x n matrix to store distances (Column major)
* \param m number of rows in X
* \param n number of rows in Y
* \param d number of columns in both X and Y
*/
template<typename DataType>
void pdist2(const mtx::Matrix<DataType>& X, const mtx::Matrix<DataType>& Y, mtx::Matrix<DataType>& D2) {
int M = X.rows();
int N = Y.rows();
int d = X.columns();
// Compute the squared norms of each row in X and Y
std::vector<DataType> X_norms(M), Y_norms(N);
for (int i = 0; i < M ; ++i) {
X_norms[i] = cblas_ddot(d, X.data() + i * d, 1, X.data() + i * d, 1);
}
for (int j = 0; j < N ; ++j) {
Y_norms[j] = cblas_ddot(d, Y.data() + j * d, 1, Y.data() + j * d, 1);
}
// Compute -2 * X * Y'
cblas_dgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, d, -2.0, X.data(), d, Y.data(), d, 0.0, D2.data(), N);
// Step 3: Add the squared norms to each entry in D2
for (int i = 0; i < M ; ++i) {
for (int j = 0; j < N; ++j) {
D2.set(D2.get(i, j) + X_norms[i] + Y_norms[j], i, j);
//D2.set(std::max(D2.get(i, j), 0.0), i, j); // Ensure non-negative
D2.set(std::sqrt(D2.get(i, j)), i, j); // Take the square root of each
}
}
}
template<typename DataType, typename IndexType>
void quickselect(std::vector<std::pair<DataType, IndexType>>& vec, int k) {
std::nth_element(
vec.begin(),
vec.begin() + k,
vec.end(),
[](const std::pair<DataType, IndexType>& a, const std::pair<DataType, IndexType>& b) {
return a.first < b.first;
});
vec.resize(k); // Keep only the k smallest elements
}
/*!
* \param C Is a MxD matrix (Corpus)
* \param Q Is a NxD matrix (Query)
* \param k The number of nearest neighbors needed
* \param idx Is the Nxk matrix with the k indexes of the C points, that are
* neighbors of the nth point of Q
* \param dst Is the Nxk matrix with the k distances to the C points of the nth
* point of Q
*/
template<typename DataType, typename IndexType>
void knnsearch(const mtx::Matrix<DataType>& C, const mtx::Matrix<DataType>& Q, int k,
mtx::Matrix<IndexType>& idx,
mtx::Matrix<DataType>& dst) {
int M = C.rows();
int N = Q.rows();
mtx::Matrix<DataType> D(M, N);
pdist2(C, Q, D);
idx.resize(N, k);
dst.resize(N, k);
for (int j = 0; j < N; ++j) {
// Create a vector of pairs (distance, index) for the j-th query
std::vector<std::pair<DataType, IndexType>> dst_idx(M);
for (int i = 0; i < M; ++i) {
dst_idx[i] = {D.data()[i * N + j], i};
}
// Find the k smallest distances using quickSelectKSmallest
quickselect(dst_idx, k);
// Sort the k smallest results by distance for consistency
std::sort(dst_idx.begin(), dst_idx.end());
// Store the indices and distances
for (int i = 0; i < k; ++i) {
idx(j, i) = dst_idx[i].second;
dst(j, i) = dst_idx[i].first;
}
}
}
}
#endif /* V0_HPP_ */
homework_1/inc/v1.hpp
+16
View File
@@ -0,0 +1,16 @@
/**
* \file v0.hpp
* \brief
*
* \author
* Christos Choutouridis AEM:8997
* <cchoutou@ece.auth.gr>
*/
#ifndef V1_HPP_
#define V1_HPP_
#endif /* V1_HPP_ */