From 1be3ccbb9d51e9cc6b17fc473beac10e40a7af56 Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 19 Feb 2019 10:49:29 +0100 Subject: [PATCH 1/2] created persipubsub cpp --- cpp/.gitignore | 22 + cpp/CMakeLists.txt | 11 + cpp/include/filesystem.h | 153 +++ cpp/include/lmdb++.h | 1913 +++++++++++++++++++++++++++++++ cpp/include/prettyprint.hpp | 446 +++++++ cpp/include/scope_exit.hpp | 39 + cpp/include/tinyformat.h | 1003 ++++++++++++++++ cpp/include/unique_resource.hpp | 78 ++ cpp/src/CMakeLists.txt | 6 + cpp/src/control.h | 135 +++ cpp/src/environment.h | 76 ++ cpp/src/filesystem.cpp | 317 +++++ cpp/src/library.cpp | 42 + cpp/src/library.h | 48 + cpp/src/publisher.h | 52 + cpp/src/queue.cpp | 391 +++++++ cpp/src/queue.h | 180 +++ cpp/src/subscriber.h | 70 ++ cpp/test/CMakeLists.txt | 13 + cpp/test/test_queue.cpp | 281 +++++ 20 files changed, 5276 insertions(+) create mode 100644 cpp/.gitignore create mode 100644 cpp/CMakeLists.txt create mode 100755 cpp/include/filesystem.h create mode 100644 cpp/include/lmdb++.h create mode 100644 cpp/include/prettyprint.hpp create mode 100644 cpp/include/scope_exit.hpp create mode 100644 cpp/include/tinyformat.h create mode 100644 cpp/include/unique_resource.hpp create mode 100644 cpp/src/CMakeLists.txt create mode 100644 cpp/src/control.h create mode 100644 cpp/src/environment.h create mode 100644 cpp/src/filesystem.cpp create mode 100644 cpp/src/library.cpp create mode 100644 cpp/src/library.h create mode 100644 cpp/src/publisher.h create mode 100644 cpp/src/queue.cpp create mode 100644 cpp/src/queue.h create mode 100644 cpp/src/subscriber.h create mode 100644 cpp/test/CMakeLists.txt create mode 100644 cpp/test/test_queue.cpp diff --git a/cpp/.gitignore b/cpp/.gitignore new file mode 100644 index 0000000..05d2a2a --- /dev/null +++ b/cpp/.gitignore @@ -0,0 +1,22 @@ +# Distribution / packaging +cmake-build-debug/ +TODO + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# IDE +.idea + +# Archive +obsolete/ + +# Project specific \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..17aab6e --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.12) +project(persipubsub) + +set(CMAKE_CXX_STANDARD 17) + +include_directories(include) + +enable_testing() +add_subdirectory(src) +add_subdirectory(test) +add_test(NAME myTest COMMAND Test) \ No newline at end of file diff --git a/cpp/include/filesystem.h b/cpp/include/filesystem.h new file mode 100755 index 0000000..039266c --- /dev/null +++ b/cpp/include/filesystem.h @@ -0,0 +1,153 @@ +// Copyright (c) 2016 Parquery AG. All rights reserved. +// Created by mristin on 10/12/16. + +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +namespace pqry { +namespace filesystem { + +/** + * Simulates Unix ls -lt + * @param directory to list + * @return absolute paths sorted by modification time + */ +std::vector ls_lt(const boost::filesystem::path& directory); + +/** + * @param directory to be listed + * @return list of absolute paths to files contained in the directory + */ +std::vector listdir(const boost::filesystem::path& directory); + +/** + * @param path to the file + * @return modified time in nanoseconds since epoch + */ +int64_t modified_time(const boost::filesystem::path& path); + +/** + * simulates `mkdtemp` command + * @return path to the temporary directory + */ +boost::filesystem::path mkdtemp(); + +/** + * Reads a file to the string + * @param path to the file + * @return content of the file + */ +std::string read(const boost::filesystem::path& path); + +/** + * Writes the whole text to the file + * @param path to the file + * @param text to be written + */ +void write(const boost::filesystem::path& path, const std::string& text); + +/** + * Writes the `bytes` to the file + * @param path to write to + * @param bytes that are to be written + * @param size of the `bytes` + */ +void write(const boost::filesystem::path& path, const char* bytes, int size); + +/** + * Copies a source directory with all contents into a new target directory recursively + * @param source directory that gets copied + * @param dest directory where source gets copied to. Will be created if non-existent + */ +void copy_directory(const boost::filesystem::path& source, const boost::filesystem::path& dest); + +/** + * Unzips an archive. + * @param path to the archive + * @param dest_dir destination where the archive will be unzipped. Expects it to exist + */ +void unzip_archive(const boost::filesystem::path& path, const boost::filesystem::path& dest_dir); + +/** + * Uses POSIX flock() for named mutexes. Use an instance of this class to prevent multiple processes running at the + * same time. + * + * Mind that the instance keeps a file descriptor open during its lifetime. + * The implementation is crash-safe, i.e. if the program crashes, the system automatically releases the lock. + * Unlike boost::interprocess::file_lock, we do not have to create the file manually first. + * boost::interprocess:named_mutex did not work on Linux with boost 1.62 (marko tested it manually). + */ +class LockFileGuard { + public: + LockFileGuard() : locked_(false) {} + + /** + * Creates the lock file and writes the given PID to it + * @param lock_file to use for locking + * @param pid of the process + * @return true if the file could be successfully locked. + */ + bool lock(const boost::filesystem::path& lock_file, int pid); + + void unlock(); + + virtual ~LockFileGuard(); + + private: + bool locked_; + boost::filesystem::path lock_file_; + int fid_; +}; + +/** + * Wait for file to exist. Panics if timeout exceeded (0 means no timeout). + * @param path to the file + * @param timeout in seconds + */ +void wait_for_file(const boost::filesystem::path& path, unsigned int timeout); + +/** + * Expands path using wordexp + * @param path to be expanded + * @return expanded path + */ +boost::filesystem::path expand_path(const boost::filesystem::path& path); + +/** + * Temporary scope file, created from a path with a random suffix and ".tmp". + * If the file exists at the destruction, it will be deleted. + * All errors will be ignored in the destructor. + */ +class NamedTempfile { + public: + explicit NamedTempfile(const boost::filesystem::path& path); + + /** + * Deletes the temporary file if it was not renamed before. + */ + ~NamedTempfile(); + + const boost::filesystem::path& path() const { return tmp_pth_; } + + /** + * Renames the temporary file to the path passed in at the constructor. + */ + void rename(); + + private: + const boost::filesystem::path pth_; + const boost::filesystem::path tmp_pth_; + bool renamed_; +}; + +} // namespace filesystem +} // namespace pqry diff --git a/cpp/include/lmdb++.h b/cpp/include/lmdb++.h new file mode 100644 index 0000000..ab75f8c --- /dev/null +++ b/cpp/include/lmdb++.h @@ -0,0 +1,1913 @@ +/* This is free and unencumbered software released into the public domain. */ + +#ifndef LMDBXX_H +#define LMDBXX_H + +/** + * - C++11 wrapper for LMDB. + * + * @author Arto Bendiken + * @see https://sourceforge.net/projects/lmdbxx/ + */ + +#ifndef __cplusplus +#error " requires a C++ compiler" +#endif + +#if __cplusplus < 201103L +#if !defined(_MSC_VER) || _MSC_VER < 1900 +#error " requires a C++11 compiler (CXXFLAGS='-std=c++11')" +#endif // _MSC_VER check +#endif + +//////////////////////////////////////////////////////////////////////////////// + +#include /* for MDB_*, mdb_*() */ + +#ifdef LMDBXX_DEBUG +#include /* for assert() */ +#endif +#include /* for std::size_t */ +#include /* for std::snprintf() */ +#include /* for std::strlen() */ +#include /* for std::runtime_error */ +#include /* for std::string */ +#include /* for std::is_pod<> */ + +namespace lmdb { + using mode = mdb_mode_t; +} + +//////////////////////////////////////////////////////////////////////////////// +/* Error Handling */ + +namespace lmdb { + class error; + class logic_error; + class fatal_error; + class runtime_error; + class key_exist_error; + class not_found_error; + class corrupted_error; + class panic_error; + class version_mismatch_error; + class map_full_error; + class bad_dbi_error; +} + +/** + * Base class for LMDB exception conditions. + * + * @see http://symas.com/mdb/doc/group__errors.html + */ +class lmdb::error : public std::runtime_error { +protected: + const int _code; + +public: + /** + * Throws an error based on the given LMDB return code. + */ + [[noreturn]] static inline void raise(const char* origin, int rc); + + /** + * Constructor. + */ + error(const char* const origin, + const int rc) noexcept + : runtime_error{origin}, + _code{rc} {} + + /** + * Returns the underlying LMDB error code. + */ + int code() const noexcept { + return _code; + } + + /** + * Returns the origin of the LMDB error. + */ + const char* origin() const noexcept { + return runtime_error::what(); + } + + /** + * Returns the underlying LMDB error code. + */ + virtual const char* what() const noexcept { + static thread_local char buffer[1024]; + std::snprintf(buffer, sizeof(buffer), + "%s: %s", origin(), ::mdb_strerror(code())); + return buffer; + } +}; + +/** + * Base class for logic error conditions. + */ +class lmdb::logic_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Base class for fatal error conditions. + */ +class lmdb::fatal_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Base class for runtime error conditions. + */ +class lmdb::runtime_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Exception class for `MDB_KEYEXIST` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga05dc5bbcc7da81a7345bd8676e8e0e3b + */ +class lmdb::key_exist_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_NOTFOUND` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gabeb52e4c4be21b329e31c4add1b71926 + */ +class lmdb::not_found_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_CORRUPTED` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gaf8148bf1b85f58e264e57194bafb03ef + */ +class lmdb::corrupted_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_PANIC` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gae37b9aedcb3767faba3de8c1cf6d3473 + */ +class lmdb::panic_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_VERSION_MISMATCH` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga909b2db047fa90fb0d37a78f86a6f99b + */ +class lmdb::version_mismatch_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_MAP_FULL` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga0a83370402a060c9175100d4bbfb9f25 + */ +class lmdb::map_full_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_BAD_DBI` errors. + * + * @since 0.9.14 (2014/09/20) + * @see http://symas.com/mdb/doc/group__errors.html#gab4c82e050391b60a18a5df08d22a7083 + */ +class lmdb::bad_dbi_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +inline void +lmdb::error::raise(const char* const origin, + const int rc) { + switch (rc) { + case MDB_KEYEXIST: throw key_exist_error{origin, rc}; + case MDB_NOTFOUND: throw not_found_error{origin, rc}; + case MDB_CORRUPTED: throw corrupted_error{origin, rc}; + case MDB_PANIC: throw panic_error{origin, rc}; + case MDB_VERSION_MISMATCH: throw version_mismatch_error{origin, rc}; + case MDB_MAP_FULL: throw map_full_error{origin, rc}; +#ifdef MDB_BAD_DBI + case MDB_BAD_DBI: throw bad_dbi_error{origin, rc}; +#endif + default: throw lmdb::runtime_error{origin, rc}; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Metadata */ + +namespace lmdb { + // TODO: mdb_version() + // TODO: mdb_strerror() +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Environment */ + +namespace lmdb { + static inline void env_create(MDB_env** env); + static inline void env_open(MDB_env* env, + const char* path, unsigned int flags, mode mode); +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + static inline void env_copy(MDB_env* env, const char* path, unsigned int flags); + static inline void env_copy_fd(MDB_env* env, mdb_filehandle_t fd, unsigned int flags); +#else + static inline void env_copy(MDB_env* env, const char* path); + static inline void env_copy_fd(MDB_env* env, mdb_filehandle_t fd); +#endif + static inline void env_stat(MDB_env* env, MDB_stat* stat); + static inline void env_info(MDB_env* env, MDB_envinfo* stat); + static inline void env_sync(MDB_env* env, bool force); + static inline void env_close(MDB_env* env) noexcept; + static inline void env_set_flags(MDB_env* env, unsigned int flags, bool onoff); + static inline void env_get_flags(MDB_env* env, unsigned int* flags); + static inline void env_get_path(MDB_env* env, const char** path); + static inline void env_get_fd(MDB_env* env, mdb_filehandle_t* fd); + static inline void env_set_mapsize(MDB_env* env, std::size_t size); + static inline void env_set_max_readers(MDB_env* env, unsigned int count); + static inline void env_get_max_readers(MDB_env* env, unsigned int* count); + static inline void env_set_max_dbs(MDB_env* env, MDB_dbi count); + static inline unsigned int env_get_max_keysize(MDB_env* env); +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) + static inline void env_set_userctx(MDB_env* env, void* ctx); + static inline void* env_get_userctx(MDB_env* env); +#endif + // TODO: mdb_env_set_assert() + // TODO: mdb_reader_list() + // TODO: mdb_reader_check() +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaad6be3d8dcd4ea01f8df436f41d158d4 + */ +static inline void +lmdb::env_create(MDB_env** env) { + const int rc = ::mdb_env_create(env); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_create", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga32a193c6bf4d7d5c5d579e71f22e9340 + */ +static inline void +lmdb::env_open(MDB_env* const env, + const char* const path, + const unsigned int flags, + const mode mode) { + const int rc = ::mdb_env_open(env, path, flags, mode); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_open", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga3bf50d7793b36aaddf6b481a44e24244 + * @see http://symas.com/mdb/doc/group__mdb.html#ga5d51d6130325f7353db0955dbedbc378 + */ +static inline void +lmdb::env_copy(MDB_env* const env, +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + const char* const path, + const unsigned int flags = 0) { + const int rc = ::mdb_env_copy2(env, path, flags); +#else + const char* const path) { + const int rc = ::mdb_env_copy(env, path); +#endif + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_copy2", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga5040d0de1f14000fa01fc0b522ff1f86 + * @see http://symas.com/mdb/doc/group__mdb.html#ga470b0bcc64ac417de5de5930f20b1a28 + */ +static inline void +lmdb::env_copy_fd(MDB_env* const env, +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + const mdb_filehandle_t fd, + const unsigned int flags = 0) { + const int rc = ::mdb_env_copyfd2(env, fd, flags); +#else + const mdb_filehandle_t fd) { + const int rc = ::mdb_env_copyfd(env, fd); +#endif + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_copyfd2", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaf881dca452050efbd434cd16e4bae255 + */ +static inline void +lmdb::env_stat(MDB_env* const env, + MDB_stat* const stat) { + const int rc = ::mdb_env_stat(env, stat); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_stat", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga18769362c7e7d6cf91889a028a5c5947 + */ +static inline void +lmdb::env_info(MDB_env* const env, + MDB_envinfo* const stat) { + const int rc = ::mdb_env_info(env, stat); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_info", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga85e61f05aa68b520cc6c3b981dba5037 + */ +static inline void +lmdb::env_sync(MDB_env* const env, + const bool force = true) { + const int rc = ::mdb_env_sync(env, force); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_sync", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga4366c43ada8874588b6a62fbda2d1e95 + */ +static inline void +lmdb::env_close(MDB_env* const env) noexcept { + ::mdb_env_close(env); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga83f66cf02bfd42119451e9468dc58445 + */ +static inline void +lmdb::env_set_flags(MDB_env* const env, + const unsigned int flags, + const bool onoff = true) { + const int rc = ::mdb_env_set_flags(env, flags, onoff ? 1 : 0); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_flags", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga2733aefc6f50beb49dd0c6eb19b067d9 + */ +static inline void +lmdb::env_get_flags(MDB_env* const env, + unsigned int* const flags) { + const int rc = ::mdb_env_get_flags(env, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_flags", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac699fdd8c4f8013577cb933fb6a757fe + */ +static inline void +lmdb::env_get_path(MDB_env* const env, + const char** path) { + const int rc = ::mdb_env_get_path(env, path); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_path", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaf1570e7c0e5a5d860fef1032cec7d5f2 + */ +static inline void +lmdb::env_get_fd(MDB_env* const env, + mdb_filehandle_t* const fd) { + const int rc = ::mdb_env_get_fd(env, fd); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_fd", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5 + */ +static inline void +lmdb::env_set_mapsize(MDB_env* const env, + const std::size_t size) { + const int rc = ::mdb_env_set_mapsize(env, size); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_mapsize", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gae687966c24b790630be2a41573fe40e2 + */ +static inline void +lmdb::env_set_max_readers(MDB_env* const env, + const unsigned int count) { + const int rc = ::mdb_env_set_maxreaders(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_maxreaders", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga70e143cf11760d869f754c9c9956e6cc + */ +static inline void +lmdb::env_get_max_readers(MDB_env* const env, + unsigned int* const count) { + const int rc = ::mdb_env_get_maxreaders(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_maxreaders", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaa2fc2f1f37cb1115e733b62cab2fcdbc + */ +static inline void +lmdb::env_set_max_dbs(MDB_env* const env, + const MDB_dbi count) { + const int rc = ::mdb_env_set_maxdbs(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_maxdbs", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94 + */ +static inline unsigned int +lmdb::env_get_max_keysize(MDB_env* const env) { + const int rc = ::mdb_env_get_maxkeysize(env); +#ifdef LMDBXX_DEBUG + assert(rc >= 0); +#endif + return static_cast(rc); +} + +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) +/** + * @throws lmdb::error on failure + * @since 0.9.11 (2014/01/15) + * @see http://symas.com/mdb/doc/group__mdb.html#gaf2fe09eb9c96eeb915a76bf713eecc46 + */ +static inline void +lmdb::env_set_userctx(MDB_env* const env, + void* const ctx) { + const int rc = ::mdb_env_set_userctx(env, ctx); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_userctx", rc); + } +} +#endif + +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) +/** + * @since 0.9.11 (2014/01/15) + * @see http://symas.com/mdb/doc/group__mdb.html#ga45df6a4fb150cda2316b5ae224ba52f1 + */ +static inline void* +lmdb::env_get_userctx(MDB_env* const env) { + return ::mdb_env_get_userctx(env); +} +#endif + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Transactions */ + +namespace lmdb { + static inline void txn_begin( + MDB_env* env, MDB_txn* parent, unsigned int flags, MDB_txn** txn); + static inline MDB_env* txn_env(MDB_txn* txn) noexcept; +#ifdef LMDBXX_TXN_ID + static inline std::size_t txn_id(MDB_txn* txn) noexcept; +#endif + static inline void txn_commit(MDB_txn* txn); + static inline void txn_abort(MDB_txn* txn) noexcept; + static inline void txn_reset(MDB_txn* txn) noexcept; + static inline void txn_renew(MDB_txn* txn); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gad7ea55da06b77513609efebd44b26920 + */ +static inline void +lmdb::txn_begin(MDB_env* const env, + MDB_txn* const parent, + const unsigned int flags, + MDB_txn** txn) { + const int rc = ::mdb_txn_begin(env, parent, flags, txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_begin", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gaeb17735b8aaa2938a78a45cab85c06a0 + */ +static inline MDB_env* +lmdb::txn_env(MDB_txn* const txn) noexcept { + return ::mdb_txn_env(txn); +} + +#ifdef LMDBXX_TXN_ID +/** + * @note Only available in HEAD, not yet in any 0.9.x release (as of 0.9.16). + */ +static inline std::size_t +lmdb::txn_id(MDB_txn* const txn) noexcept { + return ::mdb_txn_id(txn); +} +#endif + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga846fbd6f46105617ac9f4d76476f6597 + */ +static inline void +lmdb::txn_commit(MDB_txn* const txn) { + const int rc = ::mdb_txn_commit(txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_commit", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga73a5938ae4c3239ee11efa07eb22b882 + */ +static inline void +lmdb::txn_abort(MDB_txn* const txn) noexcept { + ::mdb_txn_abort(txn); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga02b06706f8a66249769503c4e88c56cd + */ +static inline void +lmdb::txn_reset(MDB_txn* const txn) noexcept { + ::mdb_txn_reset(txn); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga6c6f917959517ede1c504cf7c720ce6d + */ +static inline void +lmdb::txn_renew(MDB_txn* const txn) { + const int rc = ::mdb_txn_renew(txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_renew", rc); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Databases */ + +namespace lmdb { + static inline void dbi_open( + MDB_txn* txn, const char* name, unsigned int flags, MDB_dbi* dbi); + static inline void dbi_stat(MDB_txn* txn, MDB_dbi dbi, MDB_stat* stat); + static inline void dbi_flags(MDB_txn* txn, MDB_dbi dbi, unsigned int* flags); + static inline void dbi_close(MDB_env* env, MDB_dbi dbi) noexcept; + static inline void dbi_drop(MDB_txn* txn, MDB_dbi dbi, bool del); + static inline void dbi_set_compare(MDB_txn* txn, MDB_dbi dbi, MDB_cmp_func* cmp); + static inline void dbi_set_dupsort(MDB_txn* txn, MDB_dbi dbi, MDB_cmp_func* cmp); + static inline void dbi_set_relfunc(MDB_txn* txn, MDB_dbi dbi, MDB_rel_func* rel); + static inline void dbi_set_relctx(MDB_txn* txn, MDB_dbi dbi, void* ctx); + static inline bool dbi_get(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, MDB_val* data); + static inline bool dbi_put(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, MDB_val* data, unsigned int flags); + static inline bool dbi_del(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, const MDB_val* data); + // TODO: mdb_cmp() + // TODO: mdb_dcmp() +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a + */ +static inline void +lmdb::dbi_open(MDB_txn* const txn, + const char* const name, + const unsigned int flags, + MDB_dbi* const dbi) { + const int rc = ::mdb_dbi_open(txn, name, flags, dbi); + if (rc != MDB_SUCCESS) { + error::raise("mdb_dbi_open", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gae6c1069febe94299769dbdd032fadef6 + */ +static inline void +lmdb::dbi_stat(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_stat* const result) { + const int rc = ::mdb_stat(txn, dbi, result); + if (rc != MDB_SUCCESS) { + error::raise("mdb_stat", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga95ba4cb721035478a8705e57b91ae4d4 + */ +static inline void +lmdb::dbi_flags(MDB_txn* const txn, + const MDB_dbi dbi, + unsigned int* const flags) { + const int rc = ::mdb_dbi_flags(txn, dbi, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_dbi_flags", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga52dd98d0c542378370cd6b712ff961b5 + */ +static inline void +lmdb::dbi_close(MDB_env* const env, + const MDB_dbi dbi) noexcept { + ::mdb_dbi_close(env, dbi); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gab966fab3840fc54a6571dfb32b00f2db + */ +static inline void +lmdb::dbi_drop(MDB_txn* const txn, + const MDB_dbi dbi, + const bool del = false) { + const int rc = ::mdb_drop(txn, dbi, del ? 1 : 0); + if (rc != MDB_SUCCESS) { + error::raise("mdb_drop", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga68e47ffcf72eceec553c72b1784ee0fe + */ +static inline void +lmdb::dbi_set_compare(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cmp_func* const cmp = nullptr) { + const int rc = ::mdb_set_compare(txn, dbi, cmp); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_compare", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gacef4ec3dab0bbd9bc978b73c19c879ae + */ +static inline void +lmdb::dbi_set_dupsort(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cmp_func* const cmp = nullptr) { + const int rc = ::mdb_set_dupsort(txn, dbi, cmp); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_dupsort", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga697d82c7afe79f142207ad5adcdebfeb + */ +static inline void +lmdb::dbi_set_relfunc(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_rel_func* const rel) { + const int rc = ::mdb_set_relfunc(txn, dbi, rel); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_relfunc", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga7c34246308cee01724a1839a8f5cc594 + */ +static inline void +lmdb::dbi_set_relctx(MDB_txn* const txn, + const MDB_dbi dbi, + void* const ctx) { + const int rc = ::mdb_set_relctx(txn, dbi, ctx); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_relctx", rc); + } +} + +/** + * @retval true if the key/value pair was retrieved + * @retval false if the key wasn't found + * @see http://symas.com/mdb/doc/group__mdb.html#ga8bf10cd91d3f3a83a34d04ce6b07992d + */ +static inline bool +lmdb::dbi_get(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + MDB_val* const data) { + const int rc = ::mdb_get(txn, dbi, const_cast(key), data); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_get", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @retval true if the key/value pair was inserted + * @retval false if the key already existed + * @see http://symas.com/mdb/doc/group__mdb.html#ga4fa8573d9236d54687c61827ebf8cac0 + */ +static inline bool +lmdb::dbi_put(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + MDB_val* const data, + const unsigned int flags = 0) { + const int rc = ::mdb_put(txn, dbi, const_cast(key), data, flags); + if (rc != MDB_SUCCESS && rc != MDB_KEYEXIST) { + error::raise("mdb_put", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @retval true if the key/value pair was removed + * @retval false if the key wasn't found + * @see http://symas.com/mdb/doc/group__mdb.html#gab8182f9360ea69ac0afd4a4eaab1ddb0 + */ +static inline bool +lmdb::dbi_del(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + const MDB_val* const data = nullptr) { + const int rc = ::mdb_del(txn, dbi, const_cast(key), const_cast(data)); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_del", rc); + } + return (rc == MDB_SUCCESS); +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Cursors */ + +namespace lmdb { + static inline void cursor_open(MDB_txn* txn, MDB_dbi dbi, MDB_cursor** cursor); + static inline void cursor_close(MDB_cursor* cursor) noexcept; + static inline void cursor_renew(MDB_txn* txn, MDB_cursor* cursor); + static inline MDB_txn* cursor_txn(MDB_cursor* cursor) noexcept; + static inline MDB_dbi cursor_dbi(MDB_cursor* cursor) noexcept; + static inline bool cursor_get(MDB_cursor* cursor, MDB_val* key, MDB_val* data, MDB_cursor_op op); + static inline void cursor_put(MDB_cursor* cursor, MDB_val* key, MDB_val* data, unsigned int flags); + static inline void cursor_del(MDB_cursor* cursor, unsigned int flags); + static inline void cursor_count(MDB_cursor* cursor, std::size_t& count); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga9ff5d7bd42557fd5ee235dc1d62613aa + */ +static inline void +lmdb::cursor_open(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cursor** const cursor) { + const int rc = ::mdb_cursor_open(txn, dbi, cursor); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_open", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gad685f5d73c052715c7bd859cc4c05188 + */ +static inline void +lmdb::cursor_close(MDB_cursor* const cursor) noexcept { + ::mdb_cursor_close(cursor); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac8b57befb68793070c85ea813df481af + */ +static inline void +lmdb::cursor_renew(MDB_txn* const txn, + MDB_cursor* const cursor) { + const int rc = ::mdb_cursor_renew(txn, cursor); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_renew", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga7bf0d458f7f36b5232fcb368ebda79e0 + */ +static inline MDB_txn* +lmdb::cursor_txn(MDB_cursor* const cursor) noexcept { + return ::mdb_cursor_txn(cursor); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga2f7092cf70ee816fb3d2c3267a732372 + */ +static inline MDB_dbi +lmdb::cursor_dbi(MDB_cursor* const cursor) noexcept { + return ::mdb_cursor_dbi(cursor); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga48df35fb102536b32dfbb801a47b4cb0 + */ +static inline bool +lmdb::cursor_get(MDB_cursor* const cursor, + MDB_val* const key, + MDB_val* const data, + const MDB_cursor_op op) { + const int rc = ::mdb_cursor_get(cursor, key, data, op); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_cursor_get", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga1f83ccb40011837ff37cc32be01ad91e + */ +static inline void +lmdb::cursor_put(MDB_cursor* const cursor, + MDB_val* const key, + MDB_val* const data, + const unsigned int flags = 0) { + const int rc = ::mdb_cursor_put(cursor, key, data, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_put", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga26a52d3efcfd72e5bf6bd6960bf75f95 + */ +static inline void +lmdb::cursor_del(MDB_cursor* const cursor, + const unsigned int flags = 0) { + const int rc = ::mdb_cursor_del(cursor, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_del", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga4041fd1e1862c6b7d5f10590b86ffbe2 + */ +static inline void +lmdb::cursor_count(MDB_cursor* const cursor, + std::size_t& count) { + const int rc = ::mdb_cursor_count(cursor, &count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_count", rc); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Values */ + +namespace lmdb { + class val; +} + +/** + * Wrapper class for `MDB_val` structures. + * + * @note Instances of this class are movable and copyable both. + * @see http://symas.com/mdb/doc/group__mdb.html#structMDB__val + */ +class lmdb::val { +protected: + MDB_val _val; + +public: + /** + * Default constructor. + */ + val() noexcept = default; + + /** + * Constructor. + */ + val(const std::string& data) noexcept + : val{data.data(), data.size()} {} + + /** + * Constructor. + */ + val(const char* const data) noexcept + : val{data, std::strlen(data)} {} + + /** + * Constructor. + */ + val(const void* const data, + const std::size_t size) noexcept + : _val{size, const_cast(data)} {} + + /** + * Move constructor. + */ + val(val&& other) noexcept = default; + + /** + * Move assignment operator. + */ + val& operator=(val&& other) noexcept = default; + + /** + * Destructor. + */ + ~val() noexcept = default; + + /** + * Returns an `MDB_val*` pointer. + */ + operator MDB_val*() noexcept { + return &_val; + } + + /** + * Returns an `MDB_val*` pointer. + */ + operator const MDB_val*() const noexcept { + return &_val; + } + + /** + * Determines whether this value is empty. + */ + bool empty() const noexcept { + return size() == 0; + } + + /** + * Returns the size of the data. + */ + std::size_t size() const noexcept { + return _val.mv_size; + } + + /** + * Returns a pointer to the data. + */ + template + T* data() noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + template + const T* data() const noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + char* data() noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + const char* data() const noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Assigns the value. + */ + template + val& assign(const T* const data, + const std::size_t size) noexcept { + _val.mv_size = size; + _val.mv_data = const_cast(reinterpret_cast(data)); + return *this; + } + + /** + * Assigns the value. + */ + val& assign(const char* const data) noexcept { + return assign(data, std::strlen(data)); + } + + /** + * Assigns the value. + */ + val& assign(const std::string& data) noexcept { + return assign(data.data(), data.size()); + } +}; + +#if !(defined(__COVERITY__) || defined(_MSC_VER)) +static_assert(std::is_pod::value, "lmdb::val must be a POD type"); +static_assert(sizeof(lmdb::val) == sizeof(MDB_val), "sizeof(lmdb::val) != sizeof(MDB_val)"); +#endif + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Environment */ + +namespace lmdb { + class env; +} + +/** + * Resource class for `MDB_env*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__env + */ +class lmdb::env { +protected: + MDB_env* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + static constexpr mode default_mode = 0644; /* -rw-r--r-- */ + + /** + * Creates a new LMDB environment. + * + * @param flags + * @throws lmdb::error on failure + */ + static env create(const unsigned int flags = default_flags) { + MDB_env* handle{nullptr}; + lmdb::env_create(&handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + if (flags) { + try { + lmdb::env_set_flags(handle, flags); + } + catch (const lmdb::error&) { + lmdb::env_close(handle); + throw; + } + } + return env{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_env*` handle + */ + env(MDB_env* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + env(env&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + env& operator=(env&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~env() noexcept { + try { close(); } catch (...) {} + } + + /** + * Returns the underlying `MDB_env*` handle. + */ + operator MDB_env*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_env*` handle. + */ + MDB_env* handle() const noexcept { + return _handle; + } + + /** + * Flushes data buffers to disk. + * + * @param force + * @throws lmdb::error on failure + */ + void sync(const bool force = true) { + lmdb::env_sync(handle(), force); + } + + /** + * Closes this environment, releasing the memory map. + * + * @note this method is idempotent + * @post `handle() == nullptr` + */ + void close() noexcept { + if (handle()) { + lmdb::env_close(handle()); + _handle = nullptr; + } + } + + /** + * Opens this environment. + * + * @param path + * @param flags + * @param mode + * @throws lmdb::error on failure + */ + env& open(const char* const path, + const unsigned int flags = default_flags, + const mode mode = default_mode) { + lmdb::env_open(handle(), path, flags, mode); + return *this; + } + + /** + * @param flags + * @param onoff + * @throws lmdb::error on failure + */ + env& set_flags(const unsigned int flags, + const bool onoff = true) { + lmdb::env_set_flags(handle(), flags, onoff); + return *this; + } + + /** + * @param size + * @throws lmdb::error on failure + */ + env& set_mapsize(const std::size_t size) { + lmdb::env_set_mapsize(handle(), size); + return *this; + } + + /** + * @param count + * @throws lmdb::error on failure + */ + env& set_max_readers(const unsigned int count) { + lmdb::env_set_max_readers(handle(), count); + return *this; + } + + /** + * @param count + * @throws lmdb::error on failure + */ + env& set_max_dbs(const MDB_dbi count) { + lmdb::env_set_max_dbs(handle(), count); + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Transactions */ + +namespace lmdb { + class txn; +} + +/** + * Resource class for `MDB_txn*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__txn + */ +class lmdb::txn { +protected: + MDB_txn* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + + /** + * Creates a new LMDB transaction. + * + * @param env the environment handle + * @param parent + * @param flags + * @throws lmdb::error on failure + */ + static txn begin(MDB_env* const env, + MDB_txn* const parent = nullptr, + const unsigned int flags = default_flags) { + MDB_txn* handle{nullptr}; + lmdb::txn_begin(env, parent, flags, &handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + return txn{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_txn*` handle + */ + txn(MDB_txn* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + txn(txn&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + txn& operator=(txn&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~txn() noexcept { + if (_handle) { + try { abort(); } catch (...) {} + _handle = nullptr; + } + } + + /** + * Returns the underlying `MDB_txn*` handle. + */ + operator MDB_txn*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_txn*` handle. + */ + MDB_txn* handle() const noexcept { + return _handle; + } + + /** + * Returns the transaction's `MDB_env*` handle. + */ + MDB_env* env() const noexcept { + return lmdb::txn_env(handle()); + } + + /** + * Commits this transaction. + * + * @throws lmdb::error on failure + * @post `handle() == nullptr` + */ + void commit() { + lmdb::txn_commit(_handle); + _handle = nullptr; + } + + /** + * Aborts this transaction. + * + * @post `handle() == nullptr` + */ + void abort() noexcept { + lmdb::txn_abort(_handle); + _handle = nullptr; + } + + /** + * Resets this read-only transaction. + */ + void reset() noexcept { + lmdb::txn_reset(_handle); + } + + /** + * Renews this read-only transaction. + * + * @throws lmdb::error on failure + */ + void renew() { + lmdb::txn_renew(_handle); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Databases */ + +namespace lmdb { + class dbi; +} + +/** + * Resource class for `MDB_dbi` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__mdb.html#gadbe68a06c448dfb62da16443d251a78b + */ +class lmdb::dbi { +protected: + MDB_dbi _handle{0}; + +public: + static constexpr unsigned int default_flags = 0; + static constexpr unsigned int default_put_flags = 0; + + /** + * Opens a database handle. + * + * @param txn the transaction handle + * @param name + * @param flags + * @throws lmdb::error on failure + */ + static dbi + open(MDB_txn* const txn, + const char* const name = nullptr, + const unsigned int flags = default_flags) { + MDB_dbi handle{}; + lmdb::dbi_open(txn, name, flags, &handle); + return dbi{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_dbi` handle + */ + dbi(const MDB_dbi handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + dbi(dbi&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + dbi& operator=(dbi&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~dbi() noexcept { + if (_handle) { + /* No need to call close() here. */ + } + } + + /** + * Returns the underlying `MDB_dbi` handle. + */ + operator MDB_dbi() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_dbi` handle. + */ + MDB_dbi handle() const noexcept { + return _handle; + } + + /** + * Returns statistics for this database. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + MDB_stat stat(MDB_txn* const txn) const { + MDB_stat result; + lmdb::dbi_stat(txn, handle(), &result); + return result; + } + + /** + * Retrieves the flags for this database handle. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + unsigned int flags(MDB_txn* const txn) const { + unsigned int result{}; + lmdb::dbi_flags(txn, handle(), &result); + return result; + } + + /** + * Returns the number of records in this database. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + std::size_t size(MDB_txn* const txn) const { + return stat(txn).ms_entries; + } + + /** + * @param txn a transaction handle + * @param del + * @throws lmdb::error on failure + */ + void drop(MDB_txn* const txn, + const bool del = false) { + lmdb::dbi_drop(txn, handle(), del); + } + + /** + * Sets a custom key comparison function for this database. + * + * @param txn a transaction handle + * @param cmp the comparison function + * @throws lmdb::error on failure + */ + dbi& set_compare(MDB_txn* const txn, + MDB_cmp_func* const cmp = nullptr) { + lmdb::dbi_set_compare(txn, handle(), cmp); + return *this; + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @param data + * @throws lmdb::error on failure + */ + bool get(MDB_txn* const txn, + const val& key, + val& data) { + return lmdb::dbi_get(txn, handle(), key, data); + } + + /** + * Retrieves a key from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const K& key) const { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + return lmdb::dbi_get(txn, handle(), k, v); + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @param val + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const K& key, + V& val) const { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + const bool result = lmdb::dbi_get(txn, handle(), k, v); + if (result) { + val = *v.data(); + } + return result; + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const char* const key, + V& val) const { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{}; + const bool result = lmdb::dbi_get(txn, handle(), k, v); + if (result) { + val = *v.data(); + } + return result; + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key + * @param data + * @param flags + * @throws lmdb::error on failure + */ + bool put(MDB_txn* const txn, + const val& key, + val& data, + const unsigned int flags = default_put_flags) { + return lmdb::dbi_put(txn, handle(), key, data, flags); + } + + /** + * Stores a key into this database. + * + * @param txn a transaction handle + * @param key + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const K& key, + const unsigned int flags = default_put_flags) { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key + * @param val + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const K& key, + const V& val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{&val, sizeof(V)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const char* const key, + const V& val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{&val, sizeof(V)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val a NUL-terminated string key + * @param flags + * @throws lmdb::error on failure + */ + bool put(MDB_txn* const txn, + const char* const key, + const char* const val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{val, std::strlen(val)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Removes a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + bool del(MDB_txn* const txn, + const val& key) { + return lmdb::dbi_del(txn, handle(), key); + } + + /** + * Removes a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + template + bool del(MDB_txn* const txn, + const K& key) { + const lmdb::val k{&key, sizeof(K)}; + return lmdb::dbi_del(txn, handle(), k); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Cursors */ + +namespace lmdb { + class cursor; +} + +/** + * Resource class for `MDB_cursor*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__cursor + */ +class lmdb::cursor { +protected: + MDB_cursor* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + + /** + * Creates an LMDB cursor. + * + * @param txn the transaction handle + * @param dbi the database handle + * @throws lmdb::error on failure + */ + static cursor + open(MDB_txn* const txn, + const MDB_dbi dbi) { + MDB_cursor* handle{}; + lmdb::cursor_open(txn, dbi, &handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + return cursor{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_cursor*` handle + */ + cursor(MDB_cursor* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + cursor(cursor&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + cursor& operator=(cursor&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~cursor() noexcept { + try { close(); } catch (...) {} + } + + /** + * Returns the underlying `MDB_cursor*` handle. + */ + operator MDB_cursor*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_cursor*` handle. + */ + MDB_cursor* handle() const noexcept { + return _handle; + } + + /** + * Closes this cursor. + * + * @note this method is idempotent + * @post `handle() == nullptr` + */ + void close() noexcept { + if (_handle) { + lmdb::cursor_close(_handle); + _handle = nullptr; + } + } + + /** + * Renews this cursor. + * + * @param txn the transaction scope + * @throws lmdb::error on failure + */ + void renew(MDB_txn* const txn) { + lmdb::cursor_renew(txn, handle()); + } + + /** + * Returns the cursor's transaction handle. + */ + MDB_txn* txn() const noexcept { + return lmdb::cursor_txn(handle()); + } + + /** + * Returns the cursor's database handle. + */ + MDB_dbi dbi() const noexcept { + return lmdb::cursor_dbi(handle()); + } + + /** + * Retrieves a key from the database. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + bool get(MDB_val* const key, + const MDB_cursor_op op) { + return get(key, nullptr, op); + } + + /** + * Retrieves a key from the database. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + bool get(lmdb::val& key, + const MDB_cursor_op op) { + return get(key, nullptr, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val (may be `nullptr`) + * @param op + * @throws lmdb::error on failure + */ + bool get(MDB_val* const key, + MDB_val* const val, + const MDB_cursor_op op) { + return lmdb::cursor_get(handle(), key, val, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val + * @param op + * @throws lmdb::error on failure + */ + bool get(lmdb::val& key, + lmdb::val& val, + const MDB_cursor_op op) { + return lmdb::cursor_get(handle(), key, val, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val + * @param op + * @throws lmdb::error on failure + */ + bool get(std::string& key, + std::string& val, + const MDB_cursor_op op) { + lmdb::val k{}, v{}; + const bool found = get(k, v, op); + if (found) { + key.assign(k.data(), k.size()); + val.assign(v.data(), v.size()); + } + return found; + } + + /** + * Positions this cursor at the given key. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + template + bool find(const K& key, + const MDB_cursor_op op = MDB_SET) { + lmdb::val k{&key, sizeof(K)}; + return get(k, nullptr, op); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +#endif /* LMDBXX_H */ diff --git a/cpp/include/prettyprint.hpp b/cpp/include/prettyprint.hpp new file mode 100644 index 0000000..ebba086 --- /dev/null +++ b/cpp/include/prettyprint.hpp @@ -0,0 +1,446 @@ +// https://github.com/louisdx/cxx-prettyprint/blob/master/prettyprint.hpp 27cad81cabff867ed3f72fdf7a195d2c2a90552f +// Copyright Louis Delacroix 2010 - 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// +// A pretty printing library for C++ +// +// Usage: +// Include this header, and operator<< will "just work". + +#ifndef H_PRETTY_PRINT +#define H_PRETTY_PRINT + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pretty_print +{ + namespace detail + { + // SFINAE type trait to detect whether T::const_iterator exists. + + struct sfinae_base + { + using yes = char; + using no = yes[2]; + }; + + template + struct has_const_iterator : private sfinae_base + { + private: + template static yes & test(typename C::const_iterator*); + template static no & test(...); + public: + static const bool value = sizeof(test(nullptr)) == sizeof(yes); + using type = T; + }; + + template + struct has_begin_end : private sfinae_base + { + private: + template + static yes & f(typename std::enable_if< + std::is_same(&C::begin)), + typename C::const_iterator(C::*)() const>::value>::type *); + + template static no & f(...); + + template + static yes & g(typename std::enable_if< + std::is_same(&C::end)), + typename C::const_iterator(C::*)() const>::value, void>::type*); + + template static no & g(...); + + public: + static bool const beg_value = sizeof(f(nullptr)) == sizeof(yes); + static bool const end_value = sizeof(g(nullptr)) == sizeof(yes); + }; + + } // namespace detail + + + // Holds the delimiter values for a specific character type + + template + struct delimiters_values + { + using char_type = TChar; + const char_type * prefix; + const char_type * delimiter; + const char_type * postfix; + }; + + + // Defines the delimiter values for a specific container and character type + + template + struct delimiters + { + using type = delimiters_values; + static const type values; + }; + + + // Functor to print containers. You can use this directly if you want + // to specificy a non-default delimiters type. The printing logic can + // be customized by specializing the nested template. + + template , + typename TDelimiters = delimiters> + struct print_container_helper + { + using delimiters_type = TDelimiters; + using ostream_type = std::basic_ostream; + + template + struct printer + { + static void print_body(const U & c, ostream_type & stream) + { + using std::begin; + using std::end; + + auto it = begin(c); + const auto the_end = end(c); + + if (it != the_end) + { + for ( ; ; ) + { + stream << *it; + + if (++it == the_end) break; + + if (delimiters_type::values.delimiter != NULL) + stream << delimiters_type::values.delimiter; + } + } + } + }; + + print_container_helper(const T & container) + : container_(container) + { } + + inline void operator()(ostream_type & stream) const + { + if (delimiters_type::values.prefix != NULL) + stream << delimiters_type::values.prefix; + + printer::print_body(container_, stream); + + if (delimiters_type::values.postfix != NULL) + stream << delimiters_type::values.postfix; + } + + private: + const T & container_; + }; + + // Specialization for pairs + + template + template + struct print_container_helper::printer> + { + using ostream_type = typename print_container_helper::ostream_type; + + static void print_body(const std::pair & c, ostream_type & stream) + { + stream << c.first; + if (print_container_helper::delimiters_type::values.delimiter != NULL) + stream << print_container_helper::delimiters_type::values.delimiter; + stream << c.second; + } + }; + + // Specialization for tuples + + template + template + struct print_container_helper::printer> + { + using ostream_type = typename print_container_helper::ostream_type; + using element_type = std::tuple; + + template struct Int { }; + + static void print_body(const element_type & c, ostream_type & stream) + { + tuple_print(c, stream, Int<0>()); + } + + static void tuple_print(const element_type &, ostream_type &, Int) + { + } + + static void tuple_print(const element_type & c, ostream_type & stream, + typename std::conditional, std::nullptr_t>::type) + { + stream << std::get<0>(c); + tuple_print(c, stream, Int<1>()); + } + + template + static void tuple_print(const element_type & c, ostream_type & stream, Int) + { + if (print_container_helper::delimiters_type::values.delimiter != NULL) + stream << print_container_helper::delimiters_type::values.delimiter; + + stream << std::get(c); + + tuple_print(c, stream, Int()); + } + }; + + // Prints a print_container_helper to the specified stream. + + template + inline std::basic_ostream & operator<<( + std::basic_ostream & stream, + const print_container_helper & helper) + { + helper(stream); + return stream; + } + + + // Basic is_container template; specialize to derive from std::true_type for all desired container types + + template + struct is_container : public std::integral_constant::value && + detail::has_begin_end::beg_value && + detail::has_begin_end::end_value> { }; + + template + struct is_container : std::true_type { }; + + template + struct is_container : std::false_type { }; + + template + struct is_container> : std::true_type { }; + + template + struct is_container> : std::true_type { }; + + template + struct is_container> : std::true_type { }; + + + // Default delimiters + + template struct delimiters { static const delimiters_values values; }; + template const delimiters_values delimiters::values = { "[", ", ", "]" }; + template struct delimiters { static const delimiters_values values; }; + template const delimiters_values delimiters::values = { L"[", L", ", L"]" }; + + + // Delimiters for (multi)set and unordered_(multi)set + + template + struct delimiters< ::std::set, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::set, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::set, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::set, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::multiset, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::multiset, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::multiset, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::multiset, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::unordered_set, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_set, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::unordered_set, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_set, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::unordered_multiset, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_multiset, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::unordered_multiset, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_multiset, wchar_t>::values = { L"{", L", ", L"}" }; + + + // Delimiters for pair and tuple + + template struct delimiters, char> { static const delimiters_values values; }; + template const delimiters_values delimiters, char>::values = { "(", ", ", ")" }; + template struct delimiters< ::std::pair, wchar_t> { static const delimiters_values values; }; + template const delimiters_values delimiters< ::std::pair, wchar_t>::values = { L"(", L", ", L")" }; + + template struct delimiters, char> { static const delimiters_values values; }; + template const delimiters_values delimiters, char>::values = { "(", ", ", ")" }; + template struct delimiters< ::std::tuple, wchar_t> { static const delimiters_values values; }; + template const delimiters_values delimiters< ::std::tuple, wchar_t>::values = { L"(", L", ", L")" }; + + + // Type-erasing helper class for easy use of custom delimiters. + // Requires TCharTraits = std::char_traits and TChar = char or wchar_t, and MyDelims needs to be defined for TChar. + // Usage: "cout << pretty_print::custom_delims(x)". + + struct custom_delims_base + { + virtual ~custom_delims_base() { } + virtual std::ostream & stream(::std::ostream &) = 0; + virtual std::wostream & stream(::std::wostream &) = 0; + }; + + template + struct custom_delims_wrapper : custom_delims_base + { + custom_delims_wrapper(const T & t_) : t(t_) { } + + std::ostream & stream(std::ostream & s) + { + return s << print_container_helper, Delims>(t); + } + + std::wostream & stream(std::wostream & s) + { + return s << print_container_helper, Delims>(t); + } + + private: + const T & t; + }; + + template + struct custom_delims + { + template + custom_delims(const Container & c) : base(new custom_delims_wrapper(c)) { } + + std::unique_ptr base; + }; + + template + inline std::basic_ostream & operator<<(std::basic_ostream & s, const custom_delims & p) + { + return p.base->stream(s); + } + + + // A wrapper for a C-style array given as pointer-plus-size. + // Usage: std::cout << pretty_print_array(arr, n) << std::endl; + + template + struct array_wrapper_n + { + typedef const T * const_iterator; + typedef T value_type; + + array_wrapper_n(const T * const a, size_t n) : _array(a), _n(n) { } + inline const_iterator begin() const { return _array; } + inline const_iterator end() const { return _array + _n; } + + private: + const T * const _array; + size_t _n; + }; + + + // A wrapper for hash-table based containers that offer local iterators to each bucket. + // Usage: std::cout << bucket_print(m, 4) << std::endl; (Prints bucket 5 of container m.) + + template + struct bucket_print_wrapper + { + typedef typename T::const_local_iterator const_iterator; + typedef typename T::size_type size_type; + + const_iterator begin() const + { + return m_map.cbegin(n); + } + + const_iterator end() const + { + return m_map.cend(n); + } + + bucket_print_wrapper(const T & m, size_type bucket) : m_map(m), n(bucket) { } + + private: + const T & m_map; + const size_type n; + }; + +} // namespace pretty_print + + +// Global accessor functions for the convenience wrappers + +template +inline pretty_print::array_wrapper_n pretty_print_array(const T * const a, size_t n) +{ + return pretty_print::array_wrapper_n(a, n); +} + +template pretty_print::bucket_print_wrapper +bucket_print(const T & m, typename T::size_type n) +{ + return pretty_print::bucket_print_wrapper(m, n); +} + + +// Main magic entry point: An overload snuck into namespace std. +// Can we do better? + +namespace std +{ + // Prints a container to the stream using default delimiters + + template + inline typename enable_if< ::pretty_print::is_container::value, + basic_ostream &>::type + operator<<(basic_ostream & stream, const T & container) + { + return stream << ::pretty_print::print_container_helper(container); + } +} + + + +#endif // H_PRETTY_PRINT diff --git a/cpp/include/scope_exit.hpp b/cpp/include/scope_exit.hpp new file mode 100644 index 0000000..f7cab9c --- /dev/null +++ b/cpp/include/scope_exit.hpp @@ -0,0 +1,39 @@ +// https://gist.github.com/socantre/b45b3e23e6f1f4715f08 +#include + +#pragma once + +namespace socantre { + +template struct scope_exit { + // construction + explicit scope_exit(EF &&f) noexcept(true) + : exit_function(std::move(f)), execute_on_destruction{true} {} + // move + scope_exit(scope_exit &&rhs) noexcept(true) + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + // release + ~scope_exit() noexcept(noexcept(this->exit_function())) { + if (execute_on_destruction) + this->exit_function(); + } + void release() noexcept(true) { this->execute_on_destruction = false; } + +private: + scope_exit(scope_exit const &) = delete; + void operator=(scope_exit const &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + EF exit_function; + bool execute_on_destruction; // exposition only +}; + +template +auto make_scope_exit(EF &&exit_function) noexcept(true) -> decltype( + scope_exit>(std::forward(exit_function))) { + return scope_exit>( + std::forward(exit_function)); +} +} // namespace socantre diff --git a/cpp/include/tinyformat.h b/cpp/include/tinyformat.h new file mode 100644 index 0000000..1194d75 --- /dev/null +++ b/cpp/include/tinyformat.h @@ -0,0 +1,1003 @@ +// tinyformat.h +// Copyright (C) 2011, Chris Foster [chris42f (at) gmail (d0t) com] +// +// Boost Software License - Version 1.0 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//------------------------------------------------------------------------------ +// Tinyformat: A minimal type safe printf replacement +// +// tinyformat.h is a type safe printf replacement library in a single C++ +// header file. Design goals include: +// +// * Type safety and extensibility for user defined types. +// * C99 printf() compatibility, to the extent possible using std::ostream +// * Simplicity and minimalism. A single header file to include and distribute +// with your projects. +// * Augment rather than replace the standard stream formatting mechanism +// * C++98 support, with optional C++11 niceties +// +// +// Main interface example usage +// ---------------------------- +// +// To print a date to std::cout: +// +// std::string weekday = "Wednesday"; +// const char* month = "July"; +// size_t day = 27; +// long hour = 14; +// int min = 44; +// +// tfm::printf("%s, %s %d, %.2d:%.2d\n", weekday, month, day, hour, min); +// +// The strange types here emphasize the type safety of the interface; it is +// possible to print a std::string using the "%s" conversion, and a +// size_t using the "%d" conversion. A similar result could be achieved +// using either of the tfm::format() functions. One prints on a user provided +// stream: +// +// tfm::format(std::cerr, "%s, %s %d, %.2d:%.2d\n", +// weekday, month, day, hour, min); +// +// The other returns a std::string: +// +// std::string date = tfm::format("%s, %s %d, %.2d:%.2d\n", +// weekday, month, day, hour, min); +// std::cout << date; +// +// These are the three primary interface functions. +// +// +// User defined format functions +// ----------------------------- +// +// Simulating variadic templates in C++98 is pretty painful since it requires +// writing out the same function for each desired number of arguments. To make +// this bearable tinyformat comes with a set of macros which are used +// internally to generate the API, but which may also be used in user code. +// +// The three macros TINYFORMAT_ARGTYPES(n), TINYFORMAT_VARARGS(n) and +// TINYFORMAT_PASSARGS(n) will generate a list of n argument types, +// type/name pairs and argument names respectively when called with an integer +// n between 1 and 16. We can use these to define a macro which generates the +// desired user defined function with n arguments. To generate all 16 user +// defined function bodies, use the macro TINYFORMAT_FOREACH_ARGNUM. For an +// example, see the implementation of printf() at the end of the source file. +// +// Sometimes it's useful to be able to pass a list of format arguments through +// to a non-template function. The FormatList class is provided as a way to do +// this by storing the argument list in a type-opaque way. Continuing the +// example from above, we construct a FormatList using makeFormatList(): +// +// FormatListRef formatList = tfm::makeFormatList(weekday, month, day, hour, min); +// +// The format list can now be passed into any non-template function and used +// via a call to the vformat() function: +// +// tfm::vformat(std::cout, "%s, %s %d, %.2d:%.2d\n", formatList); +// +// +// Additional API information +// -------------------------- +// +// Error handling: Define TINYFORMAT_ERROR to customize the error handling for +// format strings which are unsupported or have the wrong number of format +// specifiers (calls assert() by default). +// +// User defined types: Uses operator<< for user defined types by default. +// Overload formatValue() for more control. + + +#ifndef TINYFORMAT_H_INCLUDED +#define TINYFORMAT_H_INCLUDED + +namespace tinyformat {} +//------------------------------------------------------------------------------ +// Config section. Customize to your liking! + +// Namespace alias to encourage brevity +namespace tfm = tinyformat; + +// Error handling; calls assert() by default. +// #define TINYFORMAT_ERROR(reasonString) your_error_handler(reasonString) + +// Define for C++11 variadic templates which make the code shorter & more +// general. If you don't define this, C++11 support is autodetected below. +// #define TINYFORMAT_USE_VARIADIC_TEMPLATES + + +//------------------------------------------------------------------------------ +// Implementation details. +#include +#include +#include +#include + +#ifndef TINYFORMAT_ERROR +# define TINYFORMAT_ERROR(reason) assert(0 && reason) +#endif + +#if !defined(TINYFORMAT_USE_VARIADIC_TEMPLATES) && !defined(TINYFORMAT_NO_VARIADIC_TEMPLATES) +# ifdef __GXX_EXPERIMENTAL_CXX0X__ +# define TINYFORMAT_USE_VARIADIC_TEMPLATES +# endif +#endif + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20080201 +// std::showpos is broken on old libstdc++ as provided with OSX. See +// http://gcc.gnu.org/ml/libstdc++/2007-11/msg00075.html +# define TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND +#endif + +namespace tinyformat { + +//------------------------------------------------------------------------------ +namespace detail { + +// Test whether type T1 is convertible to type T2 +template +struct is_convertible +{ + private: + // two types of different size + struct fail { char dummy[2]; }; + struct succeed { char dummy; }; + // Try to convert a T1 to a T2 by plugging into tryConvert + static fail tryConvert(...); + static succeed tryConvert(const T2&); + static const T1& makeT1(); + public: +# ifdef _MSC_VER + // Disable spurious loss of precision warnings in tryConvert(makeT1()) +# pragma warning(push) +# pragma warning(disable:4244) +# pragma warning(disable:4267) +# endif + // Standard trick: the (...) version of tryConvert will be chosen from + // the overload set only if the version taking a T2 doesn't match. + // Then we compare the sizes of the return types to check which + // function matched. Very neat, in a disgusting kind of way :) + static const bool value = + sizeof(tryConvert(makeT1())) == sizeof(succeed); +# ifdef _MSC_VER +# pragma warning(pop) +# endif +}; + + +// Detect when a type is not a wchar_t string +template struct is_wchar { typedef int tinyformat_wchar_is_not_supported; }; +template<> struct is_wchar {}; +template<> struct is_wchar {}; +template struct is_wchar {}; +template struct is_wchar {}; + + +// Format the value by casting to type fmtT. This default implementation +// should never be called. +template::value> +struct formatValueAsType +{ + static void invoke(std::ostream& /*out*/, const T& /*value*/) { assert(0); } +}; +// Specialized version for types that can actually be converted to fmtT, as +// indicated by the "convertible" template parameter. +template +struct formatValueAsType +{ + static void invoke(std::ostream& out, const T& value) + { out << static_cast(value); } +}; + +#ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND +template::value> +struct formatZeroIntegerWorkaround +{ + static bool invoke(std::ostream& /**/, const T& /**/) { return false; } +}; +template +struct formatZeroIntegerWorkaround +{ + static bool invoke(std::ostream& out, const T& value) + { + if (static_cast(value) == 0 && out.flags() & std::ios::showpos) + { + out << "+0"; + return true; + } + return false; + } +}; +#endif // TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND + +// Convert an arbitrary type to integer. The version with convertible=false +// throws an error. +template::value> +struct convertToInt +{ + static int invoke(const T& /*value*/) + { + TINYFORMAT_ERROR("tinyformat: Cannot convert from argument type to " + "integer for use as variable width or precision"); + return 0; + } +}; +// Specialization for convertToInt when conversion is possible +template +struct convertToInt +{ + static int invoke(const T& value) { return static_cast(value); } +}; + +// Format at most ntrunc characters to the given stream. +template +inline void formatTruncated(std::ostream& out, const T& value, int ntrunc) +{ + std::ostringstream tmp; + tmp << value; + std::string result = tmp.str(); + out.write(result.c_str(), std::min(ntrunc, static_cast(result.size()))); +} +#define TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(type) \ +inline void formatTruncated(std::ostream& out, type* value, int ntrunc) \ +{ \ + std::streamsize len = 0; \ + while(len < ntrunc && value[len] != 0) \ + ++len; \ + out.write(value, len); \ +} +// Overload for const char* and char*. Could overload for signed & unsigned +// char too, but these are technically unneeded for printf compatibility. +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(const char) +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char) +#undef TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR + +} // namespace detail + + +//------------------------------------------------------------------------------ +// Variable formatting functions. May be overridden for user-defined types if +// desired. + + +/// Format a value into a stream, delegating to operator<< by default. +/// +/// Users may override this for their own types. When this function is called, +/// the stream flags will have been modified according to the format string. +/// The format specification is provided in the range [fmtBegin, fmtEnd). For +/// truncating conversions, ntrunc is set to the desired maximum number of +/// characters, for example "%.7s" calls formatValue with ntrunc = 7. +/// +/// By default, formatValue() uses the usual stream insertion operator +/// operator<< to format the type T, with special cases for the %c and %p +/// conversions. +template +inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, + const char* fmtEnd, int ntrunc, const T& value) +{ +#ifndef TINYFORMAT_ALLOW_WCHAR_STRINGS + // Since we don't support printing of wchar_t using "%ls", make it fail at + // compile time in preference to printing as a void* at runtime. + typedef typename detail::is_wchar::tinyformat_wchar_is_not_supported DummyType; + (void) DummyType(); // avoid unused type warning with gcc-4.8 +#endif + // The mess here is to support the %c and %p conversions: if these + // conversions are active we try to convert the type to a char or const + // void* respectively and format that instead of the value itself. For the + // %p conversion it's important to avoid dereferencing the pointer, which + // could otherwise lead to a crash when printing a dangling (const char*). + const bool canConvertToChar = detail::is_convertible::value; + const bool canConvertToVoidPtr = detail::is_convertible::value; + if(canConvertToChar && *(fmtEnd-1) == 'c') + detail::formatValueAsType::invoke(out, value); + else if(canConvertToVoidPtr && *(fmtEnd-1) == 'p') + detail::formatValueAsType::invoke(out, value); +#ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND + else if(detail::formatZeroIntegerWorkaround::invoke(out, value)) /**/; +#endif + else if(ntrunc >= 0) + { + // Take care not to overread C strings in truncating conversions like + // "%.4s" where at most 4 characters may be read. + detail::formatTruncated(out, value, ntrunc); + } + else + out << value; +} + + +// Overloaded version for char types to support printing as an integer +#define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \ +inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, \ + const char* fmtEnd, int /**/, charType value) \ +{ \ + switch(*(fmtEnd-1)) \ + { \ + case 'u': case 'd': case 'i': case 'o': case 'X': case 'x': \ + out << static_cast(value); break; \ + default: \ + out << value; break; \ + } \ +} +// per 3.9.1: char, signed char and unsigned char are all distinct types +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(char) +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(signed char) +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(unsigned char) +#undef TINYFORMAT_DEFINE_FORMATVALUE_CHAR + + +//------------------------------------------------------------------------------ +// Tools for emulating variadic templates in C++98. The basic idea here is +// stolen from the boost preprocessor metaprogramming library and cut down to +// be just general enough for what we need. + +#define TINYFORMAT_ARGTYPES(n) TINYFORMAT_ARGTYPES_ ## n +#define TINYFORMAT_VARARGS(n) TINYFORMAT_VARARGS_ ## n +#define TINYFORMAT_PASSARGS(n) TINYFORMAT_PASSARGS_ ## n +#define TINYFORMAT_PASSARGS_TAIL(n) TINYFORMAT_PASSARGS_TAIL_ ## n + +// To keep it as transparent as possible, the macros below have been generated +// using python via the excellent cog.py code generation script. This avoids +// the need for a bunch of complex (but more general) preprocessor tricks as +// used in boost.preprocessor. +// +// To rerun the code generation in place, use `cog.py -r tinyformat.h` +// (see http://nedbatchelder.com/code/cog). Alternatively you can just create +// extra versions by hand. + +/*[[[cog +maxParams = 16 + +def makeCommaSepLists(lineTemplate, elemTemplate, startInd=1): + for j in range(startInd,maxParams+1): + list = ', '.join([elemTemplate % {'i':i} for i in range(startInd,j+1)]) + cog.outl(lineTemplate % {'j':j, 'list':list}) + +makeCommaSepLists('#define TINYFORMAT_ARGTYPES_%(j)d %(list)s', + 'class T%(i)d') + +cog.outl() +makeCommaSepLists('#define TINYFORMAT_VARARGS_%(j)d %(list)s', + 'const T%(i)d& v%(i)d') + +cog.outl() +makeCommaSepLists('#define TINYFORMAT_PASSARGS_%(j)d %(list)s', 'v%(i)d') + +cog.outl() +cog.outl('#define TINYFORMAT_PASSARGS_TAIL_1') +makeCommaSepLists('#define TINYFORMAT_PASSARGS_TAIL_%(j)d , %(list)s', + 'v%(i)d', startInd = 2) + +cog.outl() +cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' + + ' '.join(['m(%d)' % (j,) for j in range(1,maxParams+1)])) +]]]*/ +#define TINYFORMAT_ARGTYPES_1 class T1 +#define TINYFORMAT_ARGTYPES_2 class T1, class T2 +#define TINYFORMAT_ARGTYPES_3 class T1, class T2, class T3 +#define TINYFORMAT_ARGTYPES_4 class T1, class T2, class T3, class T4 +#define TINYFORMAT_ARGTYPES_5 class T1, class T2, class T3, class T4, class T5 +#define TINYFORMAT_ARGTYPES_6 class T1, class T2, class T3, class T4, class T5, class T6 +#define TINYFORMAT_ARGTYPES_7 class T1, class T2, class T3, class T4, class T5, class T6, class T7 +#define TINYFORMAT_ARGTYPES_8 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8 +#define TINYFORMAT_ARGTYPES_9 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9 +#define TINYFORMAT_ARGTYPES_10 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10 +#define TINYFORMAT_ARGTYPES_11 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11 +#define TINYFORMAT_ARGTYPES_12 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12 +#define TINYFORMAT_ARGTYPES_13 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13 +#define TINYFORMAT_ARGTYPES_14 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14 +#define TINYFORMAT_ARGTYPES_15 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14, class T15 +#define TINYFORMAT_ARGTYPES_16 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14, class T15, class T16 + +#define TINYFORMAT_VARARGS_1 const T1& v1 +#define TINYFORMAT_VARARGS_2 const T1& v1, const T2& v2 +#define TINYFORMAT_VARARGS_3 const T1& v1, const T2& v2, const T3& v3 +#define TINYFORMAT_VARARGS_4 const T1& v1, const T2& v2, const T3& v3, const T4& v4 +#define TINYFORMAT_VARARGS_5 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5 +#define TINYFORMAT_VARARGS_6 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6 +#define TINYFORMAT_VARARGS_7 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7 +#define TINYFORMAT_VARARGS_8 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8 +#define TINYFORMAT_VARARGS_9 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9 +#define TINYFORMAT_VARARGS_10 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10 +#define TINYFORMAT_VARARGS_11 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11 +#define TINYFORMAT_VARARGS_12 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12 +#define TINYFORMAT_VARARGS_13 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13 +#define TINYFORMAT_VARARGS_14 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14 +#define TINYFORMAT_VARARGS_15 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14, const T15& v15 +#define TINYFORMAT_VARARGS_16 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14, const T15& v15, const T16& v16 + +#define TINYFORMAT_PASSARGS_1 v1 +#define TINYFORMAT_PASSARGS_2 v1, v2 +#define TINYFORMAT_PASSARGS_3 v1, v2, v3 +#define TINYFORMAT_PASSARGS_4 v1, v2, v3, v4 +#define TINYFORMAT_PASSARGS_5 v1, v2, v3, v4, v5 +#define TINYFORMAT_PASSARGS_6 v1, v2, v3, v4, v5, v6 +#define TINYFORMAT_PASSARGS_7 v1, v2, v3, v4, v5, v6, v7 +#define TINYFORMAT_PASSARGS_8 v1, v2, v3, v4, v5, v6, v7, v8 +#define TINYFORMAT_PASSARGS_9 v1, v2, v3, v4, v5, v6, v7, v8, v9 +#define TINYFORMAT_PASSARGS_10 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 +#define TINYFORMAT_PASSARGS_11 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11 +#define TINYFORMAT_PASSARGS_12 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12 +#define TINYFORMAT_PASSARGS_13 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13 +#define TINYFORMAT_PASSARGS_14 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14 +#define TINYFORMAT_PASSARGS_15 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 +#define TINYFORMAT_PASSARGS_16 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16 + +#define TINYFORMAT_PASSARGS_TAIL_1 +#define TINYFORMAT_PASSARGS_TAIL_2 , v2 +#define TINYFORMAT_PASSARGS_TAIL_3 , v2, v3 +#define TINYFORMAT_PASSARGS_TAIL_4 , v2, v3, v4 +#define TINYFORMAT_PASSARGS_TAIL_5 , v2, v3, v4, v5 +#define TINYFORMAT_PASSARGS_TAIL_6 , v2, v3, v4, v5, v6 +#define TINYFORMAT_PASSARGS_TAIL_7 , v2, v3, v4, v5, v6, v7 +#define TINYFORMAT_PASSARGS_TAIL_8 , v2, v3, v4, v5, v6, v7, v8 +#define TINYFORMAT_PASSARGS_TAIL_9 , v2, v3, v4, v5, v6, v7, v8, v9 +#define TINYFORMAT_PASSARGS_TAIL_10 , v2, v3, v4, v5, v6, v7, v8, v9, v10 +#define TINYFORMAT_PASSARGS_TAIL_11 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11 +#define TINYFORMAT_PASSARGS_TAIL_12 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12 +#define TINYFORMAT_PASSARGS_TAIL_13 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13 +#define TINYFORMAT_PASSARGS_TAIL_14 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14 +#define TINYFORMAT_PASSARGS_TAIL_15 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 +#define TINYFORMAT_PASSARGS_TAIL_16 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16 + +#define TINYFORMAT_FOREACH_ARGNUM(m) \ + m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) m(10) m(11) m(12) m(13) m(14) m(15) m(16) +//[[[end]]] + + + +namespace detail { + +// Type-opaque holder for an argument to format(), with associated actions on +// the type held as explicit function pointers. This allows FormatArg's for +// each argument to be allocated as a homogenous array inside FormatList +// whereas a naive implementation based on inheritance does not. +class FormatArg +{ + public: + FormatArg() {} + + template + FormatArg(const T& value) + : m_value(static_cast(&value)), + m_formatImpl(&formatImpl), + m_toIntImpl(&toIntImpl) + { } + + void format(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc) const + { + m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); + } + + int toInt() const + { + return m_toIntImpl(m_value); + } + + private: + template + static void formatImpl(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value) + { + formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast(value)); + } + + template + static int toIntImpl(const void* value) + { + return convertToInt::invoke(*static_cast(value)); + } + + const void* m_value; + void (*m_formatImpl)(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value); + int (*m_toIntImpl)(const void* value); +}; + + +// Parse and return an integer from the string c, as atoi() +// On return, c is set to one past the end of the integer. +inline int parseIntAndAdvance(const char*& c) +{ + int i = 0; + for(;*c >= '0' && *c <= '9'; ++c) + i = 10*i + (*c - '0'); + return i; +} + +// Print literal part of format string and return next format spec +// position. +// +// Skips over any occurrences of '%%', printing a literal '%' to the +// output. The position of the first % character of the next +// nontrivial format spec is returned, or the end of string. +inline const char* printFormatStringLiteral(std::ostream& out, const char* fmt) +{ + const char* c = fmt; + for(; true; ++c) + { + switch(*c) + { + case '\0': + out.write(fmt, static_cast(c - fmt)); + return c; + case '%': + out.write(fmt, static_cast(c - fmt)); + if(*(c+1) != '%') + return c; + // for "%%", tack trailing % onto next literal section. + fmt = ++c; + break; + } + } +} + + +// Parse a format string and set the stream state accordingly. +// +// The format mini-language recognized here is meant to be the one from C99, +// with the form "%[flags][width][.precision][length]type". +// +// Formatting options which can't be natively represented using the ostream +// state are returned in spacePadPositive (for space padded positive numbers) +// and ntrunc (for truncating conversions). argIndex is incremented if +// necessary to pull out variable width and precision . The function returns a +// pointer to the character after the end of the current format spec. +inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositive, + int& ntrunc, const char* fmtStart, + const detail::FormatArg* formatters, + int& argIndex, int numFormatters) +{ + if(*fmtStart != '%') + { + TINYFORMAT_ERROR("tinyformat: Not enough conversion specifiers in format string"); + return fmtStart; + } + // Reset stream state to defaults. + out.width(0); + out.precision(6); + out.fill(' '); + // Reset most flags; ignore irrelevant unitbuf & skipws. + out.unsetf(std::ios::adjustfield | std::ios::basefield | + std::ios::floatfield | std::ios::showbase | std::ios::boolalpha | + std::ios::showpoint | std::ios::showpos | std::ios::uppercase); + bool precisionSet = false; + bool widthSet = false; + int widthExtra = 0; + const char* c = fmtStart + 1; + // 1) Parse flags + for(;; ++c) + { + switch(*c) + { + case '#': + out.setf(std::ios::showpoint | std::ios::showbase); + continue; + case '0': + // overridden by left alignment ('-' flag) + if(!(out.flags() & std::ios::left)) + { + // Use internal padding so that numeric values are + // formatted correctly, eg -00010 rather than 000-10 + out.fill('0'); + out.setf(std::ios::internal, std::ios::adjustfield); + } + continue; + case '-': + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + continue; + case ' ': + // overridden by show positive sign, '+' flag. + if(!(out.flags() & std::ios::showpos)) + spacePadPositive = true; + continue; + case '+': + out.setf(std::ios::showpos); + spacePadPositive = false; + widthExtra = 1; + continue; + } + break; + } + // 2) Parse width + if(*c >= '0' && *c <= '9') + { + widthSet = true; + out.width(parseIntAndAdvance(c)); + } + if(*c == '*') + { + widthSet = true; + int width = 0; + if(argIndex < numFormatters) + width = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable width"); + if(width < 0) + { + // negative widths correspond to '-' flag set + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + width = -width; + } + out.width(width); + ++c; + } + // 3) Parse precision + if(*c == '.') + { + ++c; + int precision = 0; + if(*c == '*') + { + ++c; + if(argIndex < numFormatters) + precision = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable precision"); + } + else + { + if(*c >= '0' && *c <= '9') + precision = parseIntAndAdvance(c); + else if(*c == '-') // negative precisions ignored, treated as zero. + parseIntAndAdvance(++c); + } + out.precision(precision); + precisionSet = true; + } + // 4) Ignore any C99 length modifier + while(*c == 'l' || *c == 'h' || *c == 'L' || + *c == 'j' || *c == 'z' || *c == 't') + ++c; + // 5) We're up to the conversion specifier character. + // Set stream flags based on conversion specifier (thanks to the + // boost::format class for forging the way here). + bool intConversion = false; + switch(*c) + { + case 'u': case 'd': case 'i': + out.setf(std::ios::dec, std::ios::basefield); + intConversion = true; + break; + case 'o': + out.setf(std::ios::oct, std::ios::basefield); + intConversion = true; + break; + case 'X': + out.setf(std::ios::uppercase); + case 'x': case 'p': + out.setf(std::ios::hex, std::ios::basefield); + intConversion = true; + break; + case 'E': + out.setf(std::ios::uppercase); + case 'e': + out.setf(std::ios::scientific, std::ios::floatfield); + out.setf(std::ios::dec, std::ios::basefield); + break; + case 'F': + out.setf(std::ios::uppercase); + case 'f': + out.setf(std::ios::fixed, std::ios::floatfield); + break; + case 'G': + out.setf(std::ios::uppercase); + case 'g': + out.setf(std::ios::dec, std::ios::basefield); + // As in boost::format, let stream decide float format. + out.flags(out.flags() & ~std::ios::floatfield); + break; + case 'a': case 'A': + TINYFORMAT_ERROR("tinyformat: the %a and %A conversion specs " + "are not supported"); + break; + case 'c': + // Handled as special case inside formatValue() + break; + case 's': + if(precisionSet) + ntrunc = static_cast(out.precision()); + // Make %s print booleans as "true" and "false" + out.setf(std::ios::boolalpha); + break; + case 'n': + // Not supported - will cause problems! + TINYFORMAT_ERROR("tinyformat: %n conversion spec not supported"); + break; + case '\0': + TINYFORMAT_ERROR("tinyformat: Conversion spec incorrectly " + "terminated by end of string"); + return c; + } + if(intConversion && precisionSet && !widthSet) + { + // "precision" for integers gives the minimum number of digits (to be + // padded with zeros on the left). This isn't really supported by the + // iostreams, but we can approximately simulate it with the width if + // the width isn't otherwise used. + out.width(out.precision() + widthExtra); + out.setf(std::ios::internal, std::ios::adjustfield); + out.fill('0'); + } + return c+1; +} + + +//------------------------------------------------------------------------------ +inline void formatImpl(std::ostream& out, const char* fmt, + const detail::FormatArg* formatters, + int numFormatters) +{ + // Saved stream state + std::streamsize origWidth = out.width(); + std::streamsize origPrecision = out.precision(); + std::ios::fmtflags origFlags = out.flags(); + char origFill = out.fill(); + + for (int argIndex = 0; argIndex < numFormatters; ++argIndex) + { + // Parse the format string + fmt = printFormatStringLiteral(out, fmt); + bool spacePadPositive = false; + int ntrunc = -1; + const char* fmtEnd = streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, + formatters, argIndex, numFormatters); + if (argIndex >= numFormatters) + { + // Check args remain after reading any variable width/precision + TINYFORMAT_ERROR("tinyformat: Not enough format arguments"); + return; + } + const FormatArg& arg = formatters[argIndex]; + // Format the arg into the stream. + if(!spacePadPositive) + arg.format(out, fmt, fmtEnd, ntrunc); + else + { + // The following is a special case with no direct correspondence + // between stream formatting and the printf() behaviour. Simulate + // it crudely by formatting into a temporary string stream and + // munging the resulting string. + std::ostringstream tmpStream; + tmpStream.copyfmt(out); + tmpStream.setf(std::ios::showpos); + arg.format(tmpStream, fmt, fmtEnd, ntrunc); + std::string result = tmpStream.str(); // allocates... yuck. + for(size_t i = 0, iend = result.size(); i < iend; ++i) + if(result[i] == '+') result[i] = ' '; + out << result; + } + fmt = fmtEnd; + } + + // Print remaining part of format string. + fmt = printFormatStringLiteral(out, fmt); + if(*fmt != '\0') + TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); + + // Restore stream state + out.width(origWidth); + out.precision(origPrecision); + out.flags(origFlags); + out.fill(origFill); +} + +} // namespace detail + + +/// List of template arguments format(), held in a type-opaque way. +/// +/// A const reference to FormatList (typedef'd as FormatListRef) may be +/// conveniently used to pass arguments to non-template functions: All type +/// information has been stripped from the arguments, leaving just enough of a +/// common interface to perform formatting as required. +class FormatList +{ + public: + FormatList(detail::FormatArg* formatters, int N) + : m_formatters(formatters), m_N(N) { } + + friend void vformat(std::ostream& out, const char* fmt, + const FormatList& list); + + private: + const detail::FormatArg* m_formatters; + int m_N; +}; + +/// Reference to type-opaque format list for passing to vformat() +typedef const FormatList& FormatListRef; + + +namespace detail { + +// Format list subclass with fixed storage to avoid dynamic allocation +template +class FormatListN : public FormatList +{ + public: +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + template + FormatListN(const Args&... args) + : FormatList(&m_formatterStore[0], N), + m_formatterStore{FormatArg(args)...} + { static_assert(sizeof...(args) == N, "Number of args must be N"); } +#else // C++98 version + void init(int) {} +# define TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR(n) \ + \ + template \ + FormatListN(TINYFORMAT_VARARGS(n)) \ + : FormatList(&m_formatterStore[0], n) \ + { assert(n == N); init(0, TINYFORMAT_PASSARGS(n)); } \ + \ + template \ + void init(int i, TINYFORMAT_VARARGS(n)) \ + { \ + m_formatterStore[i] = FormatArg(v1); \ + init(i+1 TINYFORMAT_PASSARGS_TAIL(n)); \ + } + + TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR) +# undef TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR +#endif + + private: + FormatArg m_formatterStore[N]; +}; + +// Special 0-arg version - MSVC says zero-sized C array in struct is nonstandard +template<> class FormatListN<0> : public FormatList +{ + public: FormatListN() : FormatList(0, 0) {} +}; + +} // namespace detail + + +//------------------------------------------------------------------------------ +// Primary API functions + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Make type-agnostic format list from list of template arguments. +/// +/// The exact return type of this function is an implementation detail and +/// shouldn't be relied upon. Instead it should be stored as a FormatListRef: +/// +/// FormatListRef formatList = makeFormatList( /*...*/ ); +template +detail::FormatListN makeFormatList(const Args&... args) +{ + return detail::FormatListN(args...); +} + +#else // C++98 version + +inline detail::FormatListN<0> makeFormatList() +{ + return detail::FormatListN<0>(); +} +#define TINYFORMAT_MAKE_MAKEFORMATLIST(n) \ +template \ +detail::FormatListN makeFormatList(TINYFORMAT_VARARGS(n)) \ +{ \ + return detail::FormatListN(TINYFORMAT_PASSARGS(n)); \ +} +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_MAKEFORMATLIST) +#undef TINYFORMAT_MAKE_MAKEFORMATLIST + +#endif + +/// Format list of arguments to the stream according to the given format string. +/// +/// The name vformat() is chosen for the semantic similarity to vprintf(): the +/// list of format arguments is held in a single function argument. +inline void vformat(std::ostream& out, const char* fmt, FormatListRef list) +{ + detail::formatImpl(out, fmt, list.m_formatters, list.m_N); +} + + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Format list of arguments to the stream according to given format string. +template +void format(std::ostream& out, const char* fmt, const Args&... args) +{ + vformat(out, fmt, makeFormatList(args...)); +} + +/// Format list of arguments according to the given format string and return +/// the result as a string. +template +std::string format(const char* fmt, const Args&... args) +{ + std::ostringstream oss; + format(oss, fmt, args...); + return oss.str(); +} + +/// Format list of arguments to std::cout, according to the given format string +template +void printf(const char* fmt, const Args&... args) +{ + format(std::cout, fmt, args...); +} + + +#else // C++98 version + +inline void format(std::ostream& out, const char* fmt) +{ + vformat(out, fmt, makeFormatList()); +} + +inline std::string format(const char* fmt) +{ + std::ostringstream oss; + format(oss, fmt); + return oss.str(); +} + +inline void printf(const char* fmt) +{ + format(std::cout, fmt); +} + +#define TINYFORMAT_MAKE_FORMAT_FUNCS(n) \ + \ +template \ +void format(std::ostream& out, const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + vformat(out, fmt, makeFormatList(TINYFORMAT_PASSARGS(n))); \ +} \ + \ +template \ +std::string format(const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + std::ostringstream oss; \ + format(oss, fmt, TINYFORMAT_PASSARGS(n)); \ + return oss.str(); \ +} \ + \ +template \ +void printf(const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + format(std::cout, fmt, TINYFORMAT_PASSARGS(n)); \ +} + +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMAT_FUNCS) +#undef TINYFORMAT_MAKE_FORMAT_FUNCS + +#endif + + +} // namespace tinyformat + +#endif // TINYFORMAT_H_INCLUDED diff --git a/cpp/include/unique_resource.hpp b/cpp/include/unique_resource.hpp new file mode 100644 index 0000000..56b3fe4 --- /dev/null +++ b/cpp/include/unique_resource.hpp @@ -0,0 +1,78 @@ +// https://gist.github.com/socantre/b45b3e23e6f1f4715f08 + +#pragma once + +#include + +namespace socantre { + +template class unique_resource { + R resource; + D deleter; + bool execute_on_destruction; // exposition only + unique_resource &operator=(unique_resource const &) = delete; + unique_resource(unique_resource const &) = delete; // no copies! + public: + // construction + explicit unique_resource(R &&resource, D &&deleter, + bool shouldrun = true) noexcept(true) + : resource(std::move(resource)), deleter(std::move(deleter)), + execute_on_destruction{shouldrun} {} + // move + unique_resource(unique_resource &&other) noexcept(true) + : resource(std::move(other.resource)), deleter(std::move(other.deleter)), + execute_on_destruction{other.execute_on_destruction} { + other.release(); + } + unique_resource & + operator=(unique_resource &&other) noexcept(noexcept(this->reset())) { + this->reset(); + this->deleter = std::move(other.deleter); + this->resource = std::move(other.resource); + this->execute_on_destruction = other.execute_on_destruction; + other.release(); + return *this; + } + // resource release + ~unique_resource() noexcept(noexcept(this->reset())) { this->reset(); } + void reset() noexcept(noexcept(this->get_deleter()(resource))) { + if (execute_on_destruction) { + this->execute_on_destruction = false; + this->get_deleter()(resource); + } + } + void reset(R &&newresource) noexcept(noexcept(this->reset())) { + this->reset(); + this->resource = std::move(newresource); + this->execute_on_destruction = true; + } + R const &release() noexcept(true) { + this->execute_on_destruction = false; + return this->get(); + } + // resource access + R const &get() const noexcept(true) { return this->resource; } + operator R const &() const noexcept(true) { return this->resource; } + R operator->() const noexcept(true) { return this->resource; } + std::add_lvalue_reference_t> operator*() const { + return *this->resource; + } + // deleter access + const D &get_deleter() const noexcept(true) { return this->deleter; } +}; +// factories +template +auto make_unique_resource(R &&r, D &&d) noexcept(true) +-> decltype(unique_resource>( + std::move(r), std::forward>(d), true)) { + return unique_resource>( + std::move(r), std::forward>(d), true); +} +template +auto make_unique_resource_checked(R r, R invalid, D d) noexcept(true) +-> decltype(unique_resource(std::move(r), std::move(d), true)) { + bool shouldrun = not bool(r == invalid); + return unique_resource(std::move(r), std::move(d), shouldrun); +} + +} // namespace socantre diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt new file mode 100644 index 0000000..cf8e5fa --- /dev/null +++ b/cpp/src/CMakeLists.txt @@ -0,0 +1,6 @@ +#For the shared library: +set ( PROJECT_LINK_LIBS liblmdb.so) +link_directories( /usr/local/lib) + +add_library(persipubsub SHARED queue.cpp queue.h library.h library.cpp filesystem.cpp) +target_link_libraries( persipubsub ${PROJECT_LINK_LIBS}) diff --git a/cpp/src/control.h b/cpp/src/control.h new file mode 100644 index 0000000..d1b2de1 --- /dev/null +++ b/cpp/src/control.h @@ -0,0 +1,135 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub { + namespace control { + /** + * Store queue parameter in lmdb. + * + * @param max_reader_num max number of reader of the lmdb + * @param max_db_num max number of named databases + * @param max_db_size_bytes max size of the lmdb in bytes + * @param env where parameters are stored + */ + void set_queue_parameters(int max_reader_num, int max_db_num, + int max_db_size_bytes, lmdb::env env); + + /** + * Set high water mark values for queue. + * + * @param hwm high water mark values + * @param env where values are stored + */ + void set_hwm(persipubsub::queue::HighWaterMark hwm, lmdb::env env); + + /** + * Set pruning strategy for queue. + * + * @param strategy pruning strategy + * @param env where strategy are stored + */ + void set_strategy(persipubsub::queue::Strategy strategy, lmdb::env env); + + /** + * Add a subscriber and create its lmdb. + * + * @param identifier ID of the subscriber which should be added + * @param env where subscriber is added + */ + void _add_sub(char identifier, lmdb::env env); + + /** + * Control and maintain one queue. + */ + class Control { + public: + /** + * Initialize control class. + * + * @param path to the queue. + */ + Control(boost::filesystem::path path) path_(path) {}; + + /** + * Initialize control with a (re)initialized queue. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_water_mark limit of the queue + * @param strategy used to prune queue + */ + void init(std::vector subscriber_ids, int max_readers, + int max_size, + persipubsub::queue::HighWaterMark high_water_mark, + persipubsub::queue::Strategy strategy); + + /** + * Check if queue is initialized. + * + * @return is initialized when all values for the given keys are set + */ + bool check_queue_is_initialized(); + + /** + * Clear all subscriber and delete all messages for queue. + */ + void clear_all_subscribers(); + + /** + * Prune all dangling messages from the lmdb. + */ + void prune_dangling_messages(); + + ~Control(); + private: + boost::filesystem::path path_; + persipubsub::queue::Queue queue_; + std::vector subscriber_ids_; + + /** + * Reinitialize the queue which is maintained by the control. + */ + void reinitialize_queue(); + + /** + * Initialize queue. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_water_mark limit of the queue + * @param strategy used to prune queue + */ + void initialize_queue(std::vector subscriber_ids, + int max_readers, int max_size, + persipubsub::queue::HighWaterMark high_water_mark, + persipubsub::queue::Strategy strategy); + + /** + * Prune all messages of a subscriber. + * + * @param identifier ID of the subscriber of which all messages should be pruned + */ + void prune_all_messages_for(std::vector identifier); + + /** + * Remove a subscriber and delete all its messages. + * + * @param identifier ID of the subscriber which should be removed + * @param env from where subscriber is removed + */ + void remove_sub(std::vector identifier, lmdb::env env); + + }; // class Control + } // namespace control +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/environment.h b/cpp/src/environment.h new file mode 100644 index 0000000..b6720b8 --- /dev/null +++ b/cpp/src/environment.h @@ -0,0 +1,76 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#pragma once + +#include +#include +#include "queue.h" +#include "control.h" +#include "publisher.h" +#include "subscriber.h" + +namespace persipubsub{ + namespace environment{ + + /** + * Fabricate persipubsub components. + */ + class Environment{ + + public: + /** + * Initialize. + * + * @param path to the queue + */ + Environment(const boost::filesystem::path& path) : path_(path) {}; + + /** + * Fabricate a new control. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_watermark limit of the queue + * @param strategy used to prune queue + * @return Control to create and maintain queue + */ + persipubsub::control::Control new_control( + std::vector subscriber_ids, int max_readers, + int max_size, + persipubsub::queue::HighWaterMark high_watermark, + persipubsub::queue::Strategy strategy); + + /** + * Fabricate a new publisher. + * + * @param autosync if True, store data automatically in lmdb + * @return Publisher to send messages + */ + persipubsub::publisher::Publisher new_publisher(bool autosync); + + /** + * Fabricate a new subscriber. + * + * @param identifier of the subscriber + * @return Subscriber to receive messages + */ + persipubsub::subscriber::Subscriber new_subscriber(char identifier); + + private: + boost::filesystem::path path_; + + }; // class Environment + + /** + * Fabricate a new environment. + * + * @param path to the queue + * @return Environment to create control, publisher and subscriber + */ + persipubsub::environment::Environment new_environment(boost::filesystem::path path); + + } // namespace environment +} // namespace persipubsub diff --git a/cpp/src/filesystem.cpp b/cpp/src/filesystem.cpp new file mode 100644 index 0000000..05ee137 --- /dev/null +++ b/cpp/src/filesystem.cpp @@ -0,0 +1,317 @@ +// Copyright (c) 2016 Parquery AG. All rights reserved. +// Created by mristin on 10/12/16. + +#include "filesystem.h" + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +std::vector pqry::filesystem::ls_lt(const fs::path& directory) { + if (!fs::exists(directory)) { + throw std::invalid_argument(tfm::format("Directory must exist: %s", directory)); + } + + if (!fs::is_directory(directory)) { + throw std::invalid_argument( + tfm::format("Not a directory: %s %s", directory, fs::status(directory).type())); + } + + std::list > ts_pths; + for (const auto& entry : boost::make_iterator_range(fs::directory_iterator(directory), {})) { + struct stat st = {}; + stat(entry.path().c_str(), &st); + + // get the time in nanoseconds + const int64_t ns = pqry::filesystem::modified_time(entry.path()); + + ts_pths.emplace_back(std::make_tuple(ns, entry.path())); + } + + ts_pths.sort(); + + std::vector res; + res.reserve(ts_pths.size()); + + for (const auto& ts_pth : ts_pths) { + res.push_back(std::get<1>(ts_pth)); + } + + return res; +} + +std::vector pqry::filesystem::listdir(const fs::path& directory) { + if (not fs::exists(directory)) { + std::printf( "Directory does not exist"); + } + + if (not fs::is_directory(directory)) { + std::printf( "Not a directory"); + } + + std::vector res; + for (const auto& entry : boost::make_iterator_range(fs::directory_iterator(directory), {})) { + res.push_back(entry.path()); + } + return res; +} + +int64_t pqry::filesystem::modified_time(const fs::path& path) { + struct stat st = {}; + stat(path.c_str(), &st); + + // get the time in nanoseconds + const int64_t ns( + static_cast(st.st_mtim.tv_sec * 1000L * 1000L * 1000L) + static_cast(st.st_mtim.tv_nsec)); + + return ns; +} + +fs::path pqry::filesystem::mkdtemp() { + const fs::path tmpdir = fs::temp_directory_path() / fs::unique_path(); + fs::create_directories(tmpdir); + + return tmpdir; +} + +std::string pqry::filesystem::read(const fs::path& path) { + std::string content; + + std::ifstream ifs(path.c_str(), std::ios::in | std::ios::binary | std::ios::ate); + + if (ifs.good()) { + auto fsize = ifs.tellg(); + if (fsize < 0) { + std::printf( "File size retrieval failed"); + } + auto fsize_nonneg = static_cast(fsize); + + ifs.seekg(0, std::ios::beg); + + std::vector bytes(fsize_nonneg); + ifs.read(&bytes[0], fsize); + + if (ifs.good()) { + content = std::string(&bytes[0], fsize_nonneg); + } else { + std::printf( "error reading"); + } + } else { + std::printf( "error reading"); + } + + return content; +} + +void pqry::filesystem::write(const fs::path& path, const std::string& text) { + std::ofstream ofs(path.c_str(), std::ios::out | std::ios::binary); + ofs << text; + + if (!ofs.good()) { + std::printf( "error writing"); + } +} + +void pqry::filesystem::write(const fs::path& path, const char* bytes, int size) { + std::ofstream ofs(path.c_str(), std::ios::out | std::ios::binary); + + if (!ofs.good()) { + std::printf( "error writing"); + } + + ofs.write(bytes, size); +} + +void pqry::filesystem::copy_directory(const fs::path& source, const fs::path& dest) { + if (not fs::exists(source)) { + std::printf( "Error attempting to copy: Source does not exist"); + } + if (not fs::is_directory(source)) { + std::printf( "Error attempting to copy: Source is not a directory"); + } + if (not fs::exists(dest)) { + if (not fs::create_directories(dest)) { + std::printf( "Error attempting to create destination folder"); + } + } + + using rec_iterator = fs::recursive_directory_iterator; + for (auto it = rec_iterator(source, fs::symlink_option::no_recurse), end = rec_iterator(); it != end; ++it) { + // Remove common root + std::string difference = it->path().string(); + boost::replace_first(difference, source.string(), ""); + + if (not fs::exists(dest / difference)) { + boost::system::error_code error; + fs::copy(it->path(), dest / difference, error); + if (error.value() != 0) { + std::printf("Failed to copy"); + } + } else { + if (not fs::is_directory(it->path())) { + fs::copy_file(it->path(), dest / difference, fs::copy_option::overwrite_if_exists); + } + } + } +} + +void pqry::filesystem::unzip_archive(const fs::path& path, const fs::path& dest_dir) { + if (not fs::exists(path)) { + std::printf( "File does not exist"); + } + if (not fs::exists(dest_dir)) { + std::printf( "Expected the destination directory to exist"); + } + if (not fs::is_directory(dest_dir)) { + std::printf( "Not a directory"); + } + + // Read from the archive + // libzippp::ZipArchive zf(path.c_str()); + // zf.open(libzippp::ZipArchive::READ_ONLY); + // + // std::vector entries = zf.getEntries(); + // std::vector::iterator it; + // for (it = entries.begin(); it != entries.end(); ++it) { + // libzippp::ZipEntry entry = *it; + // + // // Create directory if entry is a dir + // if (entry.isDirectory()) { + // fs::path dir(dest_dir / entry.getName()); + // if (not fs::is_directory(dir.parent_path())) { + // fs::create_directory(dir.parent_path()); + // } + // continue; + // } + // + // // Only process if entry is a file + // if (not entry.isFile()) { + // continue; + // } + // + // fs::path entry_path(dest_dir / entry.getName()); + // std::ofstream ofs(entry_path.string()); + // entry.readContent(ofs); + // ofs.close(); + // } +} + +bool pqry::filesystem::LockFileGuard::lock(const fs::path& lock_file, int pid) { + if (locked_) { + unlock(); + } + + if (lock_file.empty()) { + std::printf(R"(lock_file path is "")"); + } + + fid_ = ::open(lock_file.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); + if (fid_ == -1) { + std::printf( "error opening the lock file for writing"); + } + + const int rc = ::flock(fid_, LOCK_EX | LOCK_NB); + + if (fid_ >= 0 and rc < 0) { + ::close(fid_); + locked_ = false; + return locked_; + } + + const std::string pid_str = tfm::format("%d", pid); + const ssize_t n = ::write(fid_, pid_str.c_str(), pid_str.size()); + if (n != pid_str.size()) { + std::printf( "error writing to lock file"); + } + + lock_file_ = lock_file; + locked_ = true; + return locked_; +} + +void pqry::filesystem::LockFileGuard::unlock() { + if (locked_) { + std::printf("Deleting lock file"); + ::close(fid_); + fs::remove(lock_file_); + } +} + +pqry::filesystem::LockFileGuard::~LockFileGuard() { + unlock(); +} + +void pqry::filesystem::wait_for_file(const fs::path& path, unsigned int timeout) { + int i = 0; + while (not fs::exists(path)) { + ::sleep(1); + ++i; + if (timeout > 0 and i > timeout) { + std::printf( "File does not exist (waited for some seconds)"); + } + } +} + +fs::path pqry::filesystem::expand_path(const fs::path& path) { + wordexp_t p = {}; + const int ret = wordexp(path.c_str(), &p, 0); + if (ret != 0) { + wordfree(&p); + std::printf( "Wordexp failed: %d", ret); + } + + fs::path expanded(p.we_wordv[0]); // NOLINT + wordfree(&p); + + return expanded; +} + +/** + * @param path to be added a suffix for a named temporary file + * @return path with a unique suffix + */ +fs::path with_temp_suffix(const fs::path& path) { + return path.parent_path() / + (path.stem().string() + "." + boost::filesystem::unique_path().string() + ".tmp" + path.extension().string()); +} + +pqry::filesystem::NamedTempfile::NamedTempfile( + const boost::filesystem::path& path) : pth_(path), tmp_pth_(with_temp_suffix(path)), renamed_(false) {} + +pqry::filesystem::NamedTempfile::~NamedTempfile() { + if (not renamed_) { + boost::system::error_code ec; // ignore all errors + fs::remove(tmp_pth_, ec); + } +} + +void pqry::filesystem::NamedTempfile::rename() { + if (renamed_) { + std::printf( "The temporary file has been already renamed"); + } + fs::rename(tmp_pth_, pth_); + renamed_ = true; +} diff --git a/cpp/src/library.cpp b/cpp/src/library.cpp new file mode 100644 index 0000000..183fbb6 --- /dev/null +++ b/cpp/src/library.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#include "library.h" + +#include + +persipubsub::QueueData persipubsub::lookup_queue_data(const lmdb::env &env) { + auto rtxn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + auto queue_dbi = lmdb::dbi::open(rtxn, QUEUE_DB); + + lmdb::val size_key(HWM_DB_SIZE_BYTES_KEY), size; + + queue_dbi.get(rtxn, size_key, size); + unsigned long hwm_db_size_bytes = std::stoul(size.data()); + + + lmdb::val msgs_key(MAX_MSGS_NUM_KEY), msgs_num; + queue_dbi.get(rtxn, msgs_key, msgs_num); + unsigned int max_msgs_num = std::stoi(msgs_num.data()); + + lmdb::val timeout_key(MSG_TIMEOUT_SECS_KEY), timeout; + queue_dbi.get(rtxn, timeout_key, timeout); + unsigned int msg_timeout_secs = std::stoi(timeout.data()); + + lmdb::val strategy_key(STRATEGY_KEY), strategy_val; + queue_dbi.get(rtxn, strategy_key, strategy_val); + std::string strategy_str = strategy_val.data(); + queue::Strategy strategy = queue::parse_strategy(strategy_str); + + lmdb::val subscriber_ids_key(SUBSCRIBER_IDS_KEY), subscriber_ids_val; + queue_dbi.get(rtxn, subscriber_ids_key, subscriber_ids_val); + std::string subscriber_ids_str = subscriber_ids_val.data(); + std::vector subscriber_ids; + + boost::split(subscriber_ids, subscriber_ids_str, boost::is_any_of(" "), + boost::token_compress_on); + + return QueueData(msg_timeout_secs, max_msgs_num, hwm_db_size_bytes, + strategy, subscriber_ids); +} \ No newline at end of file diff --git a/cpp/src/library.h b/cpp/src/library.h new file mode 100644 index 0000000..4deadb4 --- /dev/null +++ b/cpp/src/library.h @@ -0,0 +1,48 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#pragma once + +#include +#include +#include // Include boost::for is_any_of +#include // Include for boost::split +#include +#include "queue.h" + +namespace persipubsub { + static unsigned int MAX_READER_NUM = 1024; + static unsigned int MAX_DB_NUM = 1024; + static unsigned long MAX_DB_SIZE_BYTES = 32UL * 1024UL * 1024UL * 1024UL; + + // define all database names here + static char DATA_DB[] = "data_db"; // msg_id | data + static char PENDING_DB[] = "pending_db"; // msg_id | pending subscriber + static char META_DB[] = "meta_db"; // msg_id | metadata + static char QUEUE_DB[] = "queue_db"; // queue_pth | all queue data + + static char HWM_DB_SIZE_BYTES_KEY[] = "hwm_db_size_bytes"; + static char MAX_MSGS_NUM_KEY[] = "max_msgs_num"; + static char MSG_TIMEOUT_SECS_KEY[] = "msg_timeout_secs"; + static char STRATEGY_KEY[] = "strategy"; + static char SUBSCRIBER_IDS_KEY[] = "subscriber_ids"; + + struct QueueData{ + unsigned int msg_timeout_secs_; + unsigned int max_msgs_num_; + unsigned long hwm_db_size_bytes_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + QueueData(unsigned int msg_timeout_secs, unsigned int max_msgs_num, + unsigned long hwm_db_size_bytes, + persipubsub::queue::Strategy strategy, + std::vector subscriber_ids) : + msg_timeout_secs_(msg_timeout_secs), + max_msgs_num_(max_msgs_num), + hwm_db_size_bytes_(hwm_db_size_bytes), strategy_(strategy), + subscriber_ids_(std::move(subscriber_ids)) {}; + }; + QueueData lookup_queue_data(const lmdb::env& env); +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/publisher.h b/cpp/src/publisher.h new file mode 100644 index 0000000..d63011c --- /dev/null +++ b/cpp/src/publisher.h @@ -0,0 +1,52 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub{ + namespace publisher{ + /** + * Create Publisher ready to send messages. + */ + class Publisher{ + + public: + Publisher() {}; + + /** + * Initialize. + * + * @param autosync sync after each message or after multiple messages + * @param path to the queue + */ + void init(bool autosync, boost::filesystem::path path); + + /** + * Send message to subscribers. + * + * @param msg to send to all subscribers + */ + void send(char msg); + + /** + * Send messages to subscribers. + * + * @param msgs to send to all subscribers + */ + void send_many(std::vector msgs); + + ~Publisher(); + private: + bool autosync_; + persipubsub::queue::Queue queue_; + + }; // class Publisher + } // namespace publisher +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/queue.cpp b/cpp/src/queue.cpp new file mode 100644 index 0000000..043ce84 --- /dev/null +++ b/cpp/src/queue.cpp @@ -0,0 +1,391 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#include "queue.h" + +// +// Created by selim on 01.02.19. +// + +#include "library.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +std::map strategyMap; + +constexpr char const_prune_first[] = "prune_first"; +constexpr char const_prune_last[] = "prune_last"; + +persipubsub::queue::Strategy persipubsub::queue::parse_strategy(const std::string &strategy) { + strategyMap[const_prune_first] = persipubsub::queue::Strategy::prune_first; + strategyMap[const_prune_last] = persipubsub::queue::Strategy::prune_last; + + auto result = strategyMap.find(strategy); + if (result == strategyMap.end()) + throw std::runtime_error("Unknown strategy: " + strategy); + return result->second; + + +} + + +lmdb::env persipubsub::queue::initialize_environment(const fs::path& queue_dir, + const unsigned int max_reader_num, + const unsigned int max_db_num, + const unsigned long max_db_size_bytes) { + + if (not fs::exists(queue_dir)) + throw std::runtime_error("The queue directory does not exist: " + + queue_dir.filename().string()); + + auto env = lmdb::env::create(); + env.set_mapsize(max_db_size_bytes); + env.set_max_dbs(max_db_num); + env.set_max_readers(max_reader_num); + env.open(queue_dir.filename().string().c_str(), 0, 0664); + return env; + +} + + +void persipubsub::queue::prune_dangling_messages_for(const persipubsub::queue::Queue& queue, + const std::vector &subscriber_ids){ + + auto wtxn = lmdb::txn::begin(queue.env_); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + // Definition of dangling messages: + // - having no pending subscribers + // - exists longer than timeout allows + std::set msgs_to_delete; + + auto pending_cursor = lmdb::cursor::open(wtxn, pending_dbi); + + lmdb::val pending_key, pending_subscribers_num; + while (pending_cursor.get(pending_key, pending_subscribers_num, MDB_NEXT)) { + if (std::stoi(pending_subscribers_num.data()) == 0) + msgs_to_delete.insert(std::move(pending_key)); + } + pending_cursor.close(); + + time_t timer; + time(&timer); + const auto timestamp_now = boost::numeric_cast(timer); + + std::set msgs_to_delete_timeout; + + auto meta_cursor = lmdb::cursor::open(wtxn, pending_dbi); + + lmdb::val meta_key, timestamp; + while (meta_cursor.get(meta_key, timestamp, MDB_NEXT)) { + if (timestamp_now - std::stoi(timestamp.data()) > queue.hwm_.msg_timeout_secs_) + msgs_to_delete_timeout.insert(std::move(meta_key)); + } + meta_cursor.close(); + + for (const auto& timeout_msg: msgs_to_delete_timeout){ + lmdb::val new_timeout_msg((timeout_msg).data()); + msgs_to_delete.insert(std::move(new_timeout_msg)); + } + + for (const auto& delete_it: msgs_to_delete){ + pending_dbi.del(wtxn, delete_it); + meta_dbi.del(wtxn, delete_it); + data_dbi.del(wtxn, delete_it); + } + + for (const auto& sub_it: queue.subscriber_ids_){ + auto sub_dbi = lmdb::dbi::open(wtxn, (sub_it).c_str()); + for (const auto& key_it: msgs_to_delete_timeout){ + sub_dbi.del(wtxn, key_it); + } + } + + wtxn.commit(); +} + +// todo add transaction and cursor classes +void persipubsub::queue::Queue::init(const boost::filesystem::path& path, lmdb::env env) { + path_ = path; + + if (env) + env_ = std::move(env); + else + env_ = persipubsub::queue::initialize_environment(path, persipubsub::MAX_READER_NUM, persipubsub::MAX_DB_NUM, persipubsub::MAX_DB_SIZE_BYTES); + + { + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB, MDB_CREATE); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); + // lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); // TODO ask marko how to deal with it + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB, MDB_CREATE); + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + wtxn.commit(); // TODO this will not be needed, jumping out of scope + } + + persipubsub::QueueData queue_data = persipubsub::lookup_queue_data(env_); + + hwm_ = HighWaterMark(queue_data.msg_timeout_secs_, + queue_data.max_msgs_num_, queue_data.hwm_db_size_bytes_); + + strategy_ = queue_data.strategy_; + + subscriber_ids_ = std::move(queue_data.subscriber_ids_); +} + + +void persipubsub::queue::Queue::put(const std::string& msg, const std::vector &subscriber_ids) const{ + + vacuum(); + + time_t timer; + time(&timer); + const auto time = boost::lexical_cast(timer); + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + const std::string tmp = boost::uuids::to_string(uuid); + + std::string key = (time + tmp); + lmdb::val msg_id(key.c_str()); + + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + lmdb::val pending_subs((boost::lexical_cast(subscriber_ids.size())).c_str()); + pending_dbi.put(wtxn, msg_id, pending_subs); + + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + lmdb::val time_val(time); + meta_dbi.put(wtxn, msg_id, time_val); + + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + lmdb::val data_val(msg.c_str()); + data_dbi.put(wtxn, msg_id, data_val); + for (const auto& it: subscriber_ids) { + auto sub_dbi = lmdb::dbi::open(wtxn, (it).c_str()); + lmdb::val sub_data(""); + sub_dbi.put(wtxn, msg_id, sub_data); + } + + wtxn.commit(); + +} + +void persipubsub::queue::Queue::put_many_flush_once(const std::vector &msgs, + const std::vector &subscriber_ids) const { + vacuum(); + + time_t timer; + time(&timer); + const auto time = boost::lexical_cast(timer); + + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + std::vector sub_dbis; + for (const auto& it: subscriber_ids) { + auto sub_dbi = lmdb::dbi::open(wtxn, (it).c_str()); + sub_dbis.push_back(std::move(sub_dbi)); + } + + for (const auto& msg: msgs) { + + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + const std::string tmp = boost::uuids::to_string(uuid); + std::string key = (time + tmp); + lmdb::val msg_id(key.c_str()); + + lmdb::val pending_subs((boost::lexical_cast(subscriber_ids.size())).c_str()); + pending_dbi.put(wtxn, msg_id, pending_subs); + lmdb::val time_val(time); + meta_dbi.put(wtxn, msg_id, time_val); + lmdb::val data_val(msg.c_str()); + data_dbi.put(wtxn, msg_id, data_val); + + for (auto& sub_dbi_it: sub_dbis) { + lmdb::val sub_data(""); + sub_dbi_it.put(wtxn, msg_id, sub_data); + } + } + wtxn.commit(); + +} + +void persipubsub::queue::Queue::front(const std::string& identifier, std::string* msg) const{ + + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto sub_dbi = lmdb::dbi::open(rtxn, identifier.c_str()); + auto cursor = lmdb::cursor::open(rtxn, sub_dbi); + lmdb::val key, value; + if (cursor.get(key, value, MDB_FIRST)) { + auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); + auto data_cursor = lmdb::cursor::open(rtxn, data_dbi); + lmdb::val data_value; + + bool found = data_cursor.get(key, data_value, MDB_FIRST); + if (found){ + std::string tmp_str = data_value.data(); + std::swap(*msg, tmp_str); + } + else{ + cursor.close(); + rtxn.abort(); + throw std::runtime_error("Data not found"); + } + } + cursor.close(); + rtxn.abort(); +} + +void persipubsub::queue::Queue::pop(const std::string& identifier) const { + auto wtxn = lmdb::txn::begin(env_); + auto sub_dbi = lmdb::dbi::open(wtxn, identifier.c_str()); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + + auto cursor = lmdb::cursor::open(wtxn, sub_dbi); + + lmdb::val key, value; + + if (cursor.get(key, value, MDB_FIRST)) { + sub_dbi.del(wtxn, key); + lmdb::val pending_value; + pending_dbi.get(wtxn, key, pending_value); + auto pending_num = boost::numeric_cast(std::stoi(pending_value.data())); + pending_num--; + lmdb::val pending_new_value((boost::lexical_cast(pending_num)).c_str()); + pending_dbi.put(wtxn, key, pending_new_value); + cursor.close(); + wtxn.commit(); + } + else { + cursor.close(); + wtxn.commit(); + throw std::runtime_error("No message to pop"); + } +} + + +void persipubsub::queue::Queue::prune_dangling_messages() const { + persipubsub::queue::prune_dangling_messages_for(*this, subscriber_ids_); +} + + +// todo size_t instead of long? +unsigned long persipubsub::queue::Queue::check_current_lmdb_size() const{ + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); + + MDB_stat data_stat = data_dbi.stat(rtxn); + + // todo check overflow, boost::numeric_cast to unsigned long? + unsigned long lmdb_size_bytes = data_stat.ms_psize * (data_stat.ms_branch_pages + data_stat.ms_leaf_pages + data_stat.ms_overflow_pages); + + rtxn.abort(); + + return lmdb_size_bytes; + +} + + +unsigned int persipubsub::queue::Queue::count_msgs() const { + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto meta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); + + auto msgs_num = boost::numeric_cast(meta_dbi.stat(rtxn).ms_entries); + rtxn.abort(); + return msgs_num; +} + + +void persipubsub::queue::Queue::vacuum() const { + prune_dangling_messages(); + + unsigned int msgs_num = count_msgs(); + + if (msgs_num >= hwm_.max_msgs_num_) + prune_messages(); + unsigned long lmdb_size_bytes = check_current_lmdb_size(); + if (lmdb_size_bytes >= hwm_.hwm_lmdb_size_bytes_) + prune_messages(); +} + + +void persipubsub::queue::Queue::prune_messages() const { + + std::set messages_to_delete; + + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto rmeta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); + auto entries = boost::numeric_cast(rmeta_dbi.stat(rtxn).ms_entries); + + auto cursor = lmdb::cursor::open(rtxn, rmeta_dbi); + lmdb::val key, value; + + if (strategy_ == persipubsub::queue::Strategy::prune_first){ + cursor.get(key, value, MDB_FIRST); + for (unsigned int counter = 0; counter<=(entries/2); counter++){ + messages_to_delete.insert(std::move(key)); + cursor.get(key, value, MDB_NEXT); + } + } + else if (strategy_ == persipubsub::queue::Strategy::prune_last){ + cursor.get(key, value, MDB_LAST); + for (unsigned int counter = 0; counter<=(entries/2); counter++){ + messages_to_delete.insert(std::move(key)); + cursor.get(key, value, MDB_PREV); + } + } + else { + cursor.close(); + rtxn.abort(); + throw std::runtime_error("Pruning strategy not set."); + } + + cursor.close(); + rtxn.abort(); + + auto wtxn = lmdb::txn::begin(env_); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + std::vector dbis; + dbis.push_back(std::move(pending_dbi)); + dbis.push_back(std::move(meta_dbi)); + dbis.push_back(std::move(data_dbi)); + + for (const auto& sub_it: subscriber_ids_){ + auto sub_dbi = lmdb::dbi::open(wtxn, (sub_it).c_str()); + dbis.push_back(std::move(sub_dbi)); + } + + for (const auto& msg_it: messages_to_delete) { + for (auto& db_it: dbis){ + db_it.del(wtxn, msg_it); + } + } + + wtxn.commit(); +} + +persipubsub::queue::Queue::~Queue() {} + diff --git a/cpp/src/queue.h b/cpp/src/queue.h new file mode 100644 index 0000000..4f1833e --- /dev/null +++ b/cpp/src/queue.h @@ -0,0 +1,180 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#pragma once + +#include +#include +#include + +namespace persipubsub { + namespace queue { + + class Queue; + + static unsigned int MSG_TIMEOUT_SECS = 500; + static unsigned int MAX_MSGS_NUM = 1024 * 64; + static unsigned long HWM_LMDB_SIZE_BYTES = 30UL * 1024UL * 1024UL * 1024UL; + + /** + * Store possible strategies. + */ + enum Strategy { + prune_first = 0, + prune_last + }; + + /** + * Parse overflow strategy. + * + * @param strategy Strategy stored in config + * @return set overflow strategy + */ + Strategy parse_strategy(const std::string& strategy); + + /** + * Store high water mark limits. + */ + struct HighWaterMark{ + /** + * Initialize. + * + * @param msg_timeout_secs time after which msg is classified as dangling msg (secs) + * @param max_masgs_num maximal amount of msg + * @param hwm_lmdb_size_bytes high water mark for total size of lmdb (bytes) + */ + unsigned int msg_timeout_secs_ = MSG_TIMEOUT_SECS; + unsigned int max_msgs_num_ = MAX_MSGS_NUM; + unsigned long hwm_lmdb_size_bytes_ = HWM_LMDB_SIZE_BYTES; + + HighWaterMark() = default; + + HighWaterMark(const unsigned int msg_timeout_secs, + const unsigned int max_msgs_num, + const unsigned long hwm_lmdb_size) : msg_timeout_secs_(msg_timeout_secs), max_msgs_num_(max_msgs_num), hwm_lmdb_size_bytes_(hwm_lmdb_size) {} + }; + + /** + * Initialize the queue; the queue directory is assumed to exist. + * + * @param queue_dir where the queue is stored + * @param max_reader_num maximal number of reader + * @param max_db_num maximal number of databases + * @param max_db_size_bytes maximal size of database (bytes) + * @return Load or if needed create LMDB queue from directory + */ + lmdb::env initialize_environment(const boost::filesystem::path& queue_dir, const unsigned int max_reader_num, const unsigned int max_db_num, const unsigned long max_db_size_bytes); + + /** + * Prune all dangling messages for subscribers of a queue from lmdb. + * + * @param queue of which dangling messages should be pruned + * @param subscriber_ids subscribers of which dangling messages should be pruned + */ + void prune_dangling_messages_for(const persipubsub::queue::Queue& queue, const std::vector& subscriber_ids); + + /** + * Queue messages persistently from many publishers for many subscribers. + */ + class Queue{ + + public: + /** + * Initialize class object. + */ + Queue() : path_(), + env_(nullptr), hwm_(), strategy_(), + subscriber_ids_() {} + + /** + * Initialize the queue. + * + * @param config_pth path to the JSON config file + * @param queue_dir where the queue is stored + * @param max_reader_num maximal number of reader + * @param max_db_num maximal number of databases + * @param max_db_size_bytes maximal size of database (bytes) + */ + void init(const boost::filesystem::path& path, lmdb::env env= nullptr); + + /** + * Put message to lmdb queue. + * + * @param msg message send from publisher to subscribers + * @param subscriber_ids List of subscribers + */ + void put(const std::string& msg, const std::vector& subscriber_ids) const; + + /** + * Put many message to lmdb queue. + * + * @param msgs messages send from publisher to subscribers + * @param subscriber_ids List of subscribers + */ + void put_many_flush_once(const std::vector& msgs, const std::vector& subscriber_ids) const; + + /** + * Peek at next message in lmdb queue. + * + * Load from LMDB queue into memory and process msg afterwards. + * @param identifier Subscriber ID + * @param message peek on the message + */ + void front(const std::string& identifier, std::string* msg) const; + + /** + * Remove msg from the subscriber's queue and reduce pending subscribers. + * + * @param identifier Subscriber ID + */ + void pop(const std::string& identifier) const; + + /** + * Prune dangling messages in the queue. + */ + void prune_dangling_messages() const; + + /** + * Check current lmdb size in bytes. + * + * Check size of data database by approximating size with multiplying page size with number of pages. + * @return data database size in bytes + */ + unsigned long check_current_lmdb_size() const; + + /** + * Count number of messages in database. + * + * Count number of messages stored in meta database. + * + * @return number of messages in database + */ + unsigned int count_msgs() const; + + /** + * Clean database when needed. + */ + void vacuum() const; + + /** + * Prune one half of the messages stored. + * + * Depending on the strategy the first or the last will be deleted. + */ + void prune_messages() const; + + boost::filesystem::path path_; + lmdb::env env_; + HighWaterMark hwm_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + ~Queue(); + + private: + + }; // class Queue + + } // namespace queue +} // namespace persipubsub diff --git a/cpp/src/subscriber.h b/cpp/src/subscriber.h new file mode 100644 index 0000000..5d3197f --- /dev/null +++ b/cpp/src/subscriber.h @@ -0,0 +1,70 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub{ + namespace subscriber{ + /** + * Create Subscriber ready to receive messages. + */ + class Subscriber{ + + public: + /** + * Initialize class object. + */ + Subscriber() {}; + + /** + * Initialize. + * + * @param identifier unique subscriber id + * @param path to the queue + */ + void init(std::vector identifier, boost::filesystem::path path); + + /** + * Receive messages from the publisher. + * + * @param timeout time waiting for a message. If none arrived until the timeout then None will be returned. (secs) + * @param retries number of tries to check if a msg arrived in the queue + * @return received message + */ + char receive(int timeout, int retries); + + /** + * Pops all messages until the most recent one and receive the latest. + * Used in the case that a particular subscriber cares only about the very + * last message and other subscribers care about all the messages in the + * queue. + * For another use case, when you only want to store the latest message + * and all subscribers are interested only in the latest, then use + * high water mark max_msgs_num = 1. + * + * @param timeout time waiting for a message. If none arrived until the timeout then None will be returned. (secs) + * @param retries number of tries to check if a msg arrived in the queue + * @return received message + */ + char receive_to_top(int timeout, int retries); + + ~Subscriber(); + private: + std::vector identifier_; + persipubsub::queue::Queue queue_; + + /** + * Pop a message from the subscriber's lmdb. + */ + void pop(); + + }; // class Subscriber + } // namespace subscriber +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt new file mode 100644 index 0000000..d1d1791 --- /dev/null +++ b/cpp/test/CMakeLists.txt @@ -0,0 +1,13 @@ +find_package (Boost COMPONENTS system filesystem unit_test_framework REQUIRED) +include_directories (${TEST_SOURCE_DIR}/src + ${Boost_INCLUDE_DIRS} + ) + +add_definitions (-DBOOST_TEST_DYN_LINK) +add_executable (Test test_queue.cpp) +target_link_libraries (Test + persipubsub + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} + ) \ No newline at end of file diff --git a/cpp/test/test_queue.cpp b/cpp/test/test_queue.cpp new file mode 100644 index 0000000..65e6bb5 --- /dev/null +++ b/cpp/test/test_queue.cpp @@ -0,0 +1,281 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com/marko@parquery.com) +// on 15.02.2019 + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE TestQueue +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include "../src/queue.h" +#include "../src/library.h" + +namespace fs = boost::filesystem; + +class TmpDir{ +public: + TmpDir(){ + path_ = fs::unique_path(); + fs::create_directories(path_); + } + + ~TmpDir(){ + fs::remove_all(path_); + } + fs::path path_; +private: + +}; // class TmpDir + + +void setup_queue(const lmdb::txn& wtxn, const unsigned int msg_timeout_secs, + const unsigned int max_msgs_num, const unsigned long hwm_lmdb_size, + const std::string& strategy, const std::string& sub_ids){ + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + lmdb::val msg_timeout_secs_key(persipubsub::MSG_TIMEOUT_SECS_KEY); + lmdb::val msg_timeout_secs_val(boost::lexical_cast(msg_timeout_secs)); + queue_dbi.put(wtxn, msg_timeout_secs_key, msg_timeout_secs_val); + + lmdb::val max_msgs_num_key(persipubsub::MAX_MSGS_NUM_KEY); + lmdb::val max_msgs_num_val(boost::lexical_cast(max_msgs_num)); + queue_dbi.put(wtxn, max_msgs_num_key, max_msgs_num_val); + + lmdb::val hwm_lmdb_size_key(persipubsub::HWM_DB_SIZE_BYTES_KEY); + lmdb::val hwm_lmdb_size_val(boost::lexical_cast(hwm_lmdb_size)); + queue_dbi.put(wtxn, hwm_lmdb_size_key, hwm_lmdb_size_val); + + lmdb::val strategy_key(persipubsub::STRATEGY_KEY); + lmdb::val strategy_val(strategy); + queue_dbi.put(wtxn, strategy_key, strategy_val); + + lmdb::val subscriber_ids_key(persipubsub::SUBSCRIBER_IDS_KEY); + lmdb::val subscriber_ids_val(sub_ids); + queue_dbi.put(wtxn, subscriber_ids_key, subscriber_ids_val); +} + + +BOOST_AUTO_TEST_CASE(test_initialize_environment){ + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, 1024, 32UL * 1024UL * 1024UL * 1024UL); + + auto rtxn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + auto dbi = lmdb::dbi::open(rtxn, nullptr); + + MDB_envinfo info; + lmdb::env_info(env, &info); + + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(info.me_last_txnid)); + BOOST_REQUIRE_EQUAL("1", boost::lexical_cast(info.me_last_pgno)); + BOOST_REQUIRE_EQUAL("1024", boost::lexical_cast(info.me_maxreaders)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(info.me_mapaddr)); + BOOST_REQUIRE_EQUAL("34359738368", boost::lexical_cast(info.me_mapsize)); + BOOST_REQUIRE_EQUAL("1", boost::lexical_cast(info.me_numreaders)); + + MDB_stat stat = dbi.stat(rtxn); + + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_branch_pages)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_entries)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_depth)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_leaf_pages)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_overflow_pages)); + BOOST_REQUIRE_EQUAL("4096", boost::lexical_cast(stat.ms_psize)); + + rtxn.abort(); +} + +BOOST_AUTO_TEST_CASE(test_put_to_single_subscriber) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; // TODO read about carriage return, /0 and other funky issues + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + queue.put(msg, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + +} + +BOOST_AUTO_TEST_CASE(test_put_multiple_subscriber) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub1 sub2"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub1_dbi = lmdb::dbi::open(wtxn, "sub1", MDB_CREATE); + auto sub2_dbi = lmdb::dbi::open(wtxn, "sub2", MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.emplace_back("sub1"); + subs.emplace_back("sub2"); + queue.put(msg, subs); + + std::string result1; + queue.front("sub1", &result1); + BOOST_REQUIRE_EQUAL(msg, result1); + std::string result2; + queue.front("sub2", &result2); + BOOST_REQUIRE_EQUAL(msg, result2); +} + +BOOST_AUTO_TEST_CASE(test_put_many) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, + sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + + std::vector msgs; + unsigned int test_num_msgs = 10; + for (int num_msgs = test_num_msgs; num_msgs > 0; num_msgs--) + msgs.push_back(msg); + + queue.put_many_flush_once(msgs, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + BOOST_REQUIRE_EQUAL(test_num_msgs, queue.count_msgs()); +} + +BOOST_AUTO_TEST_CASE(test_pop) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, + sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + queue.put(msg, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + + queue.pop(sub_ids); + std::string empty; + queue.front(sub_ids, &empty); + BOOST_REQUIRE_EQUAL("", empty.c_str()); +} \ No newline at end of file From 9da14ea4da203ceea11ad94b20cf581e5d504868 Mon Sep 17 00:00:00 2001 From: slimeth Date: Mon, 25 Feb 2019 10:14:21 +0100 Subject: [PATCH 2/2] temporary commit --- cpp/.gitignore | 2 +- cpp/src/library.h | 104 +++++++++++++++++++++++++++------------- cpp/src/queue.cpp | 14 +++++- cpp/test/CMakeLists.txt | 2 +- 4 files changed, 85 insertions(+), 37 deletions(-) diff --git a/cpp/.gitignore b/cpp/.gitignore index 05d2a2a..db74f61 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -19,4 +19,4 @@ venv-doc/ # Archive obsolete/ -# Project specific \ No newline at end of file +# Project specific diff --git a/cpp/src/library.h b/cpp/src/library.h index 4deadb4..964c635 100644 --- a/cpp/src/library.h +++ b/cpp/src/library.h @@ -12,37 +12,75 @@ #include "queue.h" namespace persipubsub { - static unsigned int MAX_READER_NUM = 1024; - static unsigned int MAX_DB_NUM = 1024; - static unsigned long MAX_DB_SIZE_BYTES = 32UL * 1024UL * 1024UL * 1024UL; - - // define all database names here - static char DATA_DB[] = "data_db"; // msg_id | data - static char PENDING_DB[] = "pending_db"; // msg_id | pending subscriber - static char META_DB[] = "meta_db"; // msg_id | metadata - static char QUEUE_DB[] = "queue_db"; // queue_pth | all queue data - - static char HWM_DB_SIZE_BYTES_KEY[] = "hwm_db_size_bytes"; - static char MAX_MSGS_NUM_KEY[] = "max_msgs_num"; - static char MSG_TIMEOUT_SECS_KEY[] = "msg_timeout_secs"; - static char STRATEGY_KEY[] = "strategy"; - static char SUBSCRIBER_IDS_KEY[] = "subscriber_ids"; - - struct QueueData{ - unsigned int msg_timeout_secs_; - unsigned int max_msgs_num_; - unsigned long hwm_db_size_bytes_; - persipubsub::queue::Strategy strategy_; - std::vector subscriber_ids_; - - QueueData(unsigned int msg_timeout_secs, unsigned int max_msgs_num, - unsigned long hwm_db_size_bytes, - persipubsub::queue::Strategy strategy, - std::vector subscriber_ids) : - msg_timeout_secs_(msg_timeout_secs), - max_msgs_num_(max_msgs_num), - hwm_db_size_bytes_(hwm_db_size_bytes), strategy_(strategy), - subscriber_ids_(std::move(subscriber_ids)) {}; - }; - QueueData lookup_queue_data(const lmdb::env& env); +static unsigned int MAX_READER_NUM = 1024; +static unsigned int MAX_DB_NUM = 1024; +static unsigned long MAX_DB_SIZE_BYTES = 32UL * 1024UL * 1024UL * 1024UL; + +// define all database names here +static char DATA_DB[] = "data_db"; // msg_id | data +static char PENDING_DB[] = "pending_db"; // msg_id | pending subscriber +static char META_DB[] = "meta_db"; // msg_id | metadata +static char QUEUE_DB[] = "queue_db"; // queue_pth | all queue data + +static char HWM_DB_SIZE_BYTES_KEY[] = "hwm_db_size_bytes"; +static char MAX_MSGS_NUM_KEY[] = "max_msgs_num"; +static char MSG_TIMEOUT_SECS_KEY[] = "msg_timeout_secs"; +static char STRATEGY_KEY[] = "strategy"; +static char SUBSCRIBER_IDS_KEY[] = "subscriber_ids"; + +struct QueueData { + unsigned int msg_timeout_secs_; + unsigned int max_msgs_num_; + unsigned long hwm_db_size_bytes_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + QueueData(unsigned int msg_timeout_secs, unsigned int max_msgs_num, + unsigned long hwm_db_size_bytes, + persipubsub::queue::Strategy strategy, + std::vector subscriber_ids) : + msg_timeout_secs_(msg_timeout_secs), + max_msgs_num_(max_msgs_num), + hwm_db_size_bytes_(hwm_db_size_bytes), strategy_(strategy), + subscriber_ids_(std::move(subscriber_ids)) {}; +}; + +QueueData lookup_queue_data(const lmdb::env &env); + + +class ReadTransaction { + +public: + ReadTransaction(const lmdb::env& env) : rtxn_(lmdb::txn::begin(env, nullptr, MDB_RDONLY)) {} + + ~ReadTransaction() { + rtxn_.abort(); + } + + lmdb::txn rtxn_; +}; + +class WriteTransaction { + +public: + WriteTransaction(const lmdb::env& env) : wtxn_(lmdb::txn::begin(env, nullptr)) {} + + ~WriteTransaction() { + wtxn_.commit(); + } + + lmdb::txn wtxn_; +}; + +class Cursor { + +public: + Cursor(const lmdb::txn& txn, const lmdb::dbi& dbi) : cursor_(lmdb::cursor::open(txn, dbi)) {} + + ~Cursor() { + cursor_.close(); + } + + lmdb::cursor cursor_; +}; } // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/queue.cpp b/cpp/src/queue.cpp index 043ce84..dd5e60c 100644 --- a/cpp/src/queue.cpp +++ b/cpp/src/queue.cpp @@ -64,7 +64,16 @@ lmdb::env persipubsub::queue::initialize_environment(const fs::path& queue_dir, void persipubsub::queue::prune_dangling_messages_for(const persipubsub::queue::Queue& queue, const std::vector &subscriber_ids){ - auto wtxn = lmdb::txn::begin(queue.env_); + /*todo check mistake: + * terminate called after throwing an instance of 'lmdb::runtime_error' + what(): mdb_txn_commit: Invalid argument + unknown location(0): fatal error in "test_put_to_single_subscriber": signal: SIGABRT (application abort requested) + /home/selim/workspace/pqry/persipubsub/cpp/test/test_queue.cpp(99): last checkpoint + terminate called recursively + */ + + auto wtxn = persipubsub::WriteTransaction(queue.env_).wtxn_; + //auto wtxn = lmdb::txn::begin(queue.env_); auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); @@ -116,7 +125,8 @@ void persipubsub::queue::prune_dangling_messages_for(const persipubsub::queue::Q } } - wtxn.commit(); + //todo check if needed + //wtxn.commit(); } // todo add transaction and cursor classes diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt index d1d1791..cf71ff2 100644 --- a/cpp/test/CMakeLists.txt +++ b/cpp/test/CMakeLists.txt @@ -10,4 +10,4 @@ target_link_libraries (Test ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} - ) \ No newline at end of file + )