From f6a4823880e400546b9f39c7b639dc0aa582f7fa Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 12 Dec 2018 11:24:23 +0100 Subject: [PATCH 01/35] Added why not section in README --- .gitignore | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 -- README.rst | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 .gitignore delete mode 100644 README.md create mode 100644 README.rst diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b534413 --- /dev/null +++ b/.gitignore @@ -0,0 +1,65 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Sphinx documentation +docs/build/ + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# mypy +.mypy_cache/ + +# IDE +.idea + +# Archive +obsolete/ \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index fab3032..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# persipubsub -TODO diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f877af4 --- /dev/null +++ b/README.rst @@ -0,0 +1,39 @@ +persipubsub +=========== + +*catch phrase in work* + + +Why not ... +----------- + +persist-queue +^^^^^^^^^^^^^ +* `protobuf `_ messages can't + be send except if you deserialize it and serialize it with pickle. +* Offers not all functionality expected from a queue. Has put and get function + which are basically only push and pop. So front/peak functionality is missing. + So neither can multiple subscriber be implemented nor can't be guaranteed that + no data is lost when a thread fails. + +Kafka +^^^^^ +* For our needs as local system with IPC it's an unnecessary overhead. +* Integration of Kafka written in Scala and Java in our C++/Python/Go codebase + not that simple. +* Hard to setup and maintain `Kafka `_. +* broker system eventually a bottleneck. + +RabbitMQ +^^^^^^^^ +* For our needs as local system with IPC it's an unnecessary overhead. +* broker system eventually a bottleneck. +* `RabbitMQ `_ is less scalable than Kafka, but is + supported officially by more languages. + +zeromq persistence pattern +^^^^^^^^^^^^^^^^^^^^^^^^^^ +* Only persistence pattern of `zeromq `_ is + `Titanic `_ which is also a broker + system. This takes away the purpose and advantage of zeromq to be a + lightweight library which requires no broker. From 34e06fda1cd02aedd062ab564159673305b05492 Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 3 Jan 2019 11:10:22 +0100 Subject: [PATCH 02/35] basic project setup --- CHANGELOG.rst | 3 + LICENSE | 21 ++ docs/Makefile | 19 ++ docs/source/changelog.rst | 5 + docs/source/conf.py | 189 ++++++++++++++++++ docs/source/index.rst | 22 ++ docs/source/persipubsub/index.rst | 9 + docs/source/persipubsub/persipubsub.ctl.rst | 5 + .../persipubsub/persipubsub.publisher.rst | 5 + .../persipubsub/persipubsub.subscriber.rst | 5 + docs/source/readme.rst | 4 + mypy.ini | 1 + persipubsub_meta.py | 10 + precommit.py | 106 ++++++++++ pylint.rc | 10 + requirements-doc.txt | 4 + requirements.txt | 3 + setup.py | 63 ++++++ style.yapf | 6 + tox.ini | 11 + 20 files changed, 501 insertions(+) create mode 100644 CHANGELOG.rst create mode 100644 LICENSE create mode 100644 docs/Makefile create mode 100644 docs/source/changelog.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 docs/source/persipubsub/index.rst create mode 100644 docs/source/persipubsub/persipubsub.ctl.rst create mode 100644 docs/source/persipubsub/persipubsub.publisher.rst create mode 100644 docs/source/persipubsub/persipubsub.subscriber.rst create mode 100644 docs/source/readme.rst create mode 100644 mypy.ini create mode 100644 persipubsub_meta.py create mode 100644 precommit.py create mode 100644 pylint.rc create mode 100644 requirements-doc.txt create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 style.yapf create mode 100644 tox.ini diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..43bb01b --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,3 @@ +1.0.0 +===== +* Initial version \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..706450c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Parquery AG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..ba501f6 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst new file mode 100644 index 0000000..ea1f942 --- /dev/null +++ b/docs/source/changelog.rst @@ -0,0 +1,5 @@ +********* +CHANGELOG +********* + +.. include:: ../../CHANGELOG.rst \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..ec249a8 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,189 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + +import persipubsub_meta + + +# -- Project information ----------------------------------------------------- + +project = persipubsub_meta.__title__ +copyright = persipubsub_meta.__copyright__ +author = persipubsub_meta.__author__ +description = persipubsub_meta.__description__ + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = persipubsub_meta.__version__ + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx_autodoc_typehints', + 'sphinx_icontract' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'persipubsubdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, + '{}.tex'.format(project), + '{} Documentation'.format(project), + author, + 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, + project, + '{} Documentation'.format(project), + [author], + 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, project, '{} Documentation'.format(project), + author, project, description, + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..08994e3 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,22 @@ +.. gswrap master file, created by + sphinx-quickstart on Wed Dec 5 15:14:21 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to persipubsub's documentation! +================================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + persipubsub/index.rst + readme + changelog + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` \ No newline at end of file diff --git a/docs/source/persipubsub/index.rst b/docs/source/persipubsub/index.rst new file mode 100644 index 0000000..dbf7647 --- /dev/null +++ b/docs/source/persipubsub/index.rst @@ -0,0 +1,9 @@ +########### +persipubsub +########### + +.. toctree:: + + persipubsub.ctl + persipubsub.publisher + persipubsub.subscriber \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.ctl.rst b/docs/source/persipubsub/persipubsub.ctl.rst new file mode 100644 index 0000000..65a03d6 --- /dev/null +++ b/docs/source/persipubsub/persipubsub.ctl.rst @@ -0,0 +1,5 @@ +Control +******* + +.. automodule:: persipubsub.control + :members: \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.publisher.rst b/docs/source/persipubsub/persipubsub.publisher.rst new file mode 100644 index 0000000..a6a6454 --- /dev/null +++ b/docs/source/persipubsub/persipubsub.publisher.rst @@ -0,0 +1,5 @@ +Publisher +********* + +.. automodule:: persipubsub.publisher + :members: diff --git a/docs/source/persipubsub/persipubsub.subscriber.rst b/docs/source/persipubsub/persipubsub.subscriber.rst new file mode 100644 index 0000000..dc53bdf --- /dev/null +++ b/docs/source/persipubsub/persipubsub.subscriber.rst @@ -0,0 +1,5 @@ +Subscriber +********** + +.. automodule:: persipubsub.subscriber + :members: diff --git a/docs/source/readme.rst b/docs/source/readme.rst new file mode 100644 index 0000000..787f7d6 --- /dev/null +++ b/docs/source/readme.rst @@ -0,0 +1,4 @@ +****** +README +****** +.. include:: ../../README.rst \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..a7f647e --- /dev/null +++ b/mypy.ini @@ -0,0 +1 @@ +[mypy] \ No newline at end of file diff --git a/persipubsub_meta.py b/persipubsub_meta.py new file mode 100644 index 0000000..5be4657 --- /dev/null +++ b/persipubsub_meta.py @@ -0,0 +1,10 @@ +"""Define meta information about persipubsub package.""" + +__title__ = 'persipubsub' +__description__ = 'Send messages persistent from publishers to subscribers.' +__url__ = 'https://github.com/Parquery/persipubsub' +__version__ = '1.0.0' +__author__ = 'Selim Naji, Adam Radomski and Marko Ristin' +__author_email__ = 'selim.naji@parquery.com, adam.radomski@parquery.com, marko.ristin@gmail.com' +__license__ = 'MIT' +__copyright__ = 'Copyright 2018 Parquery AG' \ No newline at end of file diff --git a/precommit.py b/precommit.py new file mode 100644 index 0000000..55b6de6 --- /dev/null +++ b/precommit.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Runs precommit checks on the repository.""" +import argparse +import os +import pathlib +import subprocess +import sys +from typing import List # pylint: disable=unused-imports + + +def main() -> int: + """" + Main routine + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + help= + "Overwrites the unformatted source files with the well-formatted code in place. " + "If not set, an exception is raised if any of the files do not conform to the style guide.", + action='store_true') + + args = parser.parse_args() + + overwrite = bool(args.overwrite) + + repo_root = pathlib.Path(__file__).parent + + print("YAPF'ing...") + if overwrite: + subprocess.check_call([ + "yapf", "--in-place", "--style=style.yapf", "--recursive", "tests", + "persipubsub", "setup.py", "precommit.py" + ], + cwd=repo_root.as_posix()) + else: + subprocess.check_call([ + "yapf", "--diff", "--style=style.yapf", "--recursive", "tests", + "persipubsub", "bin", "setup.py", "precommit.py" + ], + cwd=repo_root.as_posix()) + + print("Mypy'ing...") + subprocess.check_call(["mypy", "persipubsub", "tests"], + cwd=repo_root.as_posix()) + + print("Isort'ing...") + isort_files = [] # type: List[str] + for path in (repo_root / "persipubsub").glob("**/*.py"): + isort_files.append(path.as_posix()) + for path in (repo_root / "tests").glob("**/*.py"): + isort_files.append(path.as_posix()) + + if overwrite: + cmd = [ + "isort", "--balanced", "--multi-line", "4", "--line-width", "80", + "--dont-skip", "__init__.py", "--project", "persipubsub" + ] + cmd.extend(isort_files) + subprocess.check_call(cmd) + else: + cmd = [ + "isort", "--check-only", "--diff", "--balanced", "--multi-line", + "4", "--line-width", "80", "--dont-skip", "__init__.py", + "--project", "persipubsub" + ] + cmd.extend(isort_files) + subprocess.check_call(cmd) + + print("Pylint'ing...") + subprocess.check_call( + ["pylint", "--rcfile=pylint.rc", "tests", "persipubsub"], + cwd=repo_root.as_posix()) + + print("Pydocstyle'ing...") + subprocess.check_call(["pydocstyle", "persipubsub"], + cwd=repo_root.as_posix()) + + print("Testing...") + env = os.environ.copy() + env['ICONTRACT_SLOW'] = 'true' + + subprocess.check_call([ + "coverage", "run", "--source", "persipubsub", "-m", "unittest", + "discover", "tests" + ], + cwd=repo_root.as_posix(), + env=env) + + subprocess.check_call(["coverage", "report"]) + + print("Doctesting...") + subprocess.check_call( + ["python3", "-m", "doctest", (repo_root / "README.rst").as_posix()]) + for pth in (repo_root / "persipubsub").glob("**/*.py"): + subprocess.check_call(["python3", "-m", "doctest", pth.as_posix()]) + + print("pyicontract-lint'ing...") + for pth in (repo_root / "persipubsub").glob("**/*.py"): + subprocess.check_call(["pyicontract-lint", pth.as_posix()]) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pylint.rc b/pylint.rc new file mode 100644 index 0000000..bca2e5d --- /dev/null +++ b/pylint.rc @@ -0,0 +1,10 @@ +[TYPECHECK] +ignored-modules = numpy +ignored-classes = numpy,PurePath +generated-members=bottle\.request\.forms\.decode,bottle\.request\.query\.decode + +[FORMAT] +max-line-length=80 + +[MESSAGES CONTROL] +disable=too-few-public-methods,abstract-class-little-used,len-as-condition,bad-continuation,bad-whitespace \ No newline at end of file diff --git a/requirements-doc.txt b/requirements-doc.txt new file mode 100644 index 0000000..a488d81 --- /dev/null +++ b/requirements-doc.txt @@ -0,0 +1,4 @@ +sphinx>=1.8.2,<2 +sphinx-autodoc-typehints>=1.5.2 +sphinx-icontract>=2.0.0 +sphinx-rtd-theme>=0.4.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c42f07d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +typing-extensions>=3.6.6 +icontract>=2.0.1,<3 +asttokens>=1.1.13,<2 \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..eca6f92 --- /dev/null +++ b/setup.py @@ -0,0 +1,63 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" +import os + +from setuptools import find_packages, setup + +import persipubsub_meta + +here = os.path.abspath(os.path.dirname(__file__)) # pylint: disable=invalid-name + +with open(os.path.join(here, 'README.rst'), encoding='utf-8') as fid: + long_description = fid.read().strip() # pylint: disable=invalid-name + +with open(os.path.join(here, 'requirements.txt'), encoding='utf-8') as fid: + install_requires = [ + line for line in fid.read().splitlines() if line.strip() + ] + +setup( + name=persipubsub_meta.__title__, + version=persipubsub_meta.__version__, + description=persipubsub_meta.__description__, + long_description=long_description, + url=persipubsub_meta.__url__, + author=persipubsub_meta.__author__, + author_email=persipubsub_meta.__author_email__, + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + ], + license='License :: OSI Approved :: MIT License', + keywords= + 'persistent publisher subscriber lmdb MQ message queue thread-safe process-safe', + packages=find_packages(exclude=['tests']), + install_requires=install_requires, + extras_require={ + 'dev': [ + # yapf: disable + 'mypy==0.641', + 'pylint==2.2.2', + 'yapf==0.25.0', + 'tox>=3.5.3', + 'coverage>=4.5.2,<5', + 'pydocstyle>=3.0.0,<4', + 'pyicontract-lint>=2.0.0,<3', + 'docutils>=0.14,<1', + 'isort>=4.3.4,<5', + 'pygments>=2.3.0,<3', + 'logthis>=1.0.1,<2', + # yapf: enable + ] + }, + py_modules=['persipubsub', 'persipubsub_meta'], + include_package_data=True, + package_data={"persipubsub": ["py.typed"]}, + data_files=[('.', ['LICENSE', 'README.rst', 'requirements.txt'])]) diff --git a/style.yapf b/style.yapf new file mode 100644 index 0000000..ef924b1 --- /dev/null +++ b/style.yapf @@ -0,0 +1,6 @@ +[style] +based_on_style = pep8 +spaces_before_comment = 2 +split_before_logical_operator = true +column_limit = 80 +coalesce_brackets = true \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..f660911 --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py35,py36 + +[testenv] +deps = .[dev] +changedir = {envtmpdir} +commands = + python3 {toxinidir}/precommit.py + +setenv = + COVERAGE_FILE={envbindir}/.coverage \ No newline at end of file From f4813fe6a2797b020696cbf8b6be5d25daf2b1fe Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 3 Jan 2019 11:13:48 +0100 Subject: [PATCH 03/35] added interface --- persipubsub/__init__.py | 1 + persipubsub/control.py | 66 +++++++++++++++++++++++++++++++++++++++ persipubsub/publisher.py | 29 +++++++++++++++++ persipubsub/subscriber.py | 39 +++++++++++++++++++++++ 4 files changed, 135 insertions(+) create mode 100644 persipubsub/__init__.py create mode 100644 persipubsub/control.py create mode 100644 persipubsub/publisher.py create mode 100644 persipubsub/subscriber.py diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py new file mode 100644 index 0000000..eabc1c2 --- /dev/null +++ b/persipubsub/__init__.py @@ -0,0 +1 @@ +"""Distribute messages persistent from publisher to subscriber.""" diff --git a/persipubsub/control.py b/persipubsub/control.py new file mode 100644 index 0000000..75efe58 --- /dev/null +++ b/persipubsub/control.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python +"""Setup and control persistent distribution of messages.""" + +import pathlib +from typing import Union + +import lmdb + + +class CTL: + """Create control unit.""" + + def __init__(self, + env_dir: Union[str, pathlib.Path], + max_readers: int=1024, + max_dbs: int = 1024, + max_db_size: int = 32 * 1024**3): + """ + Initialize. + + :param env_dir: directory where lmdb is stored + :param max_dbs: maximal number of databases + :param max_db_size: maximal size of database in bytes + """ + env_dir_str = env_dir if isinstance(env_dir, str) \ + else env_dir.as_posix() + self.env = lmdb.open(path=env_dir_str, map_size=max_db_size, + subdir=True, max_readers=max_readers, + max_dbs=max_dbs) + self.hwm = HighWaterMark() + + def clear_all_subs(self): + """Clear all subscriber and delete all messages.""" + + def add_sub(self, sub_id: str): + """Add a subscriber and create its lmdb.""" + + def remove_sub(self, sub_id: str): + """Remove a subscriber and delete all its messages.""" + + def prune_dangling_messages(self): + """Prune all dangling messages from the lmdb.""" + + def prune_all_messages_for(self, sub_id: str): + """Prune all messages of a subscriber.""" + + def set_hwm(self, size: int): + """Set high-water-mark limit.""" + + def set_overflow_strategy(self, strategy: bool = True): + """ + Set the overflow strategy. + + :param strategy: + | if True, the first messages will be pruned. + | if False, the last messages will be pruned. + :return: + """ + + +class HighWaterMark: + + def __init__(self, msg_timeout: int = 500, max_msgs: int = 1024*64, hwm_lmdb_size: int = 30 * 1024**3): + self.msg_timeout = msg_timeout + self.max_msgs = max_msgs + self.hwm_lmdb_size = hwm_lmdb_size \ No newline at end of file diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py new file mode 100644 index 0000000..888fe4f --- /dev/null +++ b/persipubsub/publisher.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +"""Distribute messages persistent from publisher to subscriber.""" + +import pathlib +from typing import Union + + +class Pub: + """Create Publisher ready to send messages.""" + + def __init__(self, + pub_dir: Union[str, pathlib.Path], + autosync: bool = False): + """ + Initialize. + + :param pub_dir: Directory where publisher data is stored + :param autosync: if True, store data automatically in lmdb + """ + + def send(self, msg: bytes): + """ + Send messages to subscriber. + + :param msg: to send to all subscribers + """ + + def sync(self): + """Store message on disk in lmdb.""" diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py new file mode 100644 index 0000000..4630f89 --- /dev/null +++ b/persipubsub/subscriber.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +"""Receive messages persistent from publisher to subscriber.""" + +import contextlib +import pathlib +from typing import Union, Optional +import time + + +class Sub: + """Create Subscriber ready to receive messages.""" + + def __init__(self, sub_dir: Union[str, pathlib.Path], sub_id: str): + """ + Initialize. + + :param sub_dir: Directory where subscriber data is stored + :param sub_id: unique subscriber id + """ + + @contextlib.contextmanager + def receive(self, timeout: int = 60) -> Optional[str]: + """ + Receive messages from the publisher. + + :param timeout: time waiting for a message. If none arrived until the + timeout then None will be returned. + """ + # receive something + + def pop(self): + """Pop a message from the subscriber's lmdb.""" + + def pop_to_top(self): + """ + Pops all messages until the most recent one. + + Used for slow processes to ensure realtime processing. + """ From 5020aec0be7bd4163d6261983800cf39d90866dd Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 3 Jan 2019 11:49:53 +0100 Subject: [PATCH 04/35] minor change for method consistency --- persipubsub/control.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/persipubsub/control.py b/persipubsub/control.py index 75efe58..03d6d17 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -44,8 +44,16 @@ def prune_dangling_messages(self): def prune_all_messages_for(self, sub_id: str): """Prune all messages of a subscriber.""" - def set_hwm(self, size: int): - """Set high-water-mark limit.""" + def set_hwm(self, msg_timeout: int = 500, max_msgs: int = 1024*64, + hwm_lmdb_size: int = 30 * 1024**3): + """ + Set high-water-mark limit. + + :param msg_timeout: time after which msg is classified as dangling msg + :param max_msgs: maximal amount of msg + :param hwm_lmdb_size: high water mark for total size of lmdb + :return: + """ def set_overflow_strategy(self, strategy: bool = True): """ @@ -60,7 +68,15 @@ def set_overflow_strategy(self, strategy: bool = True): class HighWaterMark: - def __init__(self, msg_timeout: int = 500, max_msgs: int = 1024*64, hwm_lmdb_size: int = 30 * 1024**3): + def __init__(self, msg_timeout: int = 500, max_msgs: int = 1024*64, + hwm_lmdb_size: int = 30 * 1024**3): + """ + Initialize. + + :param msg_timeout: time after which msg is classified as dangling msg + :param max_msgs: maximal amount of msg + :param hwm_lmdb_size: high water mark for total size of lmdb + """ self.msg_timeout = msg_timeout self.max_msgs = max_msgs - self.hwm_lmdb_size = hwm_lmdb_size \ No newline at end of file + self.hwm_lmdb_size = hwm_lmdb_size From 4486acd3a437cb63af60ad6c63c34407bc133292 Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 24 Jan 2019 17:32:48 +0100 Subject: [PATCH 05/35] first working version --- .travis.yml | 13 + README.rst | 4 +- docs/source/persipubsub/index.rst | 1 + docs/source/persipubsub/persipubsub.queue.rst | 5 + mypy.ini | 11 +- persipubsub/__init__.py | 62 +++ persipubsub/control.py | 259 +++++++--- persipubsub/publisher.py | 70 ++- persipubsub/queue.py | 489 ++++++++++++++++++ persipubsub/subscriber.py | 80 ++- precommit.py | 6 + pylint.rc | 2 +- requirements.txt | 3 - setup.py | 19 +- tests/__init__.py | 82 +++ tests/test_control.py | 261 ++++++++++ tests/test_publisher.py | 125 +++++ tests/test_queue.py | 410 +++++++++++++++ tests/test_subscriber.py | 129 +++++ 19 files changed, 1919 insertions(+), 112 deletions(-) create mode 100644 .travis.yml create mode 100644 docs/source/persipubsub/persipubsub.queue.rst create mode 100644 persipubsub/queue.py delete mode 100644 requirements.txt create mode 100644 tests/__init__.py create mode 100644 tests/test_control.py create mode 100644 tests/test_publisher.py create mode 100644 tests/test_queue.py create mode 100644 tests/test_subscriber.py diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..fd0e703 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +language: python +python: + - "3.5" + - "3.6" +addons: + apt: + packages: +install: + - pip3 install -e .[dev] + - pip3 install coveralls +script: + - python3 precommit.py + - coveralls \ No newline at end of file diff --git a/README.rst b/README.rst index f877af4..400f6e1 100644 --- a/README.rst +++ b/README.rst @@ -9,8 +9,8 @@ Why not ... persist-queue ^^^^^^^^^^^^^ -* `protobuf `_ messages can't - be send except if you deserialize it and serialize it with pickle. +* `protobuf `_ messages can't + be send with `persist-queue `_ except if you deserialize it and serialize it with pickle. * Offers not all functionality expected from a queue. Has put and get function which are basically only push and pop. So front/peak functionality is missing. So neither can multiple subscriber be implemented nor can't be guaranteed that diff --git a/docs/source/persipubsub/index.rst b/docs/source/persipubsub/index.rst index dbf7647..96e971e 100644 --- a/docs/source/persipubsub/index.rst +++ b/docs/source/persipubsub/index.rst @@ -5,5 +5,6 @@ persipubsub .. toctree:: persipubsub.ctl + persipubsub.queue persipubsub.publisher persipubsub.subscriber \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.queue.rst b/docs/source/persipubsub/persipubsub.queue.rst new file mode 100644 index 0000000..3eccc54 --- /dev/null +++ b/docs/source/persipubsub/persipubsub.queue.rst @@ -0,0 +1,5 @@ +Queue +***** + +.. automodule:: persipubsub.queue + :members: \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index a7f647e..1a2efd2 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1 +1,10 @@ -[mypy] \ No newline at end of file +[mypy] + +[mypy-messages] +ignore_missing_imports = True + +[mypy-messages.message_pb2] +ignore_missing_imports = True + +[mypy-lmdb] +ignore_missing_imports = True \ No newline at end of file diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py index eabc1c2..30d30ee 100644 --- a/persipubsub/__init__.py +++ b/persipubsub/__init__.py @@ -1 +1,63 @@ +#!/usr/bin/env python """Distribute messages persistent from publisher to subscriber.""" +import json +import pathlib +from typing import Dict, Union + +# define all encodings here +ENCODING = 'utf-8' +BYTES_LENGTH = 8 +BYTES_ORDER = 'big' + +# define all database names here +DATA_DB = "data_db".encode(ENCODING) # msg_id | data +PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber +META_DB = "meta_db".encode(ENCODING) # msg_id | metadata + + +def encoding(string: str) -> bytes: + """ + Encode a string with utf-8 encoding. + + :param string: any string + :return: string encoded as utf-8 + """ + return string.encode(encoding=ENCODING) + + +def int_to_bytes(value: int) -> bytes: + """ + Encode an integer to an array of bytes. + + :param value: any integer + :return: integer value representation as bytes + """ + return value.to_bytes(length=BYTES_LENGTH, byteorder=BYTES_ORDER) + + +def bytes_to_int(array_of_bytes: bytes) -> int: + """ + Decode an array of bytes to an integer. + + :param array_of_bytes: any array of bytes + :return: array of bytes representation as integer value + """ + return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) + + +def get_config(path: Union[pathlib.Path, str]) -> Dict[str, Dict[str, str]]: + """ + Get queues, publishers and subscribers data. + + :param path: path to the JSON config file + :return config data with queues, publishers and subscribers + """ + pth = path if isinstance(path, pathlib.Path) else pathlib.Path(path) + + if not pth.is_file(): + raise FileNotFoundError("Config file at path '{}' not found".format( + pth.as_posix())) + + with open(pth.as_posix(), mode="rt") as file_object: + config = json.load(file_object) + return config diff --git a/persipubsub/control.py b/persipubsub/control.py index 03d6d17..f02af75 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -2,81 +2,184 @@ """Setup and control persistent distribution of messages.""" import pathlib -from typing import Union - -import lmdb - - -class CTL: - """Create control unit.""" - - def __init__(self, - env_dir: Union[str, pathlib.Path], - max_readers: int=1024, - max_dbs: int = 1024, - max_db_size: int = 32 * 1024**3): - """ - Initialize. - - :param env_dir: directory where lmdb is stored - :param max_dbs: maximal number of databases - :param max_db_size: maximal size of database in bytes - """ - env_dir_str = env_dir if isinstance(env_dir, str) \ - else env_dir.as_posix() - self.env = lmdb.open(path=env_dir_str, map_size=max_db_size, - subdir=True, max_readers=max_readers, - max_dbs=max_dbs) - self.hwm = HighWaterMark() - - def clear_all_subs(self): - """Clear all subscriber and delete all messages.""" - - def add_sub(self, sub_id: str): - """Add a subscriber and create its lmdb.""" - - def remove_sub(self, sub_id: str): - """Remove a subscriber and delete all its messages.""" - - def prune_dangling_messages(self): - """Prune all dangling messages from the lmdb.""" - - def prune_all_messages_for(self, sub_id: str): - """Prune all messages of a subscriber.""" - - def set_hwm(self, msg_timeout: int = 500, max_msgs: int = 1024*64, - hwm_lmdb_size: int = 30 * 1024**3): - """ - Set high-water-mark limit. - - :param msg_timeout: time after which msg is classified as dangling msg - :param max_msgs: maximal amount of msg - :param hwm_lmdb_size: high water mark for total size of lmdb - :return: - """ - - def set_overflow_strategy(self, strategy: bool = True): - """ - Set the overflow strategy. - - :param strategy: - | if True, the first messages will be pruned. - | if False, the last messages will be pruned. - :return: - """ - - -class HighWaterMark: - - def __init__(self, msg_timeout: int = 500, max_msgs: int = 1024*64, - hwm_lmdb_size: int = 30 * 1024**3): - """ - Initialize. - - :param msg_timeout: time after which msg is classified as dangling msg - :param max_msgs: maximal amount of msg - :param hwm_lmdb_size: high water mark for total size of lmdb - """ - self.msg_timeout = msg_timeout - self.max_msgs = max_msgs - self.hwm_lmdb_size = hwm_lmdb_size +from typing import Any, Dict, List, Union + +import persipubsub.queue + + +def initialize_all_dbs(config_pth: Union[pathlib.Path, str]) -> None: + """ + Initialize all needed queues. + + :param config_pth: path to the JSON config file + """ + config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] + queues = config["queues"] # type: Dict[str, Any] + # TODO(snaji): maybe multi-threaded especially different queues + for values in queues.values(): + queue_dir = pathlib.Path(values["path"]) + if not queue_dir.exists(): + queue_dir.mkdir(parents=True, exist_ok=True) + sub_list = values["subscribers"] + queue = persipubsub.queue.Queue() + queue.init( + config_pth=config_pth, + queue_dir=queue_dir, + max_reader_num=values["max_reader_num"], + max_db_num=values["max_db_num"], + max_db_size_bytes=values["max_db_size_bytes"]) + + for sub in sub_list: + add_sub(sub_id=sub, queue=queue) + + +def clear_all_subs(config_pth: Union[pathlib.Path, str]) -> None: + """ + Clear all subscriber and delete all messages. + + :param config_pth: path to the JSON config file + """ + config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] + queues = config["queues"] # type: Dict[str, Any] + for values in queues.values(): + queue_dir = pathlib.Path(values["path"]) + if not queue_dir.exists(): + queue_dir.mkdir(parents=True, exist_ok=True) + sub_list = values["subscribers"] # type: List[str] + queue = persipubsub.queue.Queue() + queue.init( + config_pth=config_pth, + queue_dir=queue_dir, + max_reader_num=values["max_reader_num"], + max_db_num=values["max_db_num"], + max_db_size_bytes=values["max_db_size_bytes"]) + clear_all_subs_for(queue=queue, sub_list=sub_list) + + +def clear_all_subs_for(queue: persipubsub.queue.Queue, + sub_list: List[str]) -> None: + """ + Clear all subscriber and delete all messages for one queue. + + :param queue: which will be cleared + :param sub_list: subscribers which will be removed + """ + with queue.env.begin(write=True) as txn: + for sub_id in sub_list: + sub_db = queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + txn.drop(db=sub_db) + + pending_db = queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + txn.drop(db=pending_db, delete=False) + meta_db = queue.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + txn.drop(db=meta_db, delete=False) + data_db = queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + txn.drop(db=data_db, delete=False) + + +def add_sub(sub_id: str, queue: persipubsub.queue.Queue) -> None: + """ + Add a subscriber and create its lmdb. + + :param sub_id: ID of the subscriber which should be added + :param queue: to which the subscriber is added + """ + with queue.env.begin(write=True) as txn: + _ = queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=True) + + +def remove_sub(sub_id: str, queue: persipubsub.queue.Queue) -> None: + """ + Remove a subscriber and delete all its messages. + + :param sub_id: ID of the subscriber which should be removed + :param queue: from which the subscriber is removed + """ + msg_of_sub = set() + with queue.env.begin(write=True) as txn: + sub_db = queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + # TODO(snaji): messages in pending_db? + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + for key in cursor.iternext(keys=True, values=False): + msg_of_sub.add(key) + txn.drop(db=sub_db) + + pending_db = queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + for key in msg_of_sub: + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + + +def prune_dangling_messages(config_pth: Union[pathlib.Path, str]) -> None: + """ + Prune all dangling messages from the lmdb. + + :param config_pth: path to the JSON config file + """ + config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] + queues = config["queues"] # type: Dict[str, Any] + for values in queues.values(): + queue_dir = pathlib.Path(values["path"]) + if not queue_dir.exists(): + queue_dir.mkdir(parents=True, exist_ok=True) + sub_list = values["subscribers"] # type: List[str] + queue = persipubsub.queue.Queue() + queue.init( + config_pth=config_pth, + queue_dir=queue_dir, + max_reader_num=values["max_reader_num"], + max_db_num=values["max_db_num"], + max_db_size_bytes=values["max_db_size_bytes"]) + persipubsub.queue.prune_dangling_messages_for( + queue=queue, sub_list=sub_list) + + +def prune_all_messages_for(sub_id: str, queue: persipubsub.queue.Queue) -> None: + """ + Prune all messages of a subscriber. + + :param sub_id: ID of the subscriber of which all messages should be pruned + :param queue: of which some subscriber's messages should be pruned + """ + # TODO(snaji): pending_db? + msg_of_sub = set() + with queue.env.begin(write=True) as txn: + sub_db = queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + for key in cursor.iternext(keys=True, values=False): + msg_of_sub.add(key) + + txn.drop(db=sub_db, delete=False) + + pending_db = queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + for key in msg_of_sub: + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index 888fe4f..5432a8d 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -2,28 +2,78 @@ """Distribute messages persistent from publisher to subscriber.""" import pathlib -from typing import Union +from typing import List, Union + +import persipubsub.queue class Pub: - """Create Publisher ready to send messages.""" + """ + Create Publisher ready to send messages. + + :ivar pub_id: publisher ID + :vartype pub_id: str + :ivar queue: on which messages are published + :vartype queue: persipubsub.queue.Queue + :ivar sub_list: list of subscriber to which messages are send + :vartype sub_list: List[str] + :ivar autosync: sync after each message or after multiple messages + :vartype autosync: bool + """ + + pub_id = None # type: str + queue = None # type: persipubsub.queue.Queue + sub_list = None # type: List[str] + autosync = None # type: bool - def __init__(self, - pub_dir: Union[str, pathlib.Path], - autosync: bool = False): + def __init__(self) -> None: + """Initialize class object.""" + + def init(self, + pub_id: str, + config_pth: Union[pathlib.Path, str], + autosync: bool = False) -> None: """ Initialize. - :param pub_dir: Directory where publisher data is stored + :param pub_id: constant ID of publisher + :param config_pth: path to the JSON config file :param autosync: if True, store data automatically in lmdb """ + self.pub_id = pub_id + + config = persipubsub.get_config(path=config_pth) + publisher = config[pub_id] + queue_dir = publisher["out_queue"] + self.queue = persipubsub.queue.Queue() + self.queue.init(config_pth=config_pth, queue_dir=queue_dir) + assert isinstance(publisher["subscribers"], List) + self.sub_list = publisher["subscribers"] # type: List[str] + self.autosync = autosync - def send(self, msg: bytes): + def __enter__(self) -> 'Pub': + """Enter the context and give the pub prepared in the constructor.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """Leave the context.""" + + def send(self, msg: bytes) -> None: """ - Send messages to subscriber. + Send message to subscribers. :param msg: to send to all subscribers """ + self.queue.put(msg=msg, sub_list=self.sub_list) - def sync(self): - """Store message on disk in lmdb.""" + def send_many(self, msgs: List[bytes]) -> None: + """ + Send messages to subscribers. + + :param msgs: to send to all subscribers + """ + if self.autosync: + for msg in msgs: + self.queue.put(msg=msg, sub_list=self.sub_list) + else: + self.queue.put_many_flush_once(msgs=msgs, sub_list=self.sub_list) diff --git a/persipubsub/queue.py b/persipubsub/queue.py new file mode 100644 index 0000000..4bb2dcb --- /dev/null +++ b/persipubsub/queue.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python +"""Store messages in a local LMDB queue.""" +import datetime +import enum +import pathlib +import uuid +from typing import ( # pylint: disable=unused-import + Any, Dict, List, Optional, Union) + +import icontract +import lmdb + +import persipubsub + +MSG_TIMEOUT_SECS = 500 # type: int +MAX_MSGS_NUM = 1024 * 64 +HWM_LMDB_SIZE_BYTES = 30 * 1024**3 # type: int + + +class Strategy(enum.Enum): + """Store possible strategies.""" + + prune_first = 0 + prune_last = 1 + + +def parse_strategy(strategy: str) -> Strategy: + """ + Parse overflow strategy. + + :param strategy: Strategy stored in config + :return: set overflow strategy + """ + if Strategy.prune_first.name == strategy: + return Strategy.prune_first + + return Strategy.prune_last + + +class HighWaterMark: + """Store high water mark limits.""" + + def __init__(self, + msg_timeout_secs: Optional[int] = None, + max_msgs_num: Optional[int] = None, + hwm_lmdb_size_bytes: Optional[int] = None) -> None: + """ + Initialize. + + :param msg_timeout_secs: time after which msg is classified as dangling + msg (secs) + :param max_msgs_num: maximal amount of msg + :param hwm_lmdb_size_bytes: high water mark for total size of lmdb + (bytes) + """ + self.msg_timeout_secs = MSG_TIMEOUT_SECS \ + if msg_timeout_secs is None else msg_timeout_secs # type: int + self.max_msgs_num = MAX_MSGS_NUM \ + if max_msgs_num is None else max_msgs_num # type: int + self.hwm_lmdb_size_bytes = HWM_LMDB_SIZE_BYTES \ + if hwm_lmdb_size_bytes is None else hwm_lmdb_size_bytes # type: int + + +def initialize_environment( + queue_dir: pathlib.Path, + max_reader_num: int = 1024, + max_db_num: int = 1024, + max_db_size_bytes: int = 32 * 1024**3) -> lmdb.Environment: + """ + Initialize the queue; the queue directory is assumed to exist. + + :param queue_dir: where the queue is stored + :param max_reader_num: maximal number of reader + :param max_db_num: maximal number of databases + :param max_db_size_bytes: maximal size of database (bytes) + :return: Load or if needed create LMDB queue from directory + """ + if not queue_dir.exists(): + raise RuntimeError( + "The queue directory does not exist: {}".format(queue_dir)) + + env = lmdb.open( + path=queue_dir.as_posix(), + map_size=max_db_size_bytes, + subdir=True, + max_readers=max_reader_num, + max_dbs=max_db_num) + + return env + + +def prune_dangling_messages_for(queue: 'Queue', sub_list: List[str]) -> None: + """ + Prune all dangling messages for subscribers of a queue from lmdb. + + :param queue: of which dangling messages should be pruned + :param sub_list: subscribers of which dangling messages should be pruned + """ + with queue.env.begin(write=True) as txn: + pending_db = queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + meta_db = queue.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + data_db = queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + # Definition of dangling messages: + # - having no pending subscribers + # - exists longer than timeout allows + msgs_to_delete = set() + with queue.env.begin(db=pending_db) as txn: + cursor = txn.cursor() + cursor.first() + for key, pending_subscribers_num in cursor: + if persipubsub.bytes_to_int(pending_subscribers_num) == 0: + msgs_to_delete.add(key) + + # subscriber might still await this messages after the timeout. + # This messages needs also to be removed from all subscribers. + msgs_to_delete_timeout = set() + with queue.env.begin(db=meta_db) as txn: + cursor = txn.cursor() + + timestamp_now = datetime.datetime.utcnow().timestamp() + for key, timestamp in cursor: + # TODO(snaji): check type of metadata + if int(timestamp_now) - persipubsub.bytes_to_int(timestamp) \ + > queue.hwm.msg_timeout_secs: + msgs_to_delete_timeout.add(key) + + msgs_to_delete = msgs_to_delete.union(msgs_to_delete_timeout) + with queue.env.begin(write=True) as txn: + for key in msgs_to_delete: + txn.delete(key=key, db=pending_db) + txn.delete(key=key, db=meta_db) + txn.delete(key=key, db=data_db) + + with queue.env.begin(write=True) as txn: + for sub_id in sub_list: + sub_db = queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + for key in msgs_to_delete_timeout: + txn.delete(key=key, db=sub_db) + + +class Queue: + """ + Queue messages persistently from many publishers for many subscribers. + + :ivar config_pth: path to the JSON config file + :vartype config_pth: pathlib.Path + :ivar env: LMDB environment on disk + :vartype env: lmdb.Environment + :ivar hwm: High water mark limit for queue + :vartype hwm: HighWaterMark + :ivar strategy: + Strategy which will be used to remove messages when high water mark is + reached. + :vartype strategy: Strategy + :ivar sub_list: all subscribers of the queue + :vartype sub_list: List[str] + """ + + config_pth = None # type: pathlib.Path + env = None # type: lmdb.Environment + hwm = None # type: HighWaterMark + strategy = None # type: Strategy + sub_list = None # type: List[str] + + def __init__(self) -> None: + """Initialize class object.""" + + # pylint: disable=too-many-arguments + @icontract.require(lambda max_reader_num: max_reader_num >= 0) + # There have to be at least 4 databases. + # The main db, the data db, the pending db and the meta db. + @icontract.require(lambda max_db_num: max_db_num >= 4) + @icontract.require(lambda max_db_size_bytes: max_db_size_bytes > 0) + def init(self, + config_pth: Union[pathlib.Path, str], + queue_dir: Union[pathlib.Path, str], + max_reader_num: int = 1024, + max_db_num: int = 1024, + max_db_size_bytes: int = 32 * 1024**3) -> None: + """ + Initialize the queue. + + :param config_pth: path to the JSON config file + :param queue_dir: where the queue is stored + :param max_reader_num: maximal number of reader + :param max_db_num: maximal number of databases + :param max_db_size_bytes: maximal size of database (bytes) + """ + self.config_pth = config_pth if isinstance(config_pth, pathlib.Path) \ + else pathlib.Path(config_pth) + + queue_dir = queue_dir if isinstance(queue_dir, pathlib.Path) \ + else pathlib.Path(queue_dir) + + self.env = initialize_environment( + queue_dir=queue_dir, + max_reader_num=max_reader_num, + max_db_num=max_db_num, + max_db_size_bytes=max_db_size_bytes) + + with self.env.begin(write=True) as txn: + _ = self.env.open_db(key=persipubsub.DATA_DB, txn=txn, create=True) + _ = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=True) + _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) + + config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] + queues = config["queues"] # type: Dict[str, Any] + queue = queues[queue_dir.as_posix()] # type: Dict[str, Any] + hwm = queue["high-water-mark"] + + assert isinstance(hwm['MSG_TIMEOUT_SECS'], int) + assert isinstance(hwm['MAX_MSGS_NUM'], int) + assert isinstance(hwm['HWM_LMDB_SIZE_BYTES'], int) + self.hwm = HighWaterMark( + msg_timeout_secs=hwm['MSG_TIMEOUT_SECS'], + max_msgs_num=hwm['MAX_MSGS_NUM'], + hwm_lmdb_size_bytes=hwm['HWM_LMDB_SIZE_BYTES']) + + assert isinstance(hwm["strategy"], str) + self.strategy = parse_strategy(hwm["strategy"]) + + self.sub_list = queue["subscribers"] + + def __enter__(self) -> 'Queue': + """Enter the context and give the queue prepared in the constructor.""" + # TODO(snaji): queue.init(...)? + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """Leave the context.""" + + def put(self, msg: bytes, sub_list: List[str]) -> None: + """ + Put message to lmdb queue. + + :param msg: proto message send from publisher to subscribers + :param sub_list: List of subscribers encoded in 'utf-8' + :return: + """ + # every publisher always prunes queue before sending a message. + self.vacuum() + msg_id = str(datetime.datetime.utcnow().timestamp()) + str(uuid.uuid4()) + with self.env.begin(write=True) as txn: + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + txn.put( + key=persipubsub.encoding(msg_id), + value=persipubsub.int_to_bytes(len(sub_list)), + db=pending_db) + + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + txn.put( + key=persipubsub.encoding(msg_id), + value=persipubsub.int_to_bytes( + int(datetime.datetime.utcnow().timestamp())), + db=meta_db) + + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) + + # TODO(snaji): can subscriber send messages to subset of queue + # sub_list? + for sub in sub_list: + sub_db = self.env.open_db( + key=persipubsub.encoding(sub), txn=txn, create=False) + txn.put(key=persipubsub.encoding(msg_id), db=sub_db) + + def put_many_flush_once(self, msgs: List[bytes], + sub_list: List[str]) -> None: + """ + Put many message to lmdb queue. + + :param msgs: proto messages send from publisher to subscribers + :param sub_list: List of subscribers encoded in 'utf-8' + :return: + """ + # every publisher always prunes queue before sending a message. + self.vacuum() + + with self.env.begin(write=True) as txn: + + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + sub_dbs = set() + for sub in sub_list: + sub_dbs.add( + self.env.open_db( + key=persipubsub.encoding(sub), txn=txn, create=False)) + + for msg in msgs: + msg_id = str(datetime.datetime.utcnow().timestamp()) + str( + uuid.uuid4()) + + txn.put( + key=persipubsub.encoding(msg_id), + value=persipubsub.int_to_bytes(len(sub_list)), + db=pending_db) + + txn.put( + key=persipubsub.encoding(msg_id), + value=persipubsub.int_to_bytes( + int(datetime.datetime.utcnow().timestamp())), + db=meta_db) + + txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) + + for sub_db in sub_dbs: + txn.put(key=persipubsub.encoding(msg_id), db=sub_db) + + def front(self, sub_id: str) \ + -> Optional[bytes]: + """ + Peek at next message in lmdb queue. + + Load from LMDB queue into memory and process msg afterwards. + + :param sub_id: Subscriber ID + :return: + """ + with self.env.begin(write=False) as txn: + sub_db = self.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + key = cursor.key() + + return txn.get(key=key, db=data_db) + + return None + + def pop(self, sub_id: str) -> None: + """ + Remove msg from the subscriber's queue and reduce pending subscribers. + + :param sub_id: Subscriber ID + :return: + """ + with self.env.begin(write=True) as txn: + sub_db = self.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + key = cursor.key() + cursor.pop(key=key) + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + else: + raise RuntimeError("No message to pop") + + def prune_dangling_messages(self) -> None: + """ + Prune dangling messages in the queue. + + :return: + """ + prune_dangling_messages_for(queue=self, sub_list=self.sub_list) + + # TODO(snaji): side todo add precise check size + def check_current_lmdb_size(self) -> int: + """ + Check current lmdb size in bytes. + + Check size of data database by approximating size with multiplying page + size with number of pages. + :return: data database size in bytes + """ + lmdb_size_bytes = 0 + + with self.env.begin(write=False) as txn: + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + data_stat = txn.stat(db=data_db) + lmdb_size_bytes += data_stat['psize'] * ( + data_stat['branch_pages'] + data_stat['leaf_pages'] + + data_stat['overflow_pages']) + + return lmdb_size_bytes + + def count_msgs(self) -> int: + """ + Count number of messages in database. + + Count number of messages stored in meta database. + :return: number of messages in database + """ + with self.env.begin(write=False) as txn: + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + meta_stat = txn.stat(db=meta_db) + + return meta_stat['entries'] + + def vacuum(self) -> None: + """ + Clean database when needed. + + :return: + """ + self.prune_dangling_messages() + msgs_num = self.count_msgs() + if msgs_num >= self.hwm.max_msgs_num: + self.prune_messages() + lmdb_size_bytes = self.check_current_lmdb_size() + if lmdb_size_bytes >= self.hwm.hwm_lmdb_size_bytes: + self.prune_messages() + + def prune_messages(self) -> None: + """ + Prune one half of the messages stored. + + Depending on the strategy the first or the last will be deleted. + :return: + """ + # pylint: disable=too-many-locals + messages_to_delete = set() + + with self.env.begin(write=False) as txn: + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + meta_stat = txn.stat(db=meta_db) + entries = meta_stat['entries'] + + cursor = txn.cursor(db=meta_db) + if self.strategy == Strategy.prune_first: + + cursor.first() + for index, key in enumerate( + cursor.iternext(keys=True, values=False)): + messages_to_delete.add(key) + if index >= int(entries / 2): + break + + elif self.strategy == Strategy.prune_last: + cursor.last() + for index, key in enumerate( + cursor.iterprev(keys=True, values=False)): + messages_to_delete.add(key) + if index >= int(entries / 2): + break + else: + raise RuntimeError("Pruning strategy not set.") + + with self.env.begin(write=True) as txn: + pending_db = self.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + meta_db = self.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + data_db = self.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + + dbs = [pending_db, meta_db, data_db] + + for sub in self.sub_list: + sub_db = self.env.open_db( + key=persipubsub.encoding(sub), txn=txn, create=False) + dbs.append(sub_db) + + for key in messages_to_delete: + for db in dbs: # pylint: disable=invalid-name + txn.delete(key=key, db=db) diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 4630f89..fc578ec 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -2,38 +2,100 @@ """Receive messages persistent from publisher to subscriber.""" import contextlib +import datetime import pathlib -from typing import Union, Optional import time +from typing import Generator, Union + +import icontract + +import persipubsub.queue class Sub: - """Create Subscriber ready to receive messages.""" + """ + Create Subscriber ready to receive messages. + + :ivar sub_id: subscriber ID + :vartype pub_id: str + :ivar queue: from which messages are received + :vartype queue: persipubsub.queue.Queue + """ - def __init__(self, sub_dir: Union[str, pathlib.Path], sub_id: str): + sub_id = None # type: str + queue = None # type: persipubsub.queue.Queue + + def __init__(self) -> None: + """Initialize class object.""" + + def init(self, sub_id: str, config_pth: Union[pathlib.Path, str]) -> None: """ Initialize. - :param sub_dir: Directory where subscriber data is stored :param sub_id: unique subscriber id + :param config_pth: path to the JSON config file """ + self.sub_id = sub_id + + config = persipubsub.get_config(path=config_pth) + subscriber = config[sub_id] + queue_dir = subscriber["in_queue"] + self.queue = persipubsub.queue.Queue() + self.queue.init(config_pth=config_pth, queue_dir=queue_dir) + def __enter__(self) -> 'Sub': + """Enter the context and give the sub prepared in the constructor.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """Leave the context.""" + + # TODO(snaji): return type Optional[str]:? + # TODO(snaji): rewrite with enter and exit? + # TODO(snaji): receive or front? + + @icontract.require(lambda timeout: timeout > 0) + @icontract.require(lambda retries: retries > 0) @contextlib.contextmanager - def receive(self, timeout: int = 60) -> Optional[str]: + def receive(self, timeout: int = 60, retries: int = 10) -> Generator: """ Receive messages from the publisher. :param timeout: time waiting for a message. If none arrived until the - timeout then None will be returned. + timeout then None will be returned. (secs) + :param retries: number of tries to check if a msg arrived in the queue """ - # receive something + msg = None + end = int(datetime.datetime.utcnow().timestamp()) + timeout + try: + while int(datetime.datetime.utcnow().timestamp()) <= end: + msg = self.queue.front(sub_id=self.sub_id) + if msg is not None: + break + time.sleep(timeout / retries) + yield msg + finally: + pass - def pop(self): + if msg is not None: + self.pop() + + def pop(self) -> None: """Pop a message from the subscriber's lmdb.""" + self.queue.pop(sub_id=self.sub_id) - def pop_to_top(self): + def pop_to_top(self) -> None: """ Pops all messages until the most recent one. Used for slow processes to ensure realtime processing. """ + with self.queue.env.begin(write=False) as txn: + sub_db = self.queue.env.open_db( + key=persipubsub.encoding(self.sub_id), txn=txn, create=False) + sub_stat = txn.stat(db=sub_db) + # pop all message except the most recent one + msg_to_pop_num = sub_stat['entries'] - 1 + + for _ in range(msg_to_pop_num): + self.queue.pop(sub_id=self.sub_id) diff --git a/precommit.py b/precommit.py index 55b6de6..28f6e16 100644 --- a/precommit.py +++ b/precommit.py @@ -99,6 +99,12 @@ def main() -> int: for pth in (repo_root / "persipubsub").glob("**/*.py"): subprocess.check_call(["pyicontract-lint", pth.as_posix()]) + print("Twine'ing...") + subprocess.check_call(["python3", "setup.py", "sdist", "bdist_wheel"], + cwd=repo_root.as_posix()) + subprocess.check_call(["twine", "check", "dist/*"], + cwd=repo_root.as_posix()) + return 0 diff --git a/pylint.rc b/pylint.rc index bca2e5d..b2c1595 100644 --- a/pylint.rc +++ b/pylint.rc @@ -7,4 +7,4 @@ generated-members=bottle\.request\.forms\.decode,bottle\.request\.query\.decode max-line-length=80 [MESSAGES CONTROL] -disable=too-few-public-methods,abstract-class-little-used,len-as-condition,bad-continuation,bad-whitespace \ No newline at end of file +disable=too-few-public-methods,abstract-class-little-used,len-as-condition,bad-continuation,bad-whitespace, no-member, duplicate-code, fixme \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c42f07d..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -typing-extensions>=3.6.6 -icontract>=2.0.1,<3 -asttokens>=1.1.13,<2 \ No newline at end of file diff --git a/setup.py b/setup.py index eca6f92..15fdcec 100644 --- a/setup.py +++ b/setup.py @@ -15,11 +15,6 @@ with open(os.path.join(here, 'README.rst'), encoding='utf-8') as fid: long_description = fid.read().strip() # pylint: disable=invalid-name -with open(os.path.join(here, 'requirements.txt'), encoding='utf-8') as fid: - install_requires = [ - line for line in fid.read().splitlines() if line.strip() - ] - setup( name=persipubsub_meta.__title__, version=persipubsub_meta.__version__, @@ -39,7 +34,13 @@ keywords= 'persistent publisher subscriber lmdb MQ message queue thread-safe process-safe', packages=find_packages(exclude=['tests']), - install_requires=install_requires, + install_requires=[ + # yapf: disable + 'typing-extensions>=3.7.2', + 'icontract>=2.0.1,<3', + 'lmdb>=0.94,<1' + # yapf: enable + ], extras_require={ 'dev': [ # yapf: disable @@ -53,11 +54,13 @@ 'docutils>=0.14,<1', 'isort>=4.3.4,<5', 'pygments>=2.3.0,<3', - 'logthis>=1.0.1,<2', + 'twine>=1.12.1,<2', + 'setuptools>=40.6.3,<41', + 'wheel>=0.32.3,<1' # yapf: enable ] }, py_modules=['persipubsub', 'persipubsub_meta'], include_package_data=True, package_data={"persipubsub": ["py.typed"]}, - data_files=[('.', ['LICENSE', 'README.rst', 'requirements.txt'])]) + data_files=[('.', ['LICENSE', 'README.rst'])]) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..b8e3cf8 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Test persipubsub.""" + +import json +import pathlib +import unittest +from typing import Any, Dict + +import temppathlib + +import persipubsub + +# pylint: disable=missing-docstring + +# TODO(snaji): test init.py + +LMDB_PAGE_SIZE = 4096 + +TEST_MSG_TIMEOUT = 1 # type: int +TEST_HWM_MSG_NUM = 10 # type: int +TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int + +# define all encodings here +ENCODING = 'utf-8' +BYTES_LENGTH = 8 +BYTES_ORDER = 'big' + +# define all database names here +DATA_DB = "data_db".encode(ENCODING) # msg_id | data +PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber +META_DB = "meta_db".encode(ENCODING) # msg_id | metadata + + +def generate_test_config(path: pathlib.Path) -> Dict[str, Any]: + queue_dir = path / "queue" + queue_dir.mkdir(parents=True, exist_ok=True) + + test_config = { + "pub": { + "out_queue": queue_dir.as_posix(), + "subscribers": ["sub"] + }, + "sub": { + "in_queue": queue_dir.as_posix() + }, + "queues": { + queue_dir.as_posix(): { + "path": queue_dir.as_posix(), + "max_reader_num": 1024, + "max_db_num": 1024, + "max_db_size_bytes": 34359738368, + "subscribers": ["sub"], + "high-water-mark": { + "MSG_TIMEOUT_SECS": TEST_MSG_TIMEOUT, + "MAX_MSGS_NUM": TEST_HWM_MSG_NUM, + "HWM_LMDB_SIZE_BYTES": TEST_HWM_LMDB_SIZE, + "strategy": "prune_first" + } + }, + } + } + + return test_config + + +class TestPersiPubSub(unittest.TestCase): + def test_get_config(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + expected_config = generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(expected_config, file_object) + + config = persipubsub.get_config(path=file) + + self.assertDictEqual(expected_config, config) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_control.py b/tests/test_control.py new file mode 100644 index 0000000..0f764c4 --- /dev/null +++ b/tests/test_control.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python +"""Test control unit.""" + +import datetime +import json +import unittest +from typing import List + +import temppathlib + +import persipubsub.control +import persipubsub.queue +import tests + +# pylint: disable=missing-docstring + + +class TestControl(unittest.TestCase): + def test_initialize_all(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + expected_db_keys = [b'data_db', b'meta_db', b'pending_db', b'sub'] + db_keys = [] # type: List[bytes] + + with queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_add_sub(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + persipubsub.control.add_sub(sub_id="sub1", queue=queue) + persipubsub.control.add_sub(sub_id="sub2", queue=queue) + + expected_db_keys = [ + b'data_db', b'meta_db', b'pending_db', b'sub1', b'sub2' + ] + db_keys = [] # type: List[bytes] + + with queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_del_sub(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + persipubsub.control.add_sub(sub_id="sub1", queue=queue) + persipubsub.control.add_sub(sub_id="sub2", queue=queue) + persipubsub.control.remove_sub(sub_id="sub2", queue=queue) + + expected_db_keys = [b'data_db', b'meta_db', b'pending_db', b'sub1'] + db_keys = [] # type: List[bytes] + + with queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_clear_all_subs(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + persipubsub.control.initialize_all_dbs(config_pth=file) + persipubsub.control.clear_all_subs(config_pth=file) + + expected_db_keys = [b'data_db', b'meta_db', b'pending_db'] + db_keys = [] # type: List[bytes] + + with queue.env.begin() as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + db_keys.append(key) + + self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) + + def test_prune_dangling_messages(self): + # TODO(snaji): check + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT + + with queue.env.begin(write=True) as txn: + sub_db = queue.env.open_db( + key='sub'.encode(tests.ENCODING), txn=txn, create=False) + + txn.put(key="timeout_msg".encode(tests.ENCODING), db=sub_db) + txn.put(key="valid_msg".encode(tests.ENCODING), db=sub_db) + with queue.env.begin(write=True) as txn: + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + txn.put( + key="popped_msg".encode(tests.ENCODING), + value="I'm data".encode(tests.ENCODING), + db=data_db) + txn.put( + key="timeout_msg".encode(tests.ENCODING), + value="I'm data too".encode(tests.ENCODING), + db=data_db) + txn.put( + key="valid_msg".encode(tests.ENCODING), + value="Free me!".encode(tests.ENCODING), + db=data_db) + with queue.env.begin(write=True) as txn: + pending_db = queue.env.open_db( + key=tests.PENDING_DB, txn=txn, create=False) + + txn.put( + key="popped_msg".encode(tests.ENCODING), + value=int(0).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=pending_db) + txn.put( + key="timeout_msg".encode(tests.ENCODING), + value=int(1).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=pending_db) + txn.put( + key="valid_msg".encode(tests.ENCODING), + value=int(1).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=pending_db) + + with queue.env.begin(write=True) as txn: + meta_db = queue.env.open_db( + key=tests.META_DB, txn=txn, create=False) + + txn.put( + key="popped_msg".encode(tests.ENCODING), + value=int(datetime.datetime.utcnow().timestamp()).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=meta_db) + txn.put( + key="timeout_msg".encode(tests.ENCODING), + value=int(datetime.datetime.utcnow().timestamp() - + tests.TEST_MSG_TIMEOUT - 5).to_bytes( + length=tests.BYTES_LENGTH, + byteorder=tests.BYTES_ORDER), + db=meta_db) + txn.put( + key="valid_msg".encode(tests.ENCODING), + value=int(datetime.datetime.utcnow().timestamp()).to_bytes( + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + db=meta_db) + + persipubsub.control.prune_dangling_messages(config_pth=file) + + dbs = [sub_db, data_db, pending_db, meta_db] + + expected_remaining_entries = 4 + remaining_entries = 0 + + # pylint: disable=invalid-name + for db in dbs: + with queue.env.begin(db=db) as txn: + cursor = txn.cursor() + cursor.first() + for key, value in cursor: # pylint: disable=unused-variable + remaining_entries += 1 + self.assertEqual(b'valid_msg', key) + + self.assertEqual(expected_remaining_entries, remaining_entries) + + def test_prune_all_messages_for_subscriber(self): + # TODO(snaji): discuss with Adam and Marko + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + persipubsub.control.initialize_all_dbs(config_pth=file) + + msg = persipubsub.encoding("hello world!") + + queue.put(msg=msg, sub_list=["sub"]) + queue.put(msg=msg, sub_list=["sub"]) + + with queue.env.begin(write=False) as txn: + sub_db = queue.env.open_db( + key=persipubsub.encoding('sub'), txn=txn, create=False) + sub_stat = txn.stat(db=sub_db) + self.assertEqual(2, sub_stat['entries']) + + persipubsub.control.prune_all_messages_for( + sub_id="sub", queue=queue) + + with queue.env.begin(write=False) as txn: + sub_db = queue.env.open_db( + key=persipubsub.encoding('sub'), txn=txn, create=False) + sub_stat = txn.stat(db=sub_db) + self.assertEqual(0, sub_stat['entries']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_publisher.py b/tests/test_publisher.py new file mode 100644 index 0000000..12994b6 --- /dev/null +++ b/tests/test_publisher.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +"""Test publisher.""" + +import json +import unittest + +import lmdb +import temppathlib + +import messages.message_pb2 +import persipubsub.control +import persipubsub.publisher +import persipubsub.queue +import tests + +# pylint: disable=missing-docstring + + +class TestPublisher(unittest.TestCase): + def test_send(self): + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + pub = persipubsub.publisher.Pub() + pub.init(pub_id="pub", config_pth=file) + + msg = messages.message_pb2.SomeMessage() + msg.id = 1 + msg.text = "Hello World!" + pub.send(msg=msg.SerializeToString()) + + subscriber = "sub".encode(tests.ENCODING) + with queue.env.begin(write=False) as txn: + self.assertIsNotNone(txn.get(key=subscriber)) + sub_db = queue.env.open_db( + key=subscriber, txn=txn, create=False) + cursor = txn.cursor(db=sub_db) + self.assertTrue(cursor.first()) + + key = cursor.key() + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + item = txn.get(key=key, db=data_db) + self.assertIsNotNone(item) + self.assertEqual(msg.SerializeToString(), item) + + def test_send_to_nonexisting_db(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + pub = persipubsub.publisher.Pub() + pub.init(pub_id="pub", config_pth=file) + + pub.sub_list.append('non-existent sub') + + msg = "message for every subscriber".encode(tests.ENCODING) + self.assertRaises(lmdb.NotFoundError, pub.send, msg=msg) + + def test_send_many(self): + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + pub = persipubsub.publisher.Pub() + pub.init(pub_id='pub', config_pth=file) + + # TODO(snaji): check everywhere encoding + msg = "I'm a message".encode(tests.ENCODING) + msgs = [] + msg_num = 10 + for _ in range(msg_num): + msgs.append(msg) + + subscriber = "sub" + + pub.send_many(msgs=msgs) + + with pub.queue.env.begin(write=False) as txn: + self.assertIsNotNone( + txn.get(key=subscriber.encode(tests.ENCODING))) + + sub_db = pub.queue.env.open_db( + key=subscriber.encode(tests.ENCODING), + txn=txn, + create=False) + + sub_stat = txn.stat(db=sub_db) + self.assertEqual(msg_num, sub_stat['entries']) + + data_db = pub.queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + data_stat = txn.stat(db=data_db) + self.assertEqual(msg_num, data_stat['entries']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_queue.py b/tests/test_queue.py new file mode 100644 index 0000000..c448b10 --- /dev/null +++ b/tests/test_queue.py @@ -0,0 +1,410 @@ +#!/usr/bin/env python3 +"""Test database.""" + +import json +import time +import unittest + +import temppathlib + +import messages.message_pb2 +import persipubsub.control +import persipubsub.queue +import tests + +# pylint: disable=missing-docstring + + +class TestQueue(unittest.TestCase): + def test_initialize_environment(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.queue.initialize_environment( + queue_dir=tmp_dir.path) + + self.assertDictEqual({ + 'branch_pages': 0, + 'entries': 0, + 'depth': 0, + 'leaf_pages': 0, + 'overflow_pages': 0, + 'psize': 4096 + }, env.stat()) + self.assertDictEqual({ + 'last_txnid': 0, + 'last_pgno': 1, + 'map_size': 32 * 1024**3, + 'map_addr': 0, + 'max_readers': 1024, + 'num_readers': 1 + }, env.info()) + + def test_put_to_single_subscriber(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = messages.message_pb2.SomeMessage() + msg.id = 1 + msg.text = "I'm a protobuf message." + + subscriber = "sub" + + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.put(msg=msg.SerializeToString(), sub_list=[subscriber]) + + with queue.env.begin() as txn: + self.assertIsNotNone( + txn.get(key=subscriber.encode(tests.ENCODING))) + + sub_db = queue.env.open_db( + key=subscriber.encode(tests.ENCODING), + txn=txn, + create=False) + cursor = txn.cursor(db=sub_db) + self.assertTrue(cursor.first()) + key = cursor.key() + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + value = txn.get(key=key, db=data_db) + self.assertIsNotNone(value) + self.assertEqual(msg.SerializeToString(), value) + + def test_put_multiple_subscriber(self): + # TODO(snaji): + pass + + def test_put_many(self): + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + # TODO(snaji): check everywhere encoding + msg = "I'm a message".encode(tests.ENCODING) + msgs = [] + msg_num = 10 + for _ in range(msg_num): + msgs.append(msg) + + subscriber = "sub" + + queue.put_many_flush_once(msgs=msgs, sub_list=[subscriber]) + + with queue.env.begin(write=False) as txn: + self.assertIsNotNone( + txn.get(key=subscriber.encode(tests.ENCODING))) + + sub_db = queue.env.open_db( + key=subscriber.encode(tests.ENCODING), + txn=txn, + create=False) + + sub_stat = txn.stat(db=sub_db) + self.assertEqual(msg_num, sub_stat['entries']) + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + data_stat = txn.stat(db=data_db) + self.assertEqual(msg_num, data_stat['entries']) + + def test_front(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = messages.message_pb2.SomeMessage() + msg.id = 1 + msg.text = "I'm a protobuf message." + + subscriber = "sub" + + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + # TODO(snaji): replace with lmdb.env statements? + queue.put(msg=msg.SerializeToString(), sub_list=[subscriber]) + + # pylint: disable=assignment-from-none + # pylint: disable=assignment-from-no-return + received_msg = queue.front(sub_id=subscriber) + self.assertIsNotNone(received_msg) + self.assertEqual(msg.SerializeToString(), received_msg) + + def test_pop(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = messages.message_pb2.SomeMessage() + msg.id = 1 + msg.text = "I'm a protobuf message." + + subscriber = "sub" + + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + # TODO(snaji): replace with lmdb.env statements? + queue.put(msg=msg.SerializeToString(), sub_list=[subscriber]) + + # pylint: disable=assignment-from-none + # pylint: disable=assignment-from-no-return + received_msg = queue.front(sub_id=subscriber) + self.assertIsNotNone(received_msg) + + with queue.env.begin() as txn: + pending_db = queue.env.open_db( + key=tests.PENDING_DB, txn=txn, create=False) + + cursor = txn.cursor(db=pending_db) + self.assertTrue(cursor.first()) + pending_before_pop = cursor.value() + + queue.pop(sub_id=subscriber) + + received_msg = queue.front(sub_id=subscriber) + self.assertIsNone(received_msg) + + with queue.env.begin() as txn: + pending_db = queue.env.open_db( + key=tests.PENDING_DB, txn=txn, create=False) + cursor = txn.cursor(db=pending_db) + self.assertTrue(cursor.first()) + pending_after_pop = cursor.value() + + self.assertEqual( + int.from_bytes(pending_before_pop, tests.BYTES_ORDER) - 1, + int.from_bytes(pending_after_pop, tests.BYTES_ORDER)) + + def test_pop_queue_empty(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = messages.message_pb2.SomeMessage() + msg.id = 1 + msg.text = "I'm a protobuf message." + + subscriber = "sub" + + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + self.assertRaises(RuntimeError, queue.pop, sub_id=subscriber) + + def test_queue_initialisation(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + queue_config = config['queues'][(tmp_dir.path / "queue").as_posix()] + self.assertIsNotNone(queue.env) + self.assertEqual(queue_config['path'], queue.env.path()) + hwm = queue_config['high-water-mark'] + self.assertEqual(hwm['HWM_LMDB_SIZE_BYTES'], + queue.hwm.hwm_lmdb_size_bytes) + self.assertEqual(hwm['MAX_MSGS_NUM'], queue.hwm.max_msgs_num) + self.assertEqual(hwm['MSG_TIMEOUT_SECS'], + queue.hwm.msg_timeout_secs) + self.assertEqual(hwm['strategy'], queue.strategy.name) + self.assertEqual(['sub'], queue.sub_list) + + def test_overflow_msgs_limit(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + + msg = "hello world".encode(tests.ENCODING) + + self.assertEqual(0, queue.count_msgs()) + for _ in range(tests.TEST_HWM_MSG_NUM): + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.count_msgs()) + + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertEqual( + int(tests.TEST_HWM_MSG_NUM - int(tests.TEST_HWM_MSG_NUM / 2)), + queue.count_msgs()) + + def test_overflow_limit_size(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE + + msg = ("a" * (int(tests.LMDB_PAGE_SIZE / 4))).encode(tests.ENCODING) + + while queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE: + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertTrue( + queue.check_current_lmdb_size() > tests.TEST_HWM_LMDB_SIZE) + + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertTrue( + queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE) + + def test_timeout(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT + + msg = "hello world".encode(tests.ENCODING) + + queue.put(msg=msg, sub_list=queue.sub_list) + self.assertEqual(1, queue.count_msgs()) + queue.put(msg=msg, sub_list=queue.sub_list) + self.assertEqual(2, queue.count_msgs()) + time.sleep(tests.TEST_MSG_TIMEOUT + 1) + queue.put(msg=msg, sub_list=queue.sub_list) + self.assertEqual(1, queue.count_msgs()) + + def test_strategy_prune_first(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.strategy = persipubsub.queue.Strategy.prune_first + + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + + for index in range(tests.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode(tests.ENCODING) + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertEqual("secret message 0".encode(tests.ENCODING), + queue.front(sub_id='sub')) + + msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( + tests.ENCODING) + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertEqual( + "secret message {}".format( + int((tests.TEST_HWM_MSG_NUM / 2) + 1)).encode( + tests.ENCODING), queue.front(sub_id='sub')) + + def test_strategy_prune_last(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + + queue = persipubsub.queue.Queue() + + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.strategy = persipubsub.queue.Strategy.prune_last + + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + + for index in range(tests.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode(tests.ENCODING) + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertEqual("secret message 0".encode(tests.ENCODING), + queue.front(sub_id='sub')) + + msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( + tests.ENCODING) + queue.put(msg=msg, sub_list=queue.sub_list) + + self.assertEqual("secret message 0".encode(tests.ENCODING), + queue.front(sub_id='sub')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py new file mode 100644 index 0000000..b971acf --- /dev/null +++ b/tests/test_subscriber.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +"""Test subscriber.""" + +import json +import unittest + +import temppathlib + +import messages.message_pb2 +import persipubsub.control +import persipubsub.queue +import persipubsub.subscriber +import tests + +# pylint: disable=missing-docstring + + +class TestSubscriber(unittest.TestCase): + # TODO(snaji): receive from cache or fetch from lmdb + def test_receive_message(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + sub = persipubsub.subscriber.Sub() + sub.init(sub_id="sub", config_pth=file) + + msg = messages.message_pb2.SomeMessage() + msg.id = 1 + msg.text = "Hello World!" + queue.put(msg=msg.SerializeToString(), sub_list=["sub"]) + + with sub.receive(timeout=1) as received_msg: + self.assertIsNotNone(received_msg) + self.assertEqual(msg.SerializeToString(), received_msg) + + def test_timeout_subscriber(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + sub = persipubsub.subscriber.Sub() + sub.init(sub_id='sub', config_pth=file) + + with sub.receive(timeout=1) as received_msg: + self.assertIsNone(received_msg) + msg = "message send after timeout and will not be popped" \ + "".encode(tests.ENCODING) + queue.put(msg=msg, sub_list=queue.sub_list) + self.assertIsNone(received_msg) + + self.assertEqual( + "message send after timeout and will not be popped".encode( + tests.ENCODING), queue.front(sub_id='sub')) + + with sub.receive(timeout=1) as received_msg: + self.assertIsNotNone(received_msg) + self.assertEqual( + "message send after timeout and will not be popped".encode( + tests.ENCODING), received_msg) + + self.assertIsNone(queue.front(sub_id='sub')) + + def test_pop(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + sub = persipubsub.subscriber.Sub() + sub.init(sub_id='sub', config_pth=file) + + # TODO(snaji): protobuf or str example + msg1 = "I'm a message".encode(tests.ENCODING) + queue.put(msg=msg1, sub_list=queue.sub_list) + + msg2 = "I'm a message too".encode(tests.ENCODING) + queue.put(msg=msg2, sub_list=queue.sub_list) + + sub.pop() + + with sub.receive() as msg: + self.assertIsNotNone(msg) + self.assertEqual(msg2, msg) + + def test_pop_when_empty(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + sub = persipubsub.subscriber.Sub() + sub.init(sub_id='sub', config_pth=file) + + self.assertRaises(RuntimeError, sub.pop) + + +if __name__ == '__main__': + unittest.main() From 96e249cffb2fd325cf933e31478e7f91e3e785a2 Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 24 Jan 2019 17:59:04 +0100 Subject: [PATCH 06/35] minor fixes for tox --- mypy.ini | 11 +---------- setup.py | 3 ++- tests/test_publisher.py | 9 +++------ tests/test_queue.py | 26 ++++++++------------------ tests/test_subscriber.py | 9 +++------ 5 files changed, 17 insertions(+), 41 deletions(-) diff --git a/mypy.ini b/mypy.ini index 1a2efd2..a7f647e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,10 +1 @@ -[mypy] - -[mypy-messages] -ignore_missing_imports = True - -[mypy-messages.message_pb2] -ignore_missing_imports = True - -[mypy-lmdb] -ignore_missing_imports = True \ No newline at end of file +[mypy] \ No newline at end of file diff --git a/setup.py b/setup.py index 15fdcec..ce5c9e2 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,8 @@ 'pygments>=2.3.0,<3', 'twine>=1.12.1,<2', 'setuptools>=40.6.3,<41', - 'wheel>=0.32.3,<1' + 'wheel>=0.32.3,<1', + 'temppathlib>=1.0.3,<2' # yapf: enable ] }, diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 12994b6..c9af752 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -7,7 +7,6 @@ import lmdb import temppathlib -import messages.message_pb2 import persipubsub.control import persipubsub.publisher import persipubsub.queue @@ -34,10 +33,8 @@ def test_send(self): pub = persipubsub.publisher.Pub() pub.init(pub_id="pub", config_pth=file) - msg = messages.message_pb2.SomeMessage() - msg.id = 1 - msg.text = "Hello World!" - pub.send(msg=msg.SerializeToString()) + msg = "Hello world!".encode(tests.ENCODING) + pub.send(msg=msg) subscriber = "sub".encode(tests.ENCODING) with queue.env.begin(write=False) as txn: @@ -53,7 +50,7 @@ def test_send(self): key=tests.DATA_DB, txn=txn, create=False) item = txn.get(key=key, db=data_db) self.assertIsNotNone(item) - self.assertEqual(msg.SerializeToString(), item) + self.assertEqual(msg, item) def test_send_to_nonexisting_db(self): with temppathlib.TemporaryDirectory() as tmp_dir: diff --git a/tests/test_queue.py b/tests/test_queue.py index c448b10..66a7f59 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -7,7 +7,6 @@ import temppathlib -import messages.message_pb2 import persipubsub.control import persipubsub.queue import tests @@ -40,9 +39,7 @@ def test_initialize_environment(self): def test_put_to_single_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = messages.message_pb2.SomeMessage() - msg.id = 1 - msg.text = "I'm a protobuf message." + msg = "I'm a protobuf message.".encode(tests.ENCODING) subscriber = "sub" @@ -56,7 +53,7 @@ def test_put_to_single_subscriber(self): persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue.Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.put(msg=msg.SerializeToString(), sub_list=[subscriber]) + queue.put(msg=msg, sub_list=[subscriber]) with queue.env.begin() as txn: self.assertIsNotNone( @@ -75,7 +72,7 @@ def test_put_to_single_subscriber(self): value = txn.get(key=key, db=data_db) self.assertIsNotNone(value) - self.assertEqual(msg.SerializeToString(), value) + self.assertEqual(msg, value) def test_put_multiple_subscriber(self): # TODO(snaji): @@ -126,9 +123,7 @@ def test_put_many(self): def test_front(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = messages.message_pb2.SomeMessage() - msg.id = 1 - msg.text = "I'm a protobuf message." + msg = "I'm a protobuf message.".encode(tests.ENCODING) subscriber = "sub" @@ -144,19 +139,17 @@ def test_front(self): queue = persipubsub.queue.Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") # TODO(snaji): replace with lmdb.env statements? - queue.put(msg=msg.SerializeToString(), sub_list=[subscriber]) + queue.put(msg=msg, sub_list=[subscriber]) # pylint: disable=assignment-from-none # pylint: disable=assignment-from-no-return received_msg = queue.front(sub_id=subscriber) self.assertIsNotNone(received_msg) - self.assertEqual(msg.SerializeToString(), received_msg) + self.assertEqual(msg, received_msg) def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = messages.message_pb2.SomeMessage() - msg.id = 1 - msg.text = "I'm a protobuf message." + msg = "I'm a protobuf message.".encode(tests.ENCODING) subscriber = "sub" @@ -172,7 +165,7 @@ def test_pop(self): queue = persipubsub.queue.Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") # TODO(snaji): replace with lmdb.env statements? - queue.put(msg=msg.SerializeToString(), sub_list=[subscriber]) + queue.put(msg=msg, sub_list=[subscriber]) # pylint: disable=assignment-from-none # pylint: disable=assignment-from-no-return @@ -205,9 +198,6 @@ def test_pop(self): def test_pop_queue_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = messages.message_pb2.SomeMessage() - msg.id = 1 - msg.text = "I'm a protobuf message." subscriber = "sub" diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index b971acf..eb844a8 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -6,7 +6,6 @@ import temppathlib -import messages.message_pb2 import persipubsub.control import persipubsub.queue import persipubsub.subscriber @@ -33,14 +32,12 @@ def test_receive_message(self): sub = persipubsub.subscriber.Sub() sub.init(sub_id="sub", config_pth=file) - msg = messages.message_pb2.SomeMessage() - msg.id = 1 - msg.text = "Hello World!" - queue.put(msg=msg.SerializeToString(), sub_list=["sub"]) + msg = "Hello World!".encode(tests.ENCODING) + queue.put(msg=msg, sub_list=["sub"]) with sub.receive(timeout=1) as received_msg: self.assertIsNotNone(received_msg) - self.assertEqual(msg.SerializeToString(), received_msg) + self.assertEqual(msg, received_msg) def test_timeout_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: From e6b18678189425abbee1c8cd7b38df4ddb490e0c Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 24 Jan 2019 18:11:48 +0100 Subject: [PATCH 07/35] modified README --- README.rst | 142 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 136 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 400f6e1..452a5df 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,33 @@ persipubsub =========== +.. image:: https://badges.frapsoft.com/os/mit/mit.png?v=103 + :target: https://opensource.org/licenses/mit-license.php + :alt: MIT License + +.. image:: https://badge.fury.io/py/persipubsub.svg + :target: https://badge.fury.io/py/persipubsub + :alt: PyPI - version + +.. image:: https://img.shields.io/pypi/pyversions/persipubsub.svg + :alt: PyPI - Python Version + +.. image:: https://readthedocs.org/projects/persipubsub/badge/?version=latest + :target: https://persipubsub.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + + *catch phrase in work* +What we wanted. +Motivation +Strengths -Why not ... ------------ +Related projects +================ persist-queue -^^^^^^^^^^^^^ +------------- * `protobuf `_ messages can't be send with `persist-queue `_ except if you deserialize it and serialize it with pickle. * Offers not all functionality expected from a queue. Has put and get function @@ -17,7 +36,7 @@ persist-queue no data is lost when a thread fails. Kafka -^^^^^ +----- * For our needs as local system with IPC it's an unnecessary overhead. * Integration of Kafka written in Scala and Java in our C++/Python/Go codebase not that simple. @@ -25,15 +44,126 @@ Kafka * broker system eventually a bottleneck. RabbitMQ -^^^^^^^^ +-------- * For our needs as local system with IPC it's an unnecessary overhead. * broker system eventually a bottleneck. * `RabbitMQ `_ is less scalable than Kafka, but is supported officially by more languages. zeromq persistence pattern -^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------- * Only persistence pattern of `zeromq `_ is `Titanic `_ which is also a broker system. This takes away the purpose and advantage of zeromq to be a lightweight library which requires no broker. + +Usage +===== + +*Intro text* + +Deployment +---------- + +config.json +^^^^^^^^^^^ + +Control +^^^^^^^ + +Runtime +------- + +Queue +^^^^^ + +Publisher +^^^^^^^^^ + +Subscriber +^^^^^^^^^^ + +Documentation +============= +The documentation is available on `readthedocs +`_. + +Installation +============ + +* Install persipubsub with pip: + +.. code-block:: bash + + pip3 install persipubsub + +Development +=========== + +* Check out the repository. + +* In the repository root, create the virtual environment: + +.. code-block:: bash + + python3 -m venv venv3 + +* Activate the virtual environment: + +.. code-block:: bash + + source venv3/bin/activate + +* Install the development dependencies: + +.. code-block:: bash + + pip3 install -e .[dev] + +We use tox for testing and packaging the distribution. Assuming that the virtual +environment has been activated and the development dependencies have been +installed, run: + +.. code-block:: bash + + tox + +Pre-commit Checks +----------------- + +We provide a set of pre-commit checks that lint and check code for formatting. + +Namely, we use: + +* `yapf `_ to check the formatting. +* The style of the docstrings is checked with `pydocstyle `_. +* Static type analysis is performed with `mypy `_. +* `isort `_ to sort your imports for you. +* Various linter checks are done with `pylint `_. +* Doctests are executed using the Python `doctest module `_. +* `pyicontract-lint `_ lints contracts + in Python code defined with `icontract library `_. +* `twine `_ to check the README for invalid markup + which prevents it from rendering correctly on PyPI. + +Run the pre-commit checks locally from an activated virtual environment with +development dependencies: + +.. code-block:: bash + + ./precommit.py + +* The pre-commit script can also automatically format the code: + +.. code-block:: bash + + ./precommit.py --overwrite + +Versioning +========== +We follow `Semantic Versioning `_. +The version X.Y.Z indicates: + +* X is the major version (backward-incompatible), +* Y is the minor version (backward-compatible), and +* Z is the patch version (backward-compatible bug fix). \ No newline at end of file From 07a174408298cd643b523f502088cbd55266ba99 Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 12:46:35 +0100 Subject: [PATCH 08/35] improved readme --- README.rst | 145 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 134 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 452a5df..d8fb6e8 100644 --- a/README.rst +++ b/README.rst @@ -17,23 +17,36 @@ persipubsub :alt: Documentation Status -*catch phrase in work* +``persipubsub`` implements a persistent, thread-safe and process-safe `lmdb +`_-queue for inter-process communication. -What we wanted. -Motivation -Strengths +Primarily, we used `zeromq `_ for inter-process +communication with a slight improvement through `persizmq +`_. This still didn't fulfill the level +of persistence we wanted. + +Our motivation was to replace our previous library with a one which is +similarly easy to setup and to use. Additionally, it should make it possible to +send `protobuf `_ messages +more precisely bytes persistently, thread-safely and process-safely from +many publishers to many subscribers. + +Besides basic publisher and subscriber classes the library offers control +methods for easy deployment from a config JSON file and maintenance in case +needed. Related projects ================ persist-queue ------------- -* `protobuf `_ messages can't - be send with `persist-queue `_ except if you deserialize it and serialize it with pickle. + * Offers not all functionality expected from a queue. Has put and get function - which are basically only push and pop. So front/peak functionality is missing. + which are basically only push and pop. So ``front`` functionality is missing. So neither can multiple subscriber be implemented nor can't be guaranteed that no data is lost when a thread fails. +* All messages in queues are serialized by ``pickle`` which was for us a reason + against using this library. Kafka ----- @@ -60,29 +73,139 @@ zeromq persistence pattern Usage ===== -*Intro text* +The usage of the library can be differentiated into deployment and actual +runtime of your processes. Deployment ---------- +For deployment only a proper config JSON file is needed to setup the whole +queue structure. + config.json ^^^^^^^^^^^ +Publishers, subscribers and queues need to be defined before runtime in the +config JSON file. + +.. code-block:: json + + { + "pub": { + "out_queue": "/home/user/queues/queue", + "subscribers": ["sub"] + }, + "sub": { + "in_queue": "/home/user/queues/queue" + }, + "queues": { + "/home/user/queues/queue": { + "path": "/home/user/queues/queue", + "max_reader_num": 1024, + "max_db_num": 1024, + "max_db_size_bytes": 34359738368, + "subscribers": ["sub"], + "high-water-mark": { + "MSG_TIMEOUT_SECS": 600, + "MAX_MSGS_NUM": 10000, + "HWM_LMDB_SIZE_BYTES": 1000000, + "strategy": "prune_first" + } + } + } + } Control ^^^^^^^ +Initialize all queues +""""""""""""""""""""" + +.. code-block:: python + + import persipubsub.control + + persipubsub.control.initialize_all_dbs(config_pth="/home/user/config.json") + +Prune all dangling messages +""""""""""""""""""""""""""" + +.. code-block:: python + + import persipubsub.control + + persipubsub.control.prune_dangling_messages( + config_pth="/home/user/config.json") + +Clear all messages +"""""""""""""""""" + +.. code-block:: python + + import persipubsub.control + + persipubsub.control.clear_all_subs(config_pth="/home/user/config.json") + + Runtime ------- - -Queue -^^^^^ +During runtime only publisher and subscriber are needed. Control can be +optionally be used for pruning although the queues prune itself on a regular +basis. Publisher ^^^^^^^^^ +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the publisher can +be initialized as following. + +.. code-block:: python + + import persipubsub.publisher + + pub = persipubsub.publisher.Pub() + pub.init(pub_id="pub", config_pth="/home/user/config.json") + +Send a message +"""""""""""""" + +.. code-block:: python + + msg = "Hello there!".encode('utf-8) + pub.send(msg=msg) + + # subscribers have now a message in the queue + Subscriber ^^^^^^^^^^ +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the publisher can +be initialized as following. + +.. code-block:: python + + import persipubsub.subscriber + + sub = persipubsub.subscriber.Sub() + sub.init(sub_id="sub", config_pth="/home/user/config.json") + +Receive a message +""""""""""""""""" + +.. code-block:: python + + # one message in queue + with sub.receive() as msg: + # do something with the message + print(msg) # b'Hello there!' + + # sub queue is now empty + Documentation ============= The documentation is available on `readthedocs From e449095441cbb8658d5ac457a12365603e4dc4fe Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 14:07:15 +0100 Subject: [PATCH 09/35] reformatting typing imports --- persipubsub/queue.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 4bb2dcb..69dcfff 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -4,8 +4,7 @@ import enum import pathlib import uuid -from typing import ( # pylint: disable=unused-import - Any, Dict, List, Optional, Union) +from typing import Any, Dict, List, Optional, Union import icontract import lmdb From 43b7e8f5b2b77326363b357bb16007dc77fb90ab Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 14:08:18 +0100 Subject: [PATCH 10/35] updated dependencies --- mypy.ini | 5 ++++- requirements-doc.txt | 4 ++-- setup.py | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mypy.ini b/mypy.ini index a7f647e..57dfa98 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1 +1,4 @@ -[mypy] \ No newline at end of file +[mypy] + +[mypy-lmdb] +ignore_missing_imports = True \ No newline at end of file diff --git a/requirements-doc.txt b/requirements-doc.txt index a488d81..50039e6 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,4 +1,4 @@ -sphinx>=1.8.2,<2 -sphinx-autodoc-typehints>=1.5.2 +sphinx>=1.8.3,<2 +sphinx-autodoc-typehints>=1.6.0 sphinx-icontract>=2.0.0 sphinx-rtd-theme>=0.4.2 \ No newline at end of file diff --git a/setup.py b/setup.py index ce5c9e2..a03922d 100644 --- a/setup.py +++ b/setup.py @@ -44,16 +44,16 @@ extras_require={ 'dev': [ # yapf: disable - 'mypy==0.641', + 'mypy==0.660', 'pylint==2.2.2', 'yapf==0.25.0', - 'tox>=3.5.3', + 'tox>=3.7.0', 'coverage>=4.5.2,<5', 'pydocstyle>=3.0.0,<4', 'pyicontract-lint>=2.0.0,<3', 'docutils>=0.14,<1', 'isort>=4.3.4,<5', - 'pygments>=2.3.0,<3', + 'pygments>=2.3.1,<3', 'twine>=1.12.1,<2', 'setuptools>=40.6.3,<41', 'wheel>=0.32.3,<1', From 7f7e957004aa60e02f351e4cf134971e3e86b6b7 Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 14:55:47 +0100 Subject: [PATCH 11/35] removed todos --- persipubsub/control.py | 3 -- persipubsub/queue.py | 5 ---- persipubsub/subscriber.py | 4 --- pylint.rc | 2 +- tests/__init__.py | 2 -- tests/test_control.py | 2 -- tests/test_publisher.py | 1 - tests/test_queue.py | 63 ++++++++++++++++++++++++++++++++++----- tests/test_subscriber.py | 2 -- 9 files changed, 56 insertions(+), 28 deletions(-) diff --git a/persipubsub/control.py b/persipubsub/control.py index f02af75..9fcbcac 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -15,7 +15,6 @@ def initialize_all_dbs(config_pth: Union[pathlib.Path, str]) -> None: """ config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] queues = config["queues"] # type: Dict[str, Any] - # TODO(snaji): maybe multi-threaded especially different queues for values in queues.values(): queue_dir = pathlib.Path(values["path"]) if not queue_dir.exists(): @@ -104,7 +103,6 @@ def remove_sub(sub_id: str, queue: persipubsub.queue.Queue) -> None: with queue.env.begin(write=True) as txn: sub_db = queue.env.open_db( key=persipubsub.encoding(sub_id), txn=txn, create=False) - # TODO(snaji): messages in pending_db? cursor = txn.cursor(db=sub_db) # check if database is not empty if cursor.first(): @@ -157,7 +155,6 @@ def prune_all_messages_for(sub_id: str, queue: persipubsub.queue.Queue) -> None: :param sub_id: ID of the subscriber of which all messages should be pruned :param queue: of which some subscriber's messages should be pruned """ - # TODO(snaji): pending_db? msg_of_sub = set() with queue.env.begin(write=True) as txn: sub_db = queue.env.open_db( diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 69dcfff..cb2db19 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -122,7 +122,6 @@ def prune_dangling_messages_for(queue: 'Queue', sub_list: List[str]) -> None: timestamp_now = datetime.datetime.utcnow().timestamp() for key, timestamp in cursor: - # TODO(snaji): check type of metadata if int(timestamp_now) - persipubsub.bytes_to_int(timestamp) \ > queue.hwm.msg_timeout_secs: msgs_to_delete_timeout.add(key) @@ -228,7 +227,6 @@ def init(self, def __enter__(self) -> 'Queue': """Enter the context and give the queue prepared in the constructor.""" - # TODO(snaji): queue.init(...)? return self def __exit__(self, exc_type, exc_val, exc_tb) -> None: @@ -265,8 +263,6 @@ def put(self, msg: bytes, sub_list: List[str]) -> None: key=persipubsub.DATA_DB, txn=txn, create=False) txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) - # TODO(snaji): can subscriber send messages to subset of queue - # sub_list? for sub in sub_list: sub_db = self.env.open_db( key=persipubsub.encoding(sub), txn=txn, create=False) @@ -383,7 +379,6 @@ def prune_dangling_messages(self) -> None: """ prune_dangling_messages_for(queue=self, sub_list=self.sub_list) - # TODO(snaji): side todo add precise check size def check_current_lmdb_size(self) -> int: """ Check current lmdb size in bytes. diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index fc578ec..2ad5eed 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -50,10 +50,6 @@ def __enter__(self) -> 'Sub': def __exit__(self, exc_type, exc_val, exc_tb) -> None: """Leave the context.""" - # TODO(snaji): return type Optional[str]:? - # TODO(snaji): rewrite with enter and exit? - # TODO(snaji): receive or front? - @icontract.require(lambda timeout: timeout > 0) @icontract.require(lambda retries: retries > 0) @contextlib.contextmanager diff --git a/pylint.rc b/pylint.rc index b2c1595..7b1f5c3 100644 --- a/pylint.rc +++ b/pylint.rc @@ -7,4 +7,4 @@ generated-members=bottle\.request\.forms\.decode,bottle\.request\.query\.decode max-line-length=80 [MESSAGES CONTROL] -disable=too-few-public-methods,abstract-class-little-used,len-as-condition,bad-continuation,bad-whitespace, no-member, duplicate-code, fixme \ No newline at end of file +disable=too-few-public-methods,abstract-class-little-used,len-as-condition,bad-continuation,bad-whitespace, no-member, duplicate-code \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index b8e3cf8..c30181b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -12,8 +12,6 @@ # pylint: disable=missing-docstring -# TODO(snaji): test init.py - LMDB_PAGE_SIZE = 4096 TEST_MSG_TIMEOUT = 1 # type: int diff --git a/tests/test_control.py b/tests/test_control.py index 0f764c4..9529acc 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -123,7 +123,6 @@ def test_clear_all_subs(self): self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) def test_prune_dangling_messages(self): - # TODO(snaji): check # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: config = tests.generate_test_config(path=tmp_dir.path) @@ -222,7 +221,6 @@ def test_prune_dangling_messages(self): self.assertEqual(expected_remaining_entries, remaining_entries) def test_prune_all_messages_for_subscriber(self): - # TODO(snaji): discuss with Adam and Marko with temppathlib.TemporaryDirectory() as tmp_dir: config = tests.generate_test_config(path=tmp_dir.path) diff --git a/tests/test_publisher.py b/tests/test_publisher.py index c9af752..7b6fae7 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -88,7 +88,6 @@ def test_send_many(self): pub = persipubsub.publisher.Pub() pub.init(pub_id='pub', config_pth=file) - # TODO(snaji): check everywhere encoding msg = "I'm a message".encode(tests.ENCODING) msgs = [] msg_num = 10 diff --git a/tests/test_queue.py b/tests/test_queue.py index 66a7f59..3ebf129 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -39,7 +39,7 @@ def test_initialize_environment(self): def test_put_to_single_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a protobuf message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" @@ -75,8 +75,58 @@ def test_put_to_single_subscriber(self): self.assertEqual(msg, value) def test_put_multiple_subscriber(self): - # TODO(snaji): - pass + # pylint: disable=too-many-locals + with temppathlib.TemporaryDirectory() as tmp_dir: + msg = "I'm a message.".encode(tests.ENCODING) + + sub_list = ["sub", "another_sub"] + + config = tests.generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(config, file_object) + + persipubsub.control.initialize_all_dbs(config_pth=file) + queue = persipubsub.queue.Queue() + queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + + persipubsub.control.add_sub('another_sub', queue=queue) + + queue.put(msg=msg, sub_list=sub_list) + + with queue.env.begin() as txn: + self.assertIsNotNone( + txn.get(key=sub_list[0].encode(tests.ENCODING))) + + sub_db_0 = queue.env.open_db( + key=sub_list[0].encode(tests.ENCODING), + txn=txn, + create=False) + cursor = txn.cursor(db=sub_db_0) + self.assertTrue(cursor.first()) + key_0 = cursor.key() + + self.assertIsNotNone( + txn.get(key=sub_list[1].encode(tests.ENCODING))) + + sub_db_1 = queue.env.open_db( + key=sub_list[1].encode(tests.ENCODING), + txn=txn, + create=False) + cursor = txn.cursor(db=sub_db_1) + self.assertTrue(cursor.first()) + key_1 = cursor.key() + + self.assertEqual(key_0, key_1) + + data_db = queue.env.open_db( + key=tests.DATA_DB, txn=txn, create=False) + + value = txn.get(key=key_1, db=data_db) + self.assertIsNotNone(value) + self.assertEqual(msg, value) def test_put_many(self): # pylint: disable=too-many-locals @@ -92,7 +142,6 @@ def test_put_many(self): queue = persipubsub.queue.Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - # TODO(snaji): check everywhere encoding msg = "I'm a message".encode(tests.ENCODING) msgs = [] msg_num = 10 @@ -123,7 +172,7 @@ def test_put_many(self): def test_front(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a protobuf message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" @@ -138,7 +187,6 @@ def test_front(self): queue = persipubsub.queue.Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - # TODO(snaji): replace with lmdb.env statements? queue.put(msg=msg, sub_list=[subscriber]) # pylint: disable=assignment-from-none @@ -149,7 +197,7 @@ def test_front(self): def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a protobuf message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" @@ -164,7 +212,6 @@ def test_pop(self): queue = persipubsub.queue.Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - # TODO(snaji): replace with lmdb.env statements? queue.put(msg=msg, sub_list=[subscriber]) # pylint: disable=assignment-from-none diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index eb844a8..7c93e2f 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -15,7 +15,6 @@ class TestSubscriber(unittest.TestCase): - # TODO(snaji): receive from cache or fetch from lmdb def test_receive_message(self): with temppathlib.TemporaryDirectory() as tmp_dir: config = tests.generate_test_config(path=tmp_dir.path) @@ -90,7 +89,6 @@ def test_pop(self): sub = persipubsub.subscriber.Sub() sub.init(sub_id='sub', config_pth=file) - # TODO(snaji): protobuf or str example msg1 = "I'm a message".encode(tests.ENCODING) queue.put(msg=msg1, sub_list=queue.sub_list) From 7f5a00f69344d7b100f2bf7549798147ead9ee4f Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 15:34:28 +0100 Subject: [PATCH 12/35] privatize some methods --- persipubsub/control.py | 25 ++++++++++++++----------- persipubsub/publisher.py | 4 ++-- persipubsub/queue.py | 16 ++++++++-------- persipubsub/subscriber.py | 8 ++++---- tests/test_control.py | 25 +++++++++++++------------ tests/test_publisher.py | 5 +++-- tests/test_queue.py | 29 +++++++++++++++-------------- tests/test_subscriber.py | 13 +++++++------ 8 files changed, 66 insertions(+), 59 deletions(-) diff --git a/persipubsub/control.py b/persipubsub/control.py index 9fcbcac..5412532 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -6,6 +6,8 @@ import persipubsub.queue +# pylint: disable=protected-access + def initialize_all_dbs(config_pth: Union[pathlib.Path, str]) -> None: """ @@ -20,7 +22,7 @@ def initialize_all_dbs(config_pth: Union[pathlib.Path, str]) -> None: if not queue_dir.exists(): queue_dir.mkdir(parents=True, exist_ok=True) sub_list = values["subscribers"] - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init( config_pth=config_pth, queue_dir=queue_dir, @@ -29,7 +31,7 @@ def initialize_all_dbs(config_pth: Union[pathlib.Path, str]) -> None: max_db_size_bytes=values["max_db_size_bytes"]) for sub in sub_list: - add_sub(sub_id=sub, queue=queue) + _add_sub(sub_id=sub, queue=queue) def clear_all_subs(config_pth: Union[pathlib.Path, str]) -> None: @@ -45,18 +47,18 @@ def clear_all_subs(config_pth: Union[pathlib.Path, str]) -> None: if not queue_dir.exists(): queue_dir.mkdir(parents=True, exist_ok=True) sub_list = values["subscribers"] # type: List[str] - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init( config_pth=config_pth, queue_dir=queue_dir, max_reader_num=values["max_reader_num"], max_db_num=values["max_db_num"], max_db_size_bytes=values["max_db_size_bytes"]) - clear_all_subs_for(queue=queue, sub_list=sub_list) + _clear_all_subs_for(queue=queue, sub_list=sub_list) -def clear_all_subs_for(queue: persipubsub.queue.Queue, - sub_list: List[str]) -> None: +def _clear_all_subs_for(queue: persipubsub.queue._Queue, + sub_list: List[str]) -> None: """ Clear all subscriber and delete all messages for one queue. @@ -80,7 +82,7 @@ def clear_all_subs_for(queue: persipubsub.queue.Queue, txn.drop(db=data_db, delete=False) -def add_sub(sub_id: str, queue: persipubsub.queue.Queue) -> None: +def _add_sub(sub_id: str, queue: persipubsub.queue._Queue) -> None: """ Add a subscriber and create its lmdb. @@ -92,7 +94,7 @@ def add_sub(sub_id: str, queue: persipubsub.queue.Queue) -> None: key=persipubsub.encoding(sub_id), txn=txn, create=True) -def remove_sub(sub_id: str, queue: persipubsub.queue.Queue) -> None: +def _remove_sub(sub_id: str, queue: persipubsub.queue._Queue) -> None: """ Remove a subscriber and delete all its messages. @@ -137,18 +139,19 @@ def prune_dangling_messages(config_pth: Union[pathlib.Path, str]) -> None: if not queue_dir.exists(): queue_dir.mkdir(parents=True, exist_ok=True) sub_list = values["subscribers"] # type: List[str] - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init( config_pth=config_pth, queue_dir=queue_dir, max_reader_num=values["max_reader_num"], max_db_num=values["max_db_num"], max_db_size_bytes=values["max_db_size_bytes"]) - persipubsub.queue.prune_dangling_messages_for( + persipubsub.queue._prune_dangling_messages_for( queue=queue, sub_list=sub_list) -def prune_all_messages_for(sub_id: str, queue: persipubsub.queue.Queue) -> None: +def _prune_all_messages_for(sub_id: str, + queue: persipubsub.queue._Queue) -> None: """ Prune all messages of a subscriber. diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index 5432a8d..0132290 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -22,7 +22,7 @@ class Pub: """ pub_id = None # type: str - queue = None # type: persipubsub.queue.Queue + queue = None # type: persipubsub.queue._Queue sub_list = None # type: List[str] autosync = None # type: bool @@ -45,7 +45,7 @@ def init(self, config = persipubsub.get_config(path=config_pth) publisher = config[pub_id] queue_dir = publisher["out_queue"] - self.queue = persipubsub.queue.Queue() + self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access self.queue.init(config_pth=config_pth, queue_dir=queue_dir) assert isinstance(publisher["subscribers"], List) self.sub_list = publisher["subscribers"] # type: List[str] diff --git a/persipubsub/queue.py b/persipubsub/queue.py index cb2db19..5face45 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -23,7 +23,7 @@ class Strategy(enum.Enum): prune_last = 1 -def parse_strategy(strategy: str) -> Strategy: +def _parse_strategy(strategy: str) -> Strategy: """ Parse overflow strategy. @@ -60,7 +60,7 @@ def __init__(self, if hwm_lmdb_size_bytes is None else hwm_lmdb_size_bytes # type: int -def initialize_environment( +def _initialize_environment( queue_dir: pathlib.Path, max_reader_num: int = 1024, max_db_num: int = 1024, @@ -88,7 +88,7 @@ def initialize_environment( return env -def prune_dangling_messages_for(queue: 'Queue', sub_list: List[str]) -> None: +def _prune_dangling_messages_for(queue: '_Queue', sub_list: List[str]) -> None: """ Prune all dangling messages for subscribers of a queue from lmdb. @@ -141,7 +141,7 @@ def prune_dangling_messages_for(queue: 'Queue', sub_list: List[str]) -> None: txn.delete(key=key, db=sub_db) -class Queue: +class _Queue: """ Queue messages persistently from many publishers for many subscribers. @@ -195,7 +195,7 @@ def init(self, queue_dir = queue_dir if isinstance(queue_dir, pathlib.Path) \ else pathlib.Path(queue_dir) - self.env = initialize_environment( + self.env = _initialize_environment( queue_dir=queue_dir, max_reader_num=max_reader_num, max_db_num=max_db_num, @@ -221,11 +221,11 @@ def init(self, hwm_lmdb_size_bytes=hwm['HWM_LMDB_SIZE_BYTES']) assert isinstance(hwm["strategy"], str) - self.strategy = parse_strategy(hwm["strategy"]) + self.strategy = _parse_strategy(hwm["strategy"]) self.sub_list = queue["subscribers"] - def __enter__(self) -> 'Queue': + def __enter__(self) -> '_Queue': """Enter the context and give the queue prepared in the constructor.""" return self @@ -377,7 +377,7 @@ def prune_dangling_messages(self) -> None: :return: """ - prune_dangling_messages_for(queue=self, sub_list=self.sub_list) + _prune_dangling_messages_for(queue=self, sub_list=self.sub_list) def check_current_lmdb_size(self) -> int: """ diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 2ad5eed..566d265 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -23,7 +23,7 @@ class Sub: """ sub_id = None # type: str - queue = None # type: persipubsub.queue.Queue + queue = None # type: persipubsub.queue._Queue def __init__(self) -> None: """Initialize class object.""" @@ -40,7 +40,7 @@ def init(self, sub_id: str, config_pth: Union[pathlib.Path, str]) -> None: config = persipubsub.get_config(path=config_pth) subscriber = config[sub_id] queue_dir = subscriber["in_queue"] - self.queue = persipubsub.queue.Queue() + self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access self.queue.init(config_pth=config_pth, queue_dir=queue_dir) def __enter__(self) -> 'Sub': @@ -74,9 +74,9 @@ def receive(self, timeout: int = 60, retries: int = 10) -> Generator: pass if msg is not None: - self.pop() + self._pop() - def pop(self) -> None: + def _pop(self) -> None: """Pop a message from the subscriber's lmdb.""" self.queue.pop(sub_id=self.sub_id) diff --git a/tests/test_control.py b/tests/test_control.py index 9529acc..e6bd7c8 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -13,6 +13,7 @@ import tests # pylint: disable=missing-docstring +# pylint: disable=protected-access class TestControl(unittest.TestCase): @@ -27,7 +28,7 @@ def test_initialize_all(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") expected_db_keys = [b'data_db', b'meta_db', b'pending_db', b'sub'] @@ -50,11 +51,11 @@ def test_add_sub(self): with open(file=file.as_posix(), mode='wt') as file_object: json.dump(config, file_object) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - persipubsub.control.add_sub(sub_id="sub1", queue=queue) - persipubsub.control.add_sub(sub_id="sub2", queue=queue) + persipubsub.control._add_sub(sub_id="sub1", queue=queue) + persipubsub.control._add_sub(sub_id="sub2", queue=queue) expected_db_keys = [ b'data_db', b'meta_db', b'pending_db', b'sub1', b'sub2' @@ -78,12 +79,12 @@ def test_del_sub(self): with open(file=file.as_posix(), mode='wt') as file_object: json.dump(config, file_object) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - persipubsub.control.add_sub(sub_id="sub1", queue=queue) - persipubsub.control.add_sub(sub_id="sub2", queue=queue) - persipubsub.control.remove_sub(sub_id="sub2", queue=queue) + persipubsub.control._add_sub(sub_id="sub1", queue=queue) + persipubsub.control._add_sub(sub_id="sub2", queue=queue) + persipubsub.control._remove_sub(sub_id="sub2", queue=queue) expected_db_keys = [b'data_db', b'meta_db', b'pending_db', b'sub1'] db_keys = [] # type: List[bytes] @@ -105,7 +106,7 @@ def test_clear_all_subs(self): with open(file=file.as_posix(), mode='wt') as file_object: json.dump(config, file_object) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") persipubsub.control.initialize_all_dbs(config_pth=file) @@ -134,7 +135,7 @@ def test_prune_dangling_messages(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT @@ -229,7 +230,7 @@ def test_prune_all_messages_for_subscriber(self): with open(file=file.as_posix(), mode='wt') as file_object: json.dump(config, file_object) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") persipubsub.control.initialize_all_dbs(config_pth=file) @@ -245,7 +246,7 @@ def test_prune_all_messages_for_subscriber(self): sub_stat = txn.stat(db=sub_db) self.assertEqual(2, sub_stat['entries']) - persipubsub.control.prune_all_messages_for( + persipubsub.control._prune_all_messages_for( sub_id="sub", queue=queue) with queue.env.begin(write=False) as txn: diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 7b6fae7..6bc10ed 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -13,6 +13,7 @@ import tests # pylint: disable=missing-docstring +# pylint: disable=protected-access class TestPublisher(unittest.TestCase): @@ -27,7 +28,7 @@ def test_send(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") pub = persipubsub.publisher.Pub() @@ -62,7 +63,7 @@ def test_send_to_nonexisting_db(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") pub = persipubsub.publisher.Pub() diff --git a/tests/test_queue.py b/tests/test_queue.py index 3ebf129..a247737 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -12,12 +12,13 @@ import tests # pylint: disable=missing-docstring +# pylint: disable=protected-access class TestQueue(unittest.TestCase): def test_initialize_environment(self): with temppathlib.TemporaryDirectory() as tmp_dir: - env = persipubsub.queue.initialize_environment( + env = persipubsub.queue._initialize_environment( queue_dir=tmp_dir.path) self.assertDictEqual({ @@ -51,7 +52,7 @@ def test_put_to_single_subscriber(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.put(msg=msg, sub_list=[subscriber]) @@ -89,10 +90,10 @@ def test_put_multiple_subscriber(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - persipubsub.control.add_sub('another_sub', queue=queue) + persipubsub.control._add_sub('another_sub', queue=queue) queue.put(msg=msg, sub_list=sub_list) @@ -139,7 +140,7 @@ def test_put_many(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") msg = "I'm a message".encode(tests.ENCODING) @@ -185,7 +186,7 @@ def test_front(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.put(msg=msg, sub_list=[subscriber]) @@ -210,7 +211,7 @@ def test_pop(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.put(msg=msg, sub_list=[subscriber]) @@ -257,7 +258,7 @@ def test_pop_queue_empty(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") self.assertRaises(RuntimeError, queue.pop, sub_id=subscriber) @@ -273,7 +274,7 @@ def test_queue_initialisation(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") @@ -300,7 +301,7 @@ def test_overflow_msgs_limit(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") @@ -331,7 +332,7 @@ def test_overflow_limit_size(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") @@ -361,7 +362,7 @@ def test_timeout(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT @@ -387,7 +388,7 @@ def test_strategy_prune_first(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.strategy = persipubsub.queue.Strategy.prune_first @@ -421,7 +422,7 @@ def test_strategy_prune_last(self): persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.strategy = persipubsub.queue.Strategy.prune_last diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index 7c93e2f..cc32a1b 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -12,6 +12,7 @@ import tests # pylint: disable=missing-docstring +# pylint: disable=protected-access class TestSubscriber(unittest.TestCase): @@ -25,7 +26,7 @@ def test_receive_message(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") sub = persipubsub.subscriber.Sub() @@ -48,7 +49,7 @@ def test_timeout_subscriber(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") sub = persipubsub.subscriber.Sub() @@ -83,7 +84,7 @@ def test_pop(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") sub = persipubsub.subscriber.Sub() @@ -95,7 +96,7 @@ def test_pop(self): msg2 = "I'm a message too".encode(tests.ENCODING) queue.put(msg=msg2, sub_list=queue.sub_list) - sub.pop() + sub._pop() with sub.receive() as msg: self.assertIsNotNone(msg) @@ -111,13 +112,13 @@ def test_pop_when_empty(self): json.dump(config, file_object) persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue.Queue() + queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") sub = persipubsub.subscriber.Sub() sub.init(sub_id='sub', config_pth=file) - self.assertRaises(RuntimeError, sub.pop) + self.assertRaises(RuntimeError, sub._pop) if __name__ == '__main__': From 32b6faffec857121292ac4bf9e9bb1f83ded35fa Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 16:08:21 +0100 Subject: [PATCH 13/35] privatize classes --- persipubsub/queue.py | 22 +++++++++++----------- tests/test_queue.py | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 5face45..7d410ba 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -16,27 +16,27 @@ HWM_LMDB_SIZE_BYTES = 30 * 1024**3 # type: int -class Strategy(enum.Enum): +class _Strategy(enum.Enum): """Store possible strategies.""" prune_first = 0 prune_last = 1 -def _parse_strategy(strategy: str) -> Strategy: +def _parse_strategy(strategy: str) -> _Strategy: """ Parse overflow strategy. :param strategy: Strategy stored in config :return: set overflow strategy """ - if Strategy.prune_first.name == strategy: - return Strategy.prune_first + if _Strategy.prune_first.name == strategy: + return _Strategy.prune_first - return Strategy.prune_last + return _Strategy.prune_last -class HighWaterMark: +class _HighWaterMark: """Store high water mark limits.""" def __init__(self, @@ -161,8 +161,8 @@ class _Queue: config_pth = None # type: pathlib.Path env = None # type: lmdb.Environment - hwm = None # type: HighWaterMark - strategy = None # type: Strategy + hwm = None # type: _HighWaterMark + strategy = None # type: _Strategy sub_list = None # type: List[str] def __init__(self) -> None: @@ -215,7 +215,7 @@ def init(self, assert isinstance(hwm['MSG_TIMEOUT_SECS'], int) assert isinstance(hwm['MAX_MSGS_NUM'], int) assert isinstance(hwm['HWM_LMDB_SIZE_BYTES'], int) - self.hwm = HighWaterMark( + self.hwm = _HighWaterMark( msg_timeout_secs=hwm['MSG_TIMEOUT_SECS'], max_msgs_num=hwm['MAX_MSGS_NUM'], hwm_lmdb_size_bytes=hwm['HWM_LMDB_SIZE_BYTES']) @@ -444,7 +444,7 @@ def prune_messages(self) -> None: entries = meta_stat['entries'] cursor = txn.cursor(db=meta_db) - if self.strategy == Strategy.prune_first: + if self.strategy == _Strategy.prune_first: cursor.first() for index, key in enumerate( @@ -453,7 +453,7 @@ def prune_messages(self) -> None: if index >= int(entries / 2): break - elif self.strategy == Strategy.prune_last: + elif self.strategy == _Strategy.prune_last: cursor.last() for index, key in enumerate( cursor.iterprev(keys=True, values=False)): diff --git a/tests/test_queue.py b/tests/test_queue.py index a247737..3c7c0aa 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -391,7 +391,7 @@ def test_strategy_prune_first(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.strategy = persipubsub.queue.Strategy.prune_first + queue.strategy = persipubsub.queue._Strategy.prune_first queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM @@ -425,7 +425,7 @@ def test_strategy_prune_last(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.strategy = persipubsub.queue.Strategy.prune_last + queue.strategy = persipubsub.queue._Strategy.prune_last queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM From fc9b7dcdef07c269d5122465d8163c9c2b49ec98 Mon Sep 17 00:00:00 2001 From: slimeth Date: Fri, 25 Jan 2019 16:08:42 +0100 Subject: [PATCH 14/35] extended README --- README.rst | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d8fb6e8..9628bdf 100644 --- a/README.rst +++ b/README.rst @@ -173,10 +173,22 @@ Send a message .. code-block:: python - msg = "Hello there!".encode('utf-8) + msg = "Hello there!".encode('utf-8') pub.send(msg=msg) - # subscribers have now a message in the queue + # subscribers have now a message in the queue. + +Send many messages at once +"""""""""""""""""""""""""" + +.. code-block:: python + + msgs = ["What's up?".encode('utf-8'), + "Do you like the README?".encode('utf-8')] + pub.send_many(msgs=msgs) + + # Subscribers have now both messages in the queue not necessary in the list + # order. Subscriber ^^^^^^^^^^ @@ -206,6 +218,22 @@ Receive a message # sub queue is now empty +Catch up with latest message +"""""""""""""""""""""""""""" + +In case your subscriber is slow, the queue is growing and you want to want to +receive the latest messages. + +.. code-block:: python + + # many outdated messages in queue + sub.pop_to_top() + # most recent message left in queue + with sub.receive() as msg: + # do something with the message + + # sub queue is now empty + Documentation ============= The documentation is available on `readthedocs From 3d50ab3d58376b602ed27215f29b018f1cd6ddb4 Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 30 Jan 2019 12:24:02 +0100 Subject: [PATCH 15/35] remove private queues from docs --- docs/source/persipubsub/index.rst | 1 - docs/source/persipubsub/persipubsub.queue.rst | 5 ----- 2 files changed, 6 deletions(-) delete mode 100644 docs/source/persipubsub/persipubsub.queue.rst diff --git a/docs/source/persipubsub/index.rst b/docs/source/persipubsub/index.rst index 96e971e..dbf7647 100644 --- a/docs/source/persipubsub/index.rst +++ b/docs/source/persipubsub/index.rst @@ -5,6 +5,5 @@ persipubsub .. toctree:: persipubsub.ctl - persipubsub.queue persipubsub.publisher persipubsub.subscriber \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.queue.rst b/docs/source/persipubsub/persipubsub.queue.rst deleted file mode 100644 index 3eccc54..0000000 --- a/docs/source/persipubsub/persipubsub.queue.rst +++ /dev/null @@ -1,5 +0,0 @@ -Queue -***** - -.. automodule:: persipubsub.queue - :members: \ No newline at end of file From 70c3ee1218dad560c937020d043ff0008081604d Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 30 Jan 2019 12:24:37 +0100 Subject: [PATCH 16/35] move common test methods --- tests/__init__.py | 78 ----------------------- tests/common.py | 80 ++++++++++++++++++++++++ tests/test_control.py | 77 +++++++++++++---------- tests/test_publisher.py | 24 +++---- tests/test_queue.py | 132 ++++++++++++++++++++------------------- tests/test_subscriber.py | 22 +++---- 6 files changed, 214 insertions(+), 199 deletions(-) create mode 100644 tests/common.py diff --git a/tests/__init__.py b/tests/__init__.py index c30181b..07055c7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,80 +1,2 @@ #!/usr/bin/env python3 """Test persipubsub.""" - -import json -import pathlib -import unittest -from typing import Any, Dict - -import temppathlib - -import persipubsub - -# pylint: disable=missing-docstring - -LMDB_PAGE_SIZE = 4096 - -TEST_MSG_TIMEOUT = 1 # type: int -TEST_HWM_MSG_NUM = 10 # type: int -TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int - -# define all encodings here -ENCODING = 'utf-8' -BYTES_LENGTH = 8 -BYTES_ORDER = 'big' - -# define all database names here -DATA_DB = "data_db".encode(ENCODING) # msg_id | data -PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber -META_DB = "meta_db".encode(ENCODING) # msg_id | metadata - - -def generate_test_config(path: pathlib.Path) -> Dict[str, Any]: - queue_dir = path / "queue" - queue_dir.mkdir(parents=True, exist_ok=True) - - test_config = { - "pub": { - "out_queue": queue_dir.as_posix(), - "subscribers": ["sub"] - }, - "sub": { - "in_queue": queue_dir.as_posix() - }, - "queues": { - queue_dir.as_posix(): { - "path": queue_dir.as_posix(), - "max_reader_num": 1024, - "max_db_num": 1024, - "max_db_size_bytes": 34359738368, - "subscribers": ["sub"], - "high-water-mark": { - "MSG_TIMEOUT_SECS": TEST_MSG_TIMEOUT, - "MAX_MSGS_NUM": TEST_HWM_MSG_NUM, - "HWM_LMDB_SIZE_BYTES": TEST_HWM_LMDB_SIZE, - "strategy": "prune_first" - } - }, - } - } - - return test_config - - -class TestPersiPubSub(unittest.TestCase): - def test_get_config(self): - with temppathlib.TemporaryDirectory() as tmp_dir: - expected_config = generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(expected_config, file_object) - - config = persipubsub.get_config(path=file) - - self.assertDictEqual(expected_config, config) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 0000000..c30181b --- /dev/null +++ b/tests/common.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Test persipubsub.""" + +import json +import pathlib +import unittest +from typing import Any, Dict + +import temppathlib + +import persipubsub + +# pylint: disable=missing-docstring + +LMDB_PAGE_SIZE = 4096 + +TEST_MSG_TIMEOUT = 1 # type: int +TEST_HWM_MSG_NUM = 10 # type: int +TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int + +# define all encodings here +ENCODING = 'utf-8' +BYTES_LENGTH = 8 +BYTES_ORDER = 'big' + +# define all database names here +DATA_DB = "data_db".encode(ENCODING) # msg_id | data +PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber +META_DB = "meta_db".encode(ENCODING) # msg_id | metadata + + +def generate_test_config(path: pathlib.Path) -> Dict[str, Any]: + queue_dir = path / "queue" + queue_dir.mkdir(parents=True, exist_ok=True) + + test_config = { + "pub": { + "out_queue": queue_dir.as_posix(), + "subscribers": ["sub"] + }, + "sub": { + "in_queue": queue_dir.as_posix() + }, + "queues": { + queue_dir.as_posix(): { + "path": queue_dir.as_posix(), + "max_reader_num": 1024, + "max_db_num": 1024, + "max_db_size_bytes": 34359738368, + "subscribers": ["sub"], + "high-water-mark": { + "MSG_TIMEOUT_SECS": TEST_MSG_TIMEOUT, + "MAX_MSGS_NUM": TEST_HWM_MSG_NUM, + "HWM_LMDB_SIZE_BYTES": TEST_HWM_LMDB_SIZE, + "strategy": "prune_first" + } + }, + } + } + + return test_config + + +class TestPersiPubSub(unittest.TestCase): + def test_get_config(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + expected_config = generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(expected_config, file_object) + + config = persipubsub.get_config(path=file) + + self.assertDictEqual(expected_config, config) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_control.py b/tests/test_control.py index e6bd7c8..767f813 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -10,7 +10,7 @@ import persipubsub.control import persipubsub.queue -import tests +import tests.common # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -19,7 +19,7 @@ class TestControl(unittest.TestCase): def test_initialize_all(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -44,7 +44,7 @@ def test_initialize_all(self): def test_add_sub(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -72,7 +72,7 @@ def test_add_sub(self): def test_del_sub(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -99,7 +99,7 @@ def test_del_sub(self): def test_clear_all_subs(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -126,7 +126,7 @@ def test_clear_all_subs(self): def test_prune_dangling_messages(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -137,70 +137,79 @@ def test_prune_dangling_messages(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT + queue.hwm.msg_timeout_secs = tests.common.TEST_MSG_TIMEOUT with queue.env.begin(write=True) as txn: sub_db = queue.env.open_db( - key='sub'.encode(tests.ENCODING), txn=txn, create=False) + key='sub'.encode(tests.common.ENCODING), + txn=txn, + create=False) - txn.put(key="timeout_msg".encode(tests.ENCODING), db=sub_db) - txn.put(key="valid_msg".encode(tests.ENCODING), db=sub_db) + txn.put( + key="timeout_msg".encode(tests.common.ENCODING), db=sub_db) + txn.put( + key="valid_msg".encode(tests.common.ENCODING), db=sub_db) with queue.env.begin(write=True) as txn: data_db = queue.env.open_db( - key=tests.DATA_DB, txn=txn, create=False) + key=tests.common.DATA_DB, txn=txn, create=False) txn.put( - key="popped_msg".encode(tests.ENCODING), - value="I'm data".encode(tests.ENCODING), + key="popped_msg".encode(tests.common.ENCODING), + value="I'm data".encode(tests.common.ENCODING), db=data_db) txn.put( - key="timeout_msg".encode(tests.ENCODING), - value="I'm data too".encode(tests.ENCODING), + key="timeout_msg".encode(tests.common.ENCODING), + value="I'm data too".encode(tests.common.ENCODING), db=data_db) txn.put( - key="valid_msg".encode(tests.ENCODING), - value="Free me!".encode(tests.ENCODING), + key="valid_msg".encode(tests.common.ENCODING), + value="Free me!".encode(tests.common.ENCODING), db=data_db) with queue.env.begin(write=True) as txn: pending_db = queue.env.open_db( - key=tests.PENDING_DB, txn=txn, create=False) + key=tests.common.PENDING_DB, txn=txn, create=False) txn.put( - key="popped_msg".encode(tests.ENCODING), + key="popped_msg".encode(tests.common.ENCODING), value=int(0).to_bytes( - length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + length=tests.common.BYTES_LENGTH, + byteorder=tests.common.BYTES_ORDER), db=pending_db) txn.put( - key="timeout_msg".encode(tests.ENCODING), + key="timeout_msg".encode(tests.common.ENCODING), value=int(1).to_bytes( - length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + length=tests.common.BYTES_LENGTH, + byteorder=tests.common.BYTES_ORDER), db=pending_db) txn.put( - key="valid_msg".encode(tests.ENCODING), + key="valid_msg".encode(tests.common.ENCODING), value=int(1).to_bytes( - length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + length=tests.common.BYTES_LENGTH, + byteorder=tests.common.BYTES_ORDER), db=pending_db) with queue.env.begin(write=True) as txn: meta_db = queue.env.open_db( - key=tests.META_DB, txn=txn, create=False) + key=tests.common.META_DB, txn=txn, create=False) txn.put( - key="popped_msg".encode(tests.ENCODING), + key="popped_msg".encode(tests.common.ENCODING), value=int(datetime.datetime.utcnow().timestamp()).to_bytes( - length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + length=tests.common.BYTES_LENGTH, + byteorder=tests.common.BYTES_ORDER), db=meta_db) txn.put( - key="timeout_msg".encode(tests.ENCODING), + key="timeout_msg".encode(tests.common.ENCODING), value=int(datetime.datetime.utcnow().timestamp() - - tests.TEST_MSG_TIMEOUT - 5).to_bytes( - length=tests.BYTES_LENGTH, - byteorder=tests.BYTES_ORDER), + tests.common.TEST_MSG_TIMEOUT - 5).to_bytes( + length=tests.common.BYTES_LENGTH, + byteorder=tests.common.BYTES_ORDER), db=meta_db) txn.put( - key="valid_msg".encode(tests.ENCODING), + key="valid_msg".encode(tests.common.ENCODING), value=int(datetime.datetime.utcnow().timestamp()).to_bytes( - length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), + length=tests.common.BYTES_LENGTH, + byteorder=tests.common.BYTES_ORDER), db=meta_db) persipubsub.control.prune_dangling_messages(config_pth=file) @@ -223,7 +232,7 @@ def test_prune_dangling_messages(self): def test_prune_all_messages_for_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 6bc10ed..72f2d00 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -10,7 +10,7 @@ import persipubsub.control import persipubsub.publisher import persipubsub.queue -import tests +import tests.common # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -20,7 +20,7 @@ class TestPublisher(unittest.TestCase): def test_send(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -34,10 +34,10 @@ def test_send(self): pub = persipubsub.publisher.Pub() pub.init(pub_id="pub", config_pth=file) - msg = "Hello world!".encode(tests.ENCODING) + msg = "Hello world!".encode(tests.common.ENCODING) pub.send(msg=msg) - subscriber = "sub".encode(tests.ENCODING) + subscriber = "sub".encode(tests.common.ENCODING) with queue.env.begin(write=False) as txn: self.assertIsNotNone(txn.get(key=subscriber)) sub_db = queue.env.open_db( @@ -48,14 +48,14 @@ def test_send(self): key = cursor.key() data_db = queue.env.open_db( - key=tests.DATA_DB, txn=txn, create=False) + key=tests.common.DATA_DB, txn=txn, create=False) item = txn.get(key=key, db=data_db) self.assertIsNotNone(item) self.assertEqual(msg, item) def test_send_to_nonexisting_db(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -71,13 +71,13 @@ def test_send_to_nonexisting_db(self): pub.sub_list.append('non-existent sub') - msg = "message for every subscriber".encode(tests.ENCODING) + msg = "message for every subscriber".encode(tests.common.ENCODING) self.assertRaises(lmdb.NotFoundError, pub.send, msg=msg) def test_send_many(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -89,7 +89,7 @@ def test_send_many(self): pub = persipubsub.publisher.Pub() pub.init(pub_id='pub', config_pth=file) - msg = "I'm a message".encode(tests.ENCODING) + msg = "I'm a message".encode(tests.common.ENCODING) msgs = [] msg_num = 10 for _ in range(msg_num): @@ -101,10 +101,10 @@ def test_send_many(self): with pub.queue.env.begin(write=False) as txn: self.assertIsNotNone( - txn.get(key=subscriber.encode(tests.ENCODING))) + txn.get(key=subscriber.encode(tests.common.ENCODING))) sub_db = pub.queue.env.open_db( - key=subscriber.encode(tests.ENCODING), + key=subscriber.encode(tests.common.ENCODING), txn=txn, create=False) @@ -112,7 +112,7 @@ def test_send_many(self): self.assertEqual(msg_num, sub_stat['entries']) data_db = pub.queue.env.open_db( - key=tests.DATA_DB, txn=txn, create=False) + key=tests.common.DATA_DB, txn=txn, create=False) data_stat = txn.stat(db=data_db) self.assertEqual(msg_num, data_stat['entries']) diff --git a/tests/test_queue.py b/tests/test_queue.py index 3c7c0aa..449f57c 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -9,7 +9,7 @@ import persipubsub.control import persipubsub.queue -import tests +import tests.common # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -40,11 +40,11 @@ def test_initialize_environment(self): def test_put_to_single_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.common.ENCODING) subscriber = "sub" - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -58,10 +58,10 @@ def test_put_to_single_subscriber(self): with queue.env.begin() as txn: self.assertIsNotNone( - txn.get(key=subscriber.encode(tests.ENCODING))) + txn.get(key=subscriber.encode(tests.common.ENCODING))) sub_db = queue.env.open_db( - key=subscriber.encode(tests.ENCODING), + key=subscriber.encode(tests.common.ENCODING), txn=txn, create=False) cursor = txn.cursor(db=sub_db) @@ -69,7 +69,7 @@ def test_put_to_single_subscriber(self): key = cursor.key() data_db = queue.env.open_db( - key=tests.DATA_DB, txn=txn, create=False) + key=tests.common.DATA_DB, txn=txn, create=False) value = txn.get(key=key, db=data_db) self.assertIsNotNone(value) @@ -78,11 +78,11 @@ def test_put_to_single_subscriber(self): def test_put_multiple_subscriber(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.common.ENCODING) sub_list = ["sub", "another_sub"] - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -99,10 +99,10 @@ def test_put_multiple_subscriber(self): with queue.env.begin() as txn: self.assertIsNotNone( - txn.get(key=sub_list[0].encode(tests.ENCODING))) + txn.get(key=sub_list[0].encode(tests.common.ENCODING))) sub_db_0 = queue.env.open_db( - key=sub_list[0].encode(tests.ENCODING), + key=sub_list[0].encode(tests.common.ENCODING), txn=txn, create=False) cursor = txn.cursor(db=sub_db_0) @@ -110,10 +110,10 @@ def test_put_multiple_subscriber(self): key_0 = cursor.key() self.assertIsNotNone( - txn.get(key=sub_list[1].encode(tests.ENCODING))) + txn.get(key=sub_list[1].encode(tests.common.ENCODING))) sub_db_1 = queue.env.open_db( - key=sub_list[1].encode(tests.ENCODING), + key=sub_list[1].encode(tests.common.ENCODING), txn=txn, create=False) cursor = txn.cursor(db=sub_db_1) @@ -123,7 +123,7 @@ def test_put_multiple_subscriber(self): self.assertEqual(key_0, key_1) data_db = queue.env.open_db( - key=tests.DATA_DB, txn=txn, create=False) + key=tests.common.DATA_DB, txn=txn, create=False) value = txn.get(key=key_1, db=data_db) self.assertIsNotNone(value) @@ -132,7 +132,7 @@ def test_put_multiple_subscriber(self): def test_put_many(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -143,7 +143,7 @@ def test_put_many(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - msg = "I'm a message".encode(tests.ENCODING) + msg = "I'm a message".encode(tests.common.ENCODING) msgs = [] msg_num = 10 for _ in range(msg_num): @@ -155,10 +155,10 @@ def test_put_many(self): with queue.env.begin(write=False) as txn: self.assertIsNotNone( - txn.get(key=subscriber.encode(tests.ENCODING))) + txn.get(key=subscriber.encode(tests.common.ENCODING))) sub_db = queue.env.open_db( - key=subscriber.encode(tests.ENCODING), + key=subscriber.encode(tests.common.ENCODING), txn=txn, create=False) @@ -166,18 +166,18 @@ def test_put_many(self): self.assertEqual(msg_num, sub_stat['entries']) data_db = queue.env.open_db( - key=tests.DATA_DB, txn=txn, create=False) + key=tests.common.DATA_DB, txn=txn, create=False) data_stat = txn.stat(db=data_db) self.assertEqual(msg_num, data_stat['entries']) def test_front(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.common.ENCODING) subscriber = "sub" - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -198,11 +198,11 @@ def test_front(self): def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.ENCODING) + msg = "I'm a message.".encode(tests.common.ENCODING) subscriber = "sub" - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -222,7 +222,7 @@ def test_pop(self): with queue.env.begin() as txn: pending_db = queue.env.open_db( - key=tests.PENDING_DB, txn=txn, create=False) + key=tests.common.PENDING_DB, txn=txn, create=False) cursor = txn.cursor(db=pending_db) self.assertTrue(cursor.first()) @@ -235,21 +235,21 @@ def test_pop(self): with queue.env.begin() as txn: pending_db = queue.env.open_db( - key=tests.PENDING_DB, txn=txn, create=False) + key=tests.common.PENDING_DB, txn=txn, create=False) cursor = txn.cursor(db=pending_db) self.assertTrue(cursor.first()) pending_after_pop = cursor.value() self.assertEqual( - int.from_bytes(pending_before_pop, tests.BYTES_ORDER) - 1, - int.from_bytes(pending_after_pop, tests.BYTES_ORDER)) + int.from_bytes(pending_before_pop, tests.common.BYTES_ORDER) - + 1, int.from_bytes(pending_after_pop, tests.common.BYTES_ORDER)) def test_pop_queue_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: subscriber = "sub" - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -265,7 +265,7 @@ def test_pop_queue_empty(self): def test_queue_initialisation(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -292,7 +292,7 @@ def test_queue_initialisation(self): def test_overflow_msgs_limit(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -305,25 +305,25 @@ def test_overflow_msgs_limit(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + queue.hwm.max_msgs_num = tests.common.TEST_HWM_MSG_NUM - msg = "hello world".encode(tests.ENCODING) + msg = "hello world".encode(tests.common.ENCODING) self.assertEqual(0, queue.count_msgs()) - for _ in range(tests.TEST_HWM_MSG_NUM): + for _ in range(tests.common.TEST_HWM_MSG_NUM): queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.count_msgs()) + self.assertEqual(tests.common.TEST_HWM_MSG_NUM, queue.count_msgs()) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual( - int(tests.TEST_HWM_MSG_NUM - int(tests.TEST_HWM_MSG_NUM / 2)), - queue.count_msgs()) + int(tests.common.TEST_HWM_MSG_NUM - + int(tests.common.TEST_HWM_MSG_NUM / 2)), queue.count_msgs()) def test_overflow_limit_size(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -336,24 +336,26 @@ def test_overflow_limit_size(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE + queue.hwm.hwm_lmdb_size_bytes = tests.common.TEST_HWM_LMDB_SIZE - msg = ("a" * (int(tests.LMDB_PAGE_SIZE / 4))).encode(tests.ENCODING) + msg = ("a" * (int(tests.common.LMDB_PAGE_SIZE / 4))).encode( + tests.common.ENCODING) - while queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE: + while queue.check_current_lmdb_size( + ) <= tests.common.TEST_HWM_LMDB_SIZE: queue.put(msg=msg, sub_list=queue.sub_list) - self.assertTrue( - queue.check_current_lmdb_size() > tests.TEST_HWM_LMDB_SIZE) + self.assertTrue(queue.check_current_lmdb_size() > tests.common. + TEST_HWM_LMDB_SIZE) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertTrue( - queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE) + self.assertTrue(queue.check_current_lmdb_size() <= tests.common. + TEST_HWM_LMDB_SIZE) def test_timeout(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -365,21 +367,21 @@ def test_timeout(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT + queue.hwm.msg_timeout_secs = tests.common.TEST_MSG_TIMEOUT - msg = "hello world".encode(tests.ENCODING) + msg = "hello world".encode(tests.common.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual(1, queue.count_msgs()) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual(2, queue.count_msgs()) - time.sleep(tests.TEST_MSG_TIMEOUT + 1) + time.sleep(tests.common.TEST_MSG_TIMEOUT + 1) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual(1, queue.count_msgs()) def test_strategy_prune_first(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -393,27 +395,28 @@ def test_strategy_prune_first(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.strategy = persipubsub.queue._Strategy.prune_first - queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + queue.hwm.max_msgs_num = tests.common.TEST_HWM_MSG_NUM - for index in range(tests.TEST_HWM_MSG_NUM): - msg = "secret message {}".format(index).encode(tests.ENCODING) + for index in range(tests.common.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode( + tests.common.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual("secret message 0".encode(tests.ENCODING), + self.assertEqual("secret message 0".encode(tests.common.ENCODING), queue.front(sub_id='sub')) - msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( - tests.ENCODING) + msg = "secret message {}".format( + tests.common.TEST_HWM_MSG_NUM).encode(tests.common.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual( "secret message {}".format( - int((tests.TEST_HWM_MSG_NUM / 2) + 1)).encode( - tests.ENCODING), queue.front(sub_id='sub')) + int((tests.common.TEST_HWM_MSG_NUM / 2) + 1)).encode( + tests.common.ENCODING), queue.front(sub_id='sub')) def test_strategy_prune_last(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -427,20 +430,21 @@ def test_strategy_prune_last(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.strategy = persipubsub.queue._Strategy.prune_last - queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM + queue.hwm.max_msgs_num = tests.common.TEST_HWM_MSG_NUM - for index in range(tests.TEST_HWM_MSG_NUM): - msg = "secret message {}".format(index).encode(tests.ENCODING) + for index in range(tests.common.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode( + tests.common.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual("secret message 0".encode(tests.ENCODING), + self.assertEqual("secret message 0".encode(tests.common.ENCODING), queue.front(sub_id='sub')) - msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( - tests.ENCODING) + msg = "secret message {}".format( + tests.common.TEST_HWM_MSG_NUM).encode(tests.common.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual("secret message 0".encode(tests.ENCODING), + self.assertEqual("secret message 0".encode(tests.common.ENCODING), queue.front(sub_id='sub')) diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index cc32a1b..a29c2dc 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -9,7 +9,7 @@ import persipubsub.control import persipubsub.queue import persipubsub.subscriber -import tests +import tests.common # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -18,7 +18,7 @@ class TestSubscriber(unittest.TestCase): def test_receive_message(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -32,7 +32,7 @@ def test_receive_message(self): sub = persipubsub.subscriber.Sub() sub.init(sub_id="sub", config_pth=file) - msg = "Hello World!".encode(tests.ENCODING) + msg = "Hello World!".encode(tests.common.ENCODING) queue.put(msg=msg, sub_list=["sub"]) with sub.receive(timeout=1) as received_msg: @@ -41,7 +41,7 @@ def test_receive_message(self): def test_timeout_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -58,25 +58,25 @@ def test_timeout_subscriber(self): with sub.receive(timeout=1) as received_msg: self.assertIsNone(received_msg) msg = "message send after timeout and will not be popped" \ - "".encode(tests.ENCODING) + "".encode(tests.common.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) self.assertIsNone(received_msg) self.assertEqual( "message send after timeout and will not be popped".encode( - tests.ENCODING), queue.front(sub_id='sub')) + tests.common.ENCODING), queue.front(sub_id='sub')) with sub.receive(timeout=1) as received_msg: self.assertIsNotNone(received_msg) self.assertEqual( "message send after timeout and will not be popped".encode( - tests.ENCODING), received_msg) + tests.common.ENCODING), received_msg) self.assertIsNone(queue.front(sub_id='sub')) def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -90,10 +90,10 @@ def test_pop(self): sub = persipubsub.subscriber.Sub() sub.init(sub_id='sub', config_pth=file) - msg1 = "I'm a message".encode(tests.ENCODING) + msg1 = "I'm a message".encode(tests.common.ENCODING) queue.put(msg=msg1, sub_list=queue.sub_list) - msg2 = "I'm a message too".encode(tests.ENCODING) + msg2 = "I'm a message too".encode(tests.common.ENCODING) queue.put(msg=msg2, sub_list=queue.sub_list) sub._pop() @@ -104,7 +104,7 @@ def test_pop(self): def test_pop_when_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + config = tests.common.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" From 6fe702e59522e69c36c0c928e31b2cbf3a88a031 Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 30 Jan 2019 12:26:54 +0100 Subject: [PATCH 17/35] fix mdB_BAD_RSLOT error --- persipubsub/queue.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 7d410ba..cb41d55 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -83,7 +83,8 @@ def _initialize_environment( map_size=max_db_size_bytes, subdir=True, max_readers=max_reader_num, - max_dbs=max_db_num) + max_dbs=max_db_num, + max_spare_txns=0) return env From b5028c374b272384e7b94e489a944416a407cd54 Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 30 Jan 2019 14:05:57 +0100 Subject: [PATCH 18/35] undo import tests.common --- tests/__init__.py | 78 +++++++++++++++++++++++ tests/common.py | 80 ------------------------ tests/test_control.py | 77 ++++++++++------------- tests/test_publisher.py | 24 +++---- tests/test_queue.py | 132 +++++++++++++++++++-------------------- tests/test_subscriber.py | 22 +++---- 6 files changed, 199 insertions(+), 214 deletions(-) delete mode 100644 tests/common.py diff --git a/tests/__init__.py b/tests/__init__.py index 07055c7..c30181b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,2 +1,80 @@ #!/usr/bin/env python3 """Test persipubsub.""" + +import json +import pathlib +import unittest +from typing import Any, Dict + +import temppathlib + +import persipubsub + +# pylint: disable=missing-docstring + +LMDB_PAGE_SIZE = 4096 + +TEST_MSG_TIMEOUT = 1 # type: int +TEST_HWM_MSG_NUM = 10 # type: int +TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int + +# define all encodings here +ENCODING = 'utf-8' +BYTES_LENGTH = 8 +BYTES_ORDER = 'big' + +# define all database names here +DATA_DB = "data_db".encode(ENCODING) # msg_id | data +PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber +META_DB = "meta_db".encode(ENCODING) # msg_id | metadata + + +def generate_test_config(path: pathlib.Path) -> Dict[str, Any]: + queue_dir = path / "queue" + queue_dir.mkdir(parents=True, exist_ok=True) + + test_config = { + "pub": { + "out_queue": queue_dir.as_posix(), + "subscribers": ["sub"] + }, + "sub": { + "in_queue": queue_dir.as_posix() + }, + "queues": { + queue_dir.as_posix(): { + "path": queue_dir.as_posix(), + "max_reader_num": 1024, + "max_db_num": 1024, + "max_db_size_bytes": 34359738368, + "subscribers": ["sub"], + "high-water-mark": { + "MSG_TIMEOUT_SECS": TEST_MSG_TIMEOUT, + "MAX_MSGS_NUM": TEST_HWM_MSG_NUM, + "HWM_LMDB_SIZE_BYTES": TEST_HWM_LMDB_SIZE, + "strategy": "prune_first" + } + }, + } + } + + return test_config + + +class TestPersiPubSub(unittest.TestCase): + def test_get_config(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + expected_config = generate_test_config(path=tmp_dir.path) + + file = tmp_dir.path / "config.json" + + with open(file=file.as_posix(), mode='wt') as file_object: + json.dump(expected_config, file_object) + + config = persipubsub.get_config(path=file) + + self.assertDictEqual(expected_config, config) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/common.py b/tests/common.py deleted file mode 100644 index c30181b..0000000 --- a/tests/common.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""Test persipubsub.""" - -import json -import pathlib -import unittest -from typing import Any, Dict - -import temppathlib - -import persipubsub - -# pylint: disable=missing-docstring - -LMDB_PAGE_SIZE = 4096 - -TEST_MSG_TIMEOUT = 1 # type: int -TEST_HWM_MSG_NUM = 10 # type: int -TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int - -# define all encodings here -ENCODING = 'utf-8' -BYTES_LENGTH = 8 -BYTES_ORDER = 'big' - -# define all database names here -DATA_DB = "data_db".encode(ENCODING) # msg_id | data -PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber -META_DB = "meta_db".encode(ENCODING) # msg_id | metadata - - -def generate_test_config(path: pathlib.Path) -> Dict[str, Any]: - queue_dir = path / "queue" - queue_dir.mkdir(parents=True, exist_ok=True) - - test_config = { - "pub": { - "out_queue": queue_dir.as_posix(), - "subscribers": ["sub"] - }, - "sub": { - "in_queue": queue_dir.as_posix() - }, - "queues": { - queue_dir.as_posix(): { - "path": queue_dir.as_posix(), - "max_reader_num": 1024, - "max_db_num": 1024, - "max_db_size_bytes": 34359738368, - "subscribers": ["sub"], - "high-water-mark": { - "MSG_TIMEOUT_SECS": TEST_MSG_TIMEOUT, - "MAX_MSGS_NUM": TEST_HWM_MSG_NUM, - "HWM_LMDB_SIZE_BYTES": TEST_HWM_LMDB_SIZE, - "strategy": "prune_first" - } - }, - } - } - - return test_config - - -class TestPersiPubSub(unittest.TestCase): - def test_get_config(self): - with temppathlib.TemporaryDirectory() as tmp_dir: - expected_config = generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(expected_config, file_object) - - config = persipubsub.get_config(path=file) - - self.assertDictEqual(expected_config, config) - - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_control.py b/tests/test_control.py index 767f813..e6bd7c8 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -10,7 +10,7 @@ import persipubsub.control import persipubsub.queue -import tests.common +import tests # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -19,7 +19,7 @@ class TestControl(unittest.TestCase): def test_initialize_all(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -44,7 +44,7 @@ def test_initialize_all(self): def test_add_sub(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -72,7 +72,7 @@ def test_add_sub(self): def test_del_sub(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -99,7 +99,7 @@ def test_del_sub(self): def test_clear_all_subs(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -126,7 +126,7 @@ def test_clear_all_subs(self): def test_prune_dangling_messages(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -137,79 +137,70 @@ def test_prune_dangling_messages(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.msg_timeout_secs = tests.common.TEST_MSG_TIMEOUT + queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT with queue.env.begin(write=True) as txn: sub_db = queue.env.open_db( - key='sub'.encode(tests.common.ENCODING), - txn=txn, - create=False) + key='sub'.encode(tests.ENCODING), txn=txn, create=False) - txn.put( - key="timeout_msg".encode(tests.common.ENCODING), db=sub_db) - txn.put( - key="valid_msg".encode(tests.common.ENCODING), db=sub_db) + txn.put(key="timeout_msg".encode(tests.ENCODING), db=sub_db) + txn.put(key="valid_msg".encode(tests.ENCODING), db=sub_db) with queue.env.begin(write=True) as txn: data_db = queue.env.open_db( - key=tests.common.DATA_DB, txn=txn, create=False) + key=tests.DATA_DB, txn=txn, create=False) txn.put( - key="popped_msg".encode(tests.common.ENCODING), - value="I'm data".encode(tests.common.ENCODING), + key="popped_msg".encode(tests.ENCODING), + value="I'm data".encode(tests.ENCODING), db=data_db) txn.put( - key="timeout_msg".encode(tests.common.ENCODING), - value="I'm data too".encode(tests.common.ENCODING), + key="timeout_msg".encode(tests.ENCODING), + value="I'm data too".encode(tests.ENCODING), db=data_db) txn.put( - key="valid_msg".encode(tests.common.ENCODING), - value="Free me!".encode(tests.common.ENCODING), + key="valid_msg".encode(tests.ENCODING), + value="Free me!".encode(tests.ENCODING), db=data_db) with queue.env.begin(write=True) as txn: pending_db = queue.env.open_db( - key=tests.common.PENDING_DB, txn=txn, create=False) + key=tests.PENDING_DB, txn=txn, create=False) txn.put( - key="popped_msg".encode(tests.common.ENCODING), + key="popped_msg".encode(tests.ENCODING), value=int(0).to_bytes( - length=tests.common.BYTES_LENGTH, - byteorder=tests.common.BYTES_ORDER), + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=pending_db) txn.put( - key="timeout_msg".encode(tests.common.ENCODING), + key="timeout_msg".encode(tests.ENCODING), value=int(1).to_bytes( - length=tests.common.BYTES_LENGTH, - byteorder=tests.common.BYTES_ORDER), + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=pending_db) txn.put( - key="valid_msg".encode(tests.common.ENCODING), + key="valid_msg".encode(tests.ENCODING), value=int(1).to_bytes( - length=tests.common.BYTES_LENGTH, - byteorder=tests.common.BYTES_ORDER), + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=pending_db) with queue.env.begin(write=True) as txn: meta_db = queue.env.open_db( - key=tests.common.META_DB, txn=txn, create=False) + key=tests.META_DB, txn=txn, create=False) txn.put( - key="popped_msg".encode(tests.common.ENCODING), + key="popped_msg".encode(tests.ENCODING), value=int(datetime.datetime.utcnow().timestamp()).to_bytes( - length=tests.common.BYTES_LENGTH, - byteorder=tests.common.BYTES_ORDER), + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=meta_db) txn.put( - key="timeout_msg".encode(tests.common.ENCODING), + key="timeout_msg".encode(tests.ENCODING), value=int(datetime.datetime.utcnow().timestamp() - - tests.common.TEST_MSG_TIMEOUT - 5).to_bytes( - length=tests.common.BYTES_LENGTH, - byteorder=tests.common.BYTES_ORDER), + tests.TEST_MSG_TIMEOUT - 5).to_bytes( + length=tests.BYTES_LENGTH, + byteorder=tests.BYTES_ORDER), db=meta_db) txn.put( - key="valid_msg".encode(tests.common.ENCODING), + key="valid_msg".encode(tests.ENCODING), value=int(datetime.datetime.utcnow().timestamp()).to_bytes( - length=tests.common.BYTES_LENGTH, - byteorder=tests.common.BYTES_ORDER), + length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=meta_db) persipubsub.control.prune_dangling_messages(config_pth=file) @@ -232,7 +223,7 @@ def test_prune_dangling_messages(self): def test_prune_all_messages_for_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 72f2d00..6bc10ed 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -10,7 +10,7 @@ import persipubsub.control import persipubsub.publisher import persipubsub.queue -import tests.common +import tests # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -20,7 +20,7 @@ class TestPublisher(unittest.TestCase): def test_send(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -34,10 +34,10 @@ def test_send(self): pub = persipubsub.publisher.Pub() pub.init(pub_id="pub", config_pth=file) - msg = "Hello world!".encode(tests.common.ENCODING) + msg = "Hello world!".encode(tests.ENCODING) pub.send(msg=msg) - subscriber = "sub".encode(tests.common.ENCODING) + subscriber = "sub".encode(tests.ENCODING) with queue.env.begin(write=False) as txn: self.assertIsNotNone(txn.get(key=subscriber)) sub_db = queue.env.open_db( @@ -48,14 +48,14 @@ def test_send(self): key = cursor.key() data_db = queue.env.open_db( - key=tests.common.DATA_DB, txn=txn, create=False) + key=tests.DATA_DB, txn=txn, create=False) item = txn.get(key=key, db=data_db) self.assertIsNotNone(item) self.assertEqual(msg, item) def test_send_to_nonexisting_db(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -71,13 +71,13 @@ def test_send_to_nonexisting_db(self): pub.sub_list.append('non-existent sub') - msg = "message for every subscriber".encode(tests.common.ENCODING) + msg = "message for every subscriber".encode(tests.ENCODING) self.assertRaises(lmdb.NotFoundError, pub.send, msg=msg) def test_send_many(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -89,7 +89,7 @@ def test_send_many(self): pub = persipubsub.publisher.Pub() pub.init(pub_id='pub', config_pth=file) - msg = "I'm a message".encode(tests.common.ENCODING) + msg = "I'm a message".encode(tests.ENCODING) msgs = [] msg_num = 10 for _ in range(msg_num): @@ -101,10 +101,10 @@ def test_send_many(self): with pub.queue.env.begin(write=False) as txn: self.assertIsNotNone( - txn.get(key=subscriber.encode(tests.common.ENCODING))) + txn.get(key=subscriber.encode(tests.ENCODING))) sub_db = pub.queue.env.open_db( - key=subscriber.encode(tests.common.ENCODING), + key=subscriber.encode(tests.ENCODING), txn=txn, create=False) @@ -112,7 +112,7 @@ def test_send_many(self): self.assertEqual(msg_num, sub_stat['entries']) data_db = pub.queue.env.open_db( - key=tests.common.DATA_DB, txn=txn, create=False) + key=tests.DATA_DB, txn=txn, create=False) data_stat = txn.stat(db=data_db) self.assertEqual(msg_num, data_stat['entries']) diff --git a/tests/test_queue.py b/tests/test_queue.py index 449f57c..3c7c0aa 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -9,7 +9,7 @@ import persipubsub.control import persipubsub.queue -import tests.common +import tests # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -40,11 +40,11 @@ def test_initialize_environment(self): def test_put_to_single_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.common.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -58,10 +58,10 @@ def test_put_to_single_subscriber(self): with queue.env.begin() as txn: self.assertIsNotNone( - txn.get(key=subscriber.encode(tests.common.ENCODING))) + txn.get(key=subscriber.encode(tests.ENCODING))) sub_db = queue.env.open_db( - key=subscriber.encode(tests.common.ENCODING), + key=subscriber.encode(tests.ENCODING), txn=txn, create=False) cursor = txn.cursor(db=sub_db) @@ -69,7 +69,7 @@ def test_put_to_single_subscriber(self): key = cursor.key() data_db = queue.env.open_db( - key=tests.common.DATA_DB, txn=txn, create=False) + key=tests.DATA_DB, txn=txn, create=False) value = txn.get(key=key, db=data_db) self.assertIsNotNone(value) @@ -78,11 +78,11 @@ def test_put_to_single_subscriber(self): def test_put_multiple_subscriber(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.common.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) sub_list = ["sub", "another_sub"] - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -99,10 +99,10 @@ def test_put_multiple_subscriber(self): with queue.env.begin() as txn: self.assertIsNotNone( - txn.get(key=sub_list[0].encode(tests.common.ENCODING))) + txn.get(key=sub_list[0].encode(tests.ENCODING))) sub_db_0 = queue.env.open_db( - key=sub_list[0].encode(tests.common.ENCODING), + key=sub_list[0].encode(tests.ENCODING), txn=txn, create=False) cursor = txn.cursor(db=sub_db_0) @@ -110,10 +110,10 @@ def test_put_multiple_subscriber(self): key_0 = cursor.key() self.assertIsNotNone( - txn.get(key=sub_list[1].encode(tests.common.ENCODING))) + txn.get(key=sub_list[1].encode(tests.ENCODING))) sub_db_1 = queue.env.open_db( - key=sub_list[1].encode(tests.common.ENCODING), + key=sub_list[1].encode(tests.ENCODING), txn=txn, create=False) cursor = txn.cursor(db=sub_db_1) @@ -123,7 +123,7 @@ def test_put_multiple_subscriber(self): self.assertEqual(key_0, key_1) data_db = queue.env.open_db( - key=tests.common.DATA_DB, txn=txn, create=False) + key=tests.DATA_DB, txn=txn, create=False) value = txn.get(key=key_1, db=data_db) self.assertIsNotNone(value) @@ -132,7 +132,7 @@ def test_put_multiple_subscriber(self): def test_put_many(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -143,7 +143,7 @@ def test_put_many(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - msg = "I'm a message".encode(tests.common.ENCODING) + msg = "I'm a message".encode(tests.ENCODING) msgs = [] msg_num = 10 for _ in range(msg_num): @@ -155,10 +155,10 @@ def test_put_many(self): with queue.env.begin(write=False) as txn: self.assertIsNotNone( - txn.get(key=subscriber.encode(tests.common.ENCODING))) + txn.get(key=subscriber.encode(tests.ENCODING))) sub_db = queue.env.open_db( - key=subscriber.encode(tests.common.ENCODING), + key=subscriber.encode(tests.ENCODING), txn=txn, create=False) @@ -166,18 +166,18 @@ def test_put_many(self): self.assertEqual(msg_num, sub_stat['entries']) data_db = queue.env.open_db( - key=tests.common.DATA_DB, txn=txn, create=False) + key=tests.DATA_DB, txn=txn, create=False) data_stat = txn.stat(db=data_db) self.assertEqual(msg_num, data_stat['entries']) def test_front(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.common.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -198,11 +198,11 @@ def test_front(self): def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - msg = "I'm a message.".encode(tests.common.ENCODING) + msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -222,7 +222,7 @@ def test_pop(self): with queue.env.begin() as txn: pending_db = queue.env.open_db( - key=tests.common.PENDING_DB, txn=txn, create=False) + key=tests.PENDING_DB, txn=txn, create=False) cursor = txn.cursor(db=pending_db) self.assertTrue(cursor.first()) @@ -235,21 +235,21 @@ def test_pop(self): with queue.env.begin() as txn: pending_db = queue.env.open_db( - key=tests.common.PENDING_DB, txn=txn, create=False) + key=tests.PENDING_DB, txn=txn, create=False) cursor = txn.cursor(db=pending_db) self.assertTrue(cursor.first()) pending_after_pop = cursor.value() self.assertEqual( - int.from_bytes(pending_before_pop, tests.common.BYTES_ORDER) - - 1, int.from_bytes(pending_after_pop, tests.common.BYTES_ORDER)) + int.from_bytes(pending_before_pop, tests.BYTES_ORDER) - 1, + int.from_bytes(pending_after_pop, tests.BYTES_ORDER)) def test_pop_queue_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: subscriber = "sub" - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -265,7 +265,7 @@ def test_pop_queue_empty(self): def test_queue_initialisation(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -292,7 +292,7 @@ def test_queue_initialisation(self): def test_overflow_msgs_limit(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -305,25 +305,25 @@ def test_overflow_msgs_limit(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.max_msgs_num = tests.common.TEST_HWM_MSG_NUM + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM - msg = "hello world".encode(tests.common.ENCODING) + msg = "hello world".encode(tests.ENCODING) self.assertEqual(0, queue.count_msgs()) - for _ in range(tests.common.TEST_HWM_MSG_NUM): + for _ in range(tests.TEST_HWM_MSG_NUM): queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual(tests.common.TEST_HWM_MSG_NUM, queue.count_msgs()) + self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.count_msgs()) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual( - int(tests.common.TEST_HWM_MSG_NUM - - int(tests.common.TEST_HWM_MSG_NUM / 2)), queue.count_msgs()) + int(tests.TEST_HWM_MSG_NUM - int(tests.TEST_HWM_MSG_NUM / 2)), + queue.count_msgs()) def test_overflow_limit_size(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -336,26 +336,24 @@ def test_overflow_limit_size(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.hwm_lmdb_size_bytes = tests.common.TEST_HWM_LMDB_SIZE + queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE - msg = ("a" * (int(tests.common.LMDB_PAGE_SIZE / 4))).encode( - tests.common.ENCODING) + msg = ("a" * (int(tests.LMDB_PAGE_SIZE / 4))).encode(tests.ENCODING) - while queue.check_current_lmdb_size( - ) <= tests.common.TEST_HWM_LMDB_SIZE: + while queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE: queue.put(msg=msg, sub_list=queue.sub_list) - self.assertTrue(queue.check_current_lmdb_size() > tests.common. - TEST_HWM_LMDB_SIZE) + self.assertTrue( + queue.check_current_lmdb_size() > tests.TEST_HWM_LMDB_SIZE) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertTrue(queue.check_current_lmdb_size() <= tests.common. - TEST_HWM_LMDB_SIZE) + self.assertTrue( + queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE) def test_timeout(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -367,21 +365,21 @@ def test_timeout(self): queue = persipubsub.queue._Queue() queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.msg_timeout_secs = tests.common.TEST_MSG_TIMEOUT + queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT - msg = "hello world".encode(tests.common.ENCODING) + msg = "hello world".encode(tests.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual(1, queue.count_msgs()) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual(2, queue.count_msgs()) - time.sleep(tests.common.TEST_MSG_TIMEOUT + 1) + time.sleep(tests.TEST_MSG_TIMEOUT + 1) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual(1, queue.count_msgs()) def test_strategy_prune_first(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -395,28 +393,27 @@ def test_strategy_prune_first(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.strategy = persipubsub.queue._Strategy.prune_first - queue.hwm.max_msgs_num = tests.common.TEST_HWM_MSG_NUM + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM - for index in range(tests.common.TEST_HWM_MSG_NUM): - msg = "secret message {}".format(index).encode( - tests.common.ENCODING) + for index in range(tests.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode(tests.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual("secret message 0".encode(tests.common.ENCODING), + self.assertEqual("secret message 0".encode(tests.ENCODING), queue.front(sub_id='sub')) - msg = "secret message {}".format( - tests.common.TEST_HWM_MSG_NUM).encode(tests.common.ENCODING) + msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( + tests.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) self.assertEqual( "secret message {}".format( - int((tests.common.TEST_HWM_MSG_NUM / 2) + 1)).encode( - tests.common.ENCODING), queue.front(sub_id='sub')) + int((tests.TEST_HWM_MSG_NUM / 2) + 1)).encode( + tests.ENCODING), queue.front(sub_id='sub')) def test_strategy_prune_last(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -430,21 +427,20 @@ def test_strategy_prune_last(self): queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") queue.strategy = persipubsub.queue._Strategy.prune_last - queue.hwm.max_msgs_num = tests.common.TEST_HWM_MSG_NUM + queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM - for index in range(tests.common.TEST_HWM_MSG_NUM): - msg = "secret message {}".format(index).encode( - tests.common.ENCODING) + for index in range(tests.TEST_HWM_MSG_NUM): + msg = "secret message {}".format(index).encode(tests.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual("secret message 0".encode(tests.common.ENCODING), + self.assertEqual("secret message 0".encode(tests.ENCODING), queue.front(sub_id='sub')) - msg = "secret message {}".format( - tests.common.TEST_HWM_MSG_NUM).encode(tests.common.ENCODING) + msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( + tests.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) - self.assertEqual("secret message 0".encode(tests.common.ENCODING), + self.assertEqual("secret message 0".encode(tests.ENCODING), queue.front(sub_id='sub')) diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index a29c2dc..cc32a1b 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -9,7 +9,7 @@ import persipubsub.control import persipubsub.queue import persipubsub.subscriber -import tests.common +import tests # pylint: disable=missing-docstring # pylint: disable=protected-access @@ -18,7 +18,7 @@ class TestSubscriber(unittest.TestCase): def test_receive_message(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -32,7 +32,7 @@ def test_receive_message(self): sub = persipubsub.subscriber.Sub() sub.init(sub_id="sub", config_pth=file) - msg = "Hello World!".encode(tests.common.ENCODING) + msg = "Hello World!".encode(tests.ENCODING) queue.put(msg=msg, sub_list=["sub"]) with sub.receive(timeout=1) as received_msg: @@ -41,7 +41,7 @@ def test_receive_message(self): def test_timeout_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -58,25 +58,25 @@ def test_timeout_subscriber(self): with sub.receive(timeout=1) as received_msg: self.assertIsNone(received_msg) msg = "message send after timeout and will not be popped" \ - "".encode(tests.common.ENCODING) + "".encode(tests.ENCODING) queue.put(msg=msg, sub_list=queue.sub_list) self.assertIsNone(received_msg) self.assertEqual( "message send after timeout and will not be popped".encode( - tests.common.ENCODING), queue.front(sub_id='sub')) + tests.ENCODING), queue.front(sub_id='sub')) with sub.receive(timeout=1) as received_msg: self.assertIsNotNone(received_msg) self.assertEqual( "message send after timeout and will not be popped".encode( - tests.common.ENCODING), received_msg) + tests.ENCODING), received_msg) self.assertIsNone(queue.front(sub_id='sub')) def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" @@ -90,10 +90,10 @@ def test_pop(self): sub = persipubsub.subscriber.Sub() sub.init(sub_id='sub', config_pth=file) - msg1 = "I'm a message".encode(tests.common.ENCODING) + msg1 = "I'm a message".encode(tests.ENCODING) queue.put(msg=msg1, sub_list=queue.sub_list) - msg2 = "I'm a message too".encode(tests.common.ENCODING) + msg2 = "I'm a message too".encode(tests.ENCODING) queue.put(msg=msg2, sub_list=queue.sub_list) sub._pop() @@ -104,7 +104,7 @@ def test_pop(self): def test_pop_when_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.common.generate_test_config(path=tmp_dir.path) + config = tests.generate_test_config(path=tmp_dir.path) file = tmp_dir.path / "config.json" From a30c91b0f9fd6e982d9a1f4c706f8d14b0c594ab Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 30 Jan 2019 14:06:14 +0100 Subject: [PATCH 19/35] add isort config file --- .isort.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..822d029 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,2 @@ +[settings] +known_first_party=persipubsub, tests \ No newline at end of file From 62ddb553983b11cf427cfaca7083488cd75dd7a6 Mon Sep 17 00:00:00 2001 From: slimeth Date: Thu, 31 Jan 2019 11:10:33 +0100 Subject: [PATCH 20/35] adjust README (review) --- README.rst | 85 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/README.rst b/README.rst index 9628bdf..a0c7b48 100644 --- a/README.rst +++ b/README.rst @@ -22,14 +22,14 @@ persipubsub Primarily, we used `zeromq `_ for inter-process communication with a slight improvement through `persizmq -`_. This still didn't fulfill the level +`_. This still did not fulfill the level of persistence we wanted. Our motivation was to replace our previous library with a one which is similarly easy to setup and to use. Additionally, it should make it possible to send `protobuf `_ messages -more precisely bytes persistently, thread-safely and process-safely from -many publishers to many subscribers. +(bytes) thread-safely and process-safely from many publishers to many +subscribers. Besides basic publisher and subscriber classes the library offers control methods for easy deployment from a config JSON file and maintenance in case @@ -41,52 +41,65 @@ Related projects persist-queue ------------- -* Offers not all functionality expected from a queue. Has put and get function - which are basically only push and pop. So ``front`` functionality is missing. - So neither can multiple subscriber be implemented nor can't be guaranteed that - no data is lost when a thread fails. -* All messages in queues are serialized by ``pickle`` which was for us a reason - against using this library. +* The library offers not all functionality expected from a queue. It has put + and get function which are basically only push and pop. Therefore ``front`` + functionality is missing. In consequence neither can the queue have multiple + subscribers nor can be guaranteed that no data is lost when a thread fails. +* All messages in queues are serialized by ``pickle`` which was for us the main + reason not to use this library. Kafka ----- -* For our needs as local system with IPC it's an unnecessary overhead. -* Integration of Kafka written in Scala and Java in our C++/Python/Go codebase - not that simple. -* Hard to setup and maintain `Kafka `_. + +* Hence we only need Inter Process Communication, the TCP ability of `Kafka + `_ is an unnecessary overhead. +* Integration of ``Kafka`` written in Scala and Java in our C++/Python/Go + codebase is challenging. +* Hard to setup and maintain ``Kafka``. * broker system eventually a bottleneck. RabbitMQ -------- -* For our needs as local system with IPC it's an unnecessary overhead. + +* Hence we only need Inter Process Communication, the TCP ability of `RabbitMQ + `_ is an unnecessary overhead. * broker system eventually a bottleneck. -* `RabbitMQ `_ is less scalable than Kafka, but is - supported officially by more languages. +* ``RabbitMQ`` is less scalable than ``Kafka``, but + is supported officially by more languages. zeromq persistence pattern -------------------------- -* Only persistence pattern of `zeromq `_ is - `Titanic `_ which is also a broker - system. This takes away the purpose and advantage of zeromq to be a + +* `Titanic `_ is the only persistence + pattern of `zeromq `_ which is also a broker system. + This takes away the purpose and advantage of ``zeromq`` to be a lightweight library which requires no broker. Usage ===== -The usage of the library can be differentiated into deployment and actual -runtime of your processes. +The usage of the library consists of two steps: deployment and runtime Deployment ---------- -For deployment only a proper config JSON file is needed to setup the whole -queue structure. +In the deployment stage the library sets up the queue structure which is saved +in a JSON file. config.json ^^^^^^^^^^^ + Publishers, subscribers and queues need to be defined before runtime in the config JSON file. +.. note:: + + The high water marks are limits for the queue. The message is deleted in + case that it reaches the timeout. In the other case of an overflow one + of two strategies is used to prune half of the queue. The choice is between + prune_first, which deletes the oldest messages, and prune_last, which + deletes the latest messages. + .. code-block:: json { @@ -117,6 +130,8 @@ config JSON file. Control ^^^^^^^ +A control unit to initialize and maintain queues. + Initialize all queues """"""""""""""""""""" @@ -148,9 +163,15 @@ Clear all messages Runtime ------- -During runtime only publisher and subscriber are needed. Control can be -optionally be used for pruning although the queues prune itself on a regular -basis. + +During runtime only publisher and subscriber are needed. + +.. note:: + + Control can be optionally be used for pruning although the queues prune + itself on a regular basis when high water mark is reached. The high water + mark includes a timeout, maximum number of messages and the maximum bytes + size of the queue. Publisher ^^^^^^^^^ @@ -187,8 +208,8 @@ Send many messages at once "Do you like the README?".encode('utf-8')] pub.send_many(msgs=msgs) - # Subscribers have now both messages in the queue not necessary in the list - # order. + # Both messages are now available for the subscribers. Note that the order + # of the messages are not necessarily kept. Subscriber ^^^^^^^^^^ @@ -196,7 +217,7 @@ Subscriber Initialization """""""""""""" -Assuming that all queues were initialized during deployment the publisher can +Assuming that all queues were initialized during deployment the subscriber can be initialized as following. .. code-block:: python @@ -221,8 +242,8 @@ Receive a message Catch up with latest message """""""""""""""""""""""""""" -In case your subscriber is slow, the queue is growing and you want to want to -receive the latest messages. +In case when the subscriber's loop is spinning slower then the publisher's, +there is a possibility receive the latest message and discard the others ones. .. code-block:: python @@ -236,6 +257,7 @@ receive the latest messages. Documentation ============= + The documentation is available on `readthedocs `_. @@ -312,6 +334,7 @@ development dependencies: Versioning ========== + We follow `Semantic Versioning `_. The version X.Y.Z indicates: From 94c9d364f3bfff09979027b85ce7af0784d1d615 Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 5 Feb 2019 10:53:56 +0100 Subject: [PATCH 21/35] removed config file --- persipubsub/__init__.py | 75 ++++-- persipubsub/control.py | 464 +++++++++++++++++++++++++------------ persipubsub/environment.py | 66 ++++++ persipubsub/publisher.py | 40 ++-- persipubsub/queue.py | 146 +++++++----- persipubsub/subscriber.py | 43 ++-- tests/__init__.py | 62 ++--- tests/test_control.py | 166 +++++-------- tests/test_environment.py | 50 ++++ tests/test_publisher.py | 73 +++--- tests/test_queue.py | 213 ++++++----------- tests/test_subscriber.py | 85 ++++--- 12 files changed, 837 insertions(+), 646 deletions(-) create mode 100644 persipubsub/environment.py create mode 100644 tests/test_environment.py diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py index 30d30ee..5d73d8a 100644 --- a/persipubsub/__init__.py +++ b/persipubsub/__init__.py @@ -2,17 +2,35 @@ """Distribute messages persistent from publisher to subscriber.""" import json import pathlib -from typing import Dict, Union +from typing import Any, Dict, Union + +import lmdb # define all encodings here -ENCODING = 'utf-8' -BYTES_LENGTH = 8 -BYTES_ORDER = 'big' +ENCODING = 'utf-8' # type: str +BYTES_LENGTH = 8 # type: int +BYTES_ORDER = 'big' # type: str # define all database names here DATA_DB = "data_db".encode(ENCODING) # msg_id | data PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber META_DB = "meta_db".encode(ENCODING) # msg_id | metadata +QUEUE_DB = "queue_db".encode(ENCODING) # queue_pth | all queue data + +# queues default +MAX_DB_SIZE_BYTES = 32 * 1024**3 # type: int +MAX_READER_NUM = 1024 # type: int +MAX_DB_NUM = 1024 # type: int + +# queues default keys +MAX_DB_SIZE_BYTES_KEY = "max_db_size_bytes".encode(ENCODING) +MAX_READER_NUM_KEY = "max_reader_num".encode(ENCODING) +MAX_DB_NUM_KEY = "max_db_num".encode(ENCODING) +HWM_DB_SIZE_BYTES_KEY = "hwm_db_size_bytes".encode(ENCODING) +MAX_MSGS_NUM_KEY = "max_msgs_num".encode(ENCODING) +MSG_TIMEOUT_SECS_KEY = "msg_timeout_secs".encode(ENCODING) +STRATEGY_KEY = "strategy".encode(ENCODING) +SUBSCRIBER_IDS_KEY = "subscriber_ids".encode(ENCODING) def encoding(string: str) -> bytes: @@ -45,19 +63,46 @@ def bytes_to_int(array_of_bytes: bytes) -> int: return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) -def get_config(path: Union[pathlib.Path, str]) -> Dict[str, Dict[str, str]]: +def get_queue_data(path: pathlib.Path, key: bytes) -> bytes: """ - Get queues, publishers and subscribers data. + Get queue data. - :param path: path to the JSON config file - :return config data with queues, publishers and subscribers + :param path: to the queue + :param key: for lookup + :return: lookup result """ - pth = path if isinstance(path, pathlib.Path) else pathlib.Path(path) + env = lmdb.open( + path=path.as_posix(), + map_size=MAX_DB_SIZE_BYTES, + subdir=True, + max_readers=MAX_READER_NUM, + max_dbs=MAX_READER_NUM, + max_spare_txns=0) + + with env.begin(write=False) as txn: + queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) + data = txn.get(key=key, db=queue_db) - if not pth.is_file(): - raise FileNotFoundError("Config file at path '{}' not found".format( - pth.as_posix())) + return data - with open(pth.as_posix(), mode="rt") as file_object: - config = json.load(file_object) - return config + +def put_queue_data(path: pathlib.Path, key: bytes, value: bytes) -> None: + """ + Get queue data. + + :param path: to the queue + :param key: of value to store + :param value: to store + :return: + """ + env = lmdb.open( + path=path, + map_size=MAX_DB_SIZE_BYTES, + subdir=True, + max_readers=MAX_READER_NUM, + max_dbs=MAX_READER_NUM, + max_spare_txns=0) + + with env.begin(write=True) as txn: + queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) + txn.put(key=key, value=value, db=queue_db) diff --git a/persipubsub/control.py b/persipubsub/control.py index 5412532..af504ce 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -1,185 +1,343 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Setup and control persistent distribution of messages.""" import pathlib -from typing import Any, Dict, List, Union +from typing import List, Optional, Sequence, Set + +import icontract +import lmdb # pylint: disable=unused-import import persipubsub.queue # pylint: disable=protected-access -def initialize_all_dbs(config_pth: Union[pathlib.Path, str]) -> None: +def set_queue_parameters(max_reader_num: int, max_db_num: int, + max_db_size_bytes: int, env: lmdb.Environment) -> None: """ - Initialize all needed queues. + Store queue parameter in lmdb. - :param config_pth: path to the JSON config file + :param max_reader_num: max number of reader of the lmdb + :param max_db_num: max number of named databases + :param max_db_size_bytes: max size of the lmdb in bytes + :return: """ - config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] - queues = config["queues"] # type: Dict[str, Any] - for values in queues.values(): - queue_dir = pathlib.Path(values["path"]) - if not queue_dir.exists(): - queue_dir.mkdir(parents=True, exist_ok=True) - sub_list = values["subscribers"] - queue = persipubsub.queue._Queue() - queue.init( - config_pth=config_pth, - queue_dir=queue_dir, - max_reader_num=values["max_reader_num"], - max_db_num=values["max_db_num"], - max_db_size_bytes=values["max_db_size_bytes"]) - - for sub in sub_list: - _add_sub(sub_id=sub, queue=queue) - - -def clear_all_subs(config_pth: Union[pathlib.Path, str]) -> None: + with env.begin(write=True) as txn: + queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.MAX_READER_NUM_KEY, + value=persipubsub.int_to_bytes(max_reader_num), + db=queue_db) + txn.put( + key=persipubsub.MAX_DB_NUM_KEY, + value=persipubsub.int_to_bytes(max_db_num), + db=queue_db) + txn.put( + key=persipubsub.MAX_DB_SIZE_BYTES_KEY, + value=persipubsub.int_to_bytes(max_db_size_bytes), + db=queue_db) + + +def set_hwm(hwm: persipubsub.queue._HighWaterMark, + env: lmdb.Environment) -> None: """ - Clear all subscriber and delete all messages. + Set high water mark values for queue. - :param config_pth: path to the JSON config file + :param hwm: high water mark values + :return: """ - config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] - queues = config["queues"] # type: Dict[str, Any] - for values in queues.values(): - queue_dir = pathlib.Path(values["path"]) - if not queue_dir.exists(): - queue_dir.mkdir(parents=True, exist_ok=True) - sub_list = values["subscribers"] # type: List[str] - queue = persipubsub.queue._Queue() - queue.init( - config_pth=config_pth, - queue_dir=queue_dir, - max_reader_num=values["max_reader_num"], - max_db_num=values["max_db_num"], - max_db_size_bytes=values["max_db_size_bytes"]) - _clear_all_subs_for(queue=queue, sub_list=sub_list) - - -def _clear_all_subs_for(queue: persipubsub.queue._Queue, - sub_list: List[str]) -> None: + with env.begin(write=True) as txn: + queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, + value=persipubsub.int_to_bytes(hwm.hwm_lmdb_size_bytes), + db=queue_db) + txn.put( + key=persipubsub.MAX_MSGS_NUM_KEY, + value=persipubsub.int_to_bytes(hwm.max_msgs_num), + db=queue_db) + txn.put( + key=persipubsub.MSG_TIMEOUT_SECS_KEY, + value=persipubsub.int_to_bytes(hwm.msg_timeout_secs), + db=queue_db) + + +def set_strategy(strategy: persipubsub.queue._Strategy, + env: lmdb.Environment) -> None: """ - Clear all subscriber and delete all messages for one queue. + Set pruning strategy for queue. - :param queue: which will be cleared - :param sub_list: subscribers which will be removed + :param strategy: pruning strategy + :return: """ - with queue.env.begin(write=True) as txn: - for sub_id in sub_list: - sub_db = queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) - txn.drop(db=sub_db) - - pending_db = queue.env.open_db( - key=persipubsub.PENDING_DB, txn=txn, create=False) - txn.drop(db=pending_db, delete=False) - meta_db = queue.env.open_db( - key=persipubsub.META_DB, txn=txn, create=False) - txn.drop(db=meta_db, delete=False) - data_db = queue.env.open_db( - key=persipubsub.DATA_DB, txn=txn, create=False) - txn.drop(db=data_db, delete=False) + with env.begin(write=True) as txn: + queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.STRATEGY_KEY, + value=persipubsub.encoding(str(strategy.name)), + db=queue_db) -def _add_sub(sub_id: str, queue: persipubsub.queue._Queue) -> None: +@icontract.require(lambda sub_id: sub_id.find(' ') == -1) +def _add_sub(sub_id: str, env: lmdb.Environment) -> None: """ Add a subscriber and create its lmdb. :param sub_id: ID of the subscriber which should be added - :param queue: to which the subscriber is added - """ - with queue.env.begin(write=True) as txn: - _ = queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=True) - - -def _remove_sub(sub_id: str, queue: persipubsub.queue._Queue) -> None: - """ - Remove a subscriber and delete all its messages. - - :param sub_id: ID of the subscriber which should be removed - :param queue: from which the subscriber is removed """ - msg_of_sub = set() - with queue.env.begin(write=True) as txn: - sub_db = queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) - cursor = txn.cursor(db=sub_db) - # check if database is not empty - if cursor.first(): - for key in cursor.iternext(keys=True, values=False): - msg_of_sub.add(key) - txn.drop(db=sub_db) - - pending_db = queue.env.open_db( - key=persipubsub.PENDING_DB, txn=txn, create=False) - - for key in msg_of_sub: - pending_value = txn.get(key=key, db=pending_db) - pending_num = persipubsub.bytes_to_int(pending_value) - decreased_pending_num = pending_num - 1 - assert decreased_pending_num >= 0 - txn.put( - key=key, - value=persipubsub.int_to_bytes(decreased_pending_num), - db=pending_db) + with env.begin(write=True) as txn: + _ = env.open_db(key=persipubsub.encoding(sub_id), txn=txn, create=True) + + queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=False) + subscriber_ids = txn.get( + key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) + + if subscriber_ids is None: + subscriber_list = [] # type: List[str] + else: + subscriber_list = subscriber_ids.decode( + persipubsub.ENCODING).split(' ') + subscriber_set = set(subscriber_list) + subscriber_set.add(sub_id) + subscriber_str = " ".join(subscriber_set) + txn.put( + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=subscriber_str.encode(persipubsub.ENCODING), + db=queue_db) + + +class Control: + """Control and maintain one queue.""" + + def __init__(self, path: pathlib.Path): + """ + Initialize control class. + + :param path: to the queue. + """ + self.path = path + self.queue = None # type: Optional[persipubsub.queue._Queue] + self.subscriber_ids = set() # type: Set[str] + + # pylint: disable=too-many-arguments + def init(self, + subscriber_ids: Optional[Sequence[str]] = None, + max_readers: int = 1024, + max_size: int = 32 * 1024**3, + high_watermark: persipubsub.queue._HighWaterMark = persipubsub. + queue._HighWaterMark(), + strategy: persipubsub.queue._Strategy = persipubsub.queue. + _Strategy.prune_first) -> None: + """ + Initialize control with a (re)initialized queue. + + :param subscriber_ids: subscribers of the queue + :param max_readers: max number of reader of the lmdb + :param max_size: max size of the lmdb in bytes + :param high_watermark: high water mark limit of the queue + :return: + """ + if self.check_queue_is_initialized(): + self.reinitialize_queue() + else: + if subscriber_ids is None: + subscriber_ids = [] + assert isinstance(subscriber_ids, Sequence) + self.initialize_queue( + subscriber_ids=subscriber_ids, + max_readers=max_readers, + max_size=max_size, + high_watermark=high_watermark, + strategy=strategy) + + def reinitialize_queue(self) -> None: + """Reinitialize the queue which is maintained by the control.""" + self.queue = persipubsub.queue._Queue() + self.queue.init(path=self.path) + assert isinstance(self.queue.sub_list, List) + self.subscriber_ids = set(self.queue.sub_list) + + # pylint: disable=too-many-arguments + def initialize_queue(self, + subscriber_ids: Sequence[str], + max_readers: int = 1024, + max_size: int = 32 * 1024**3, + high_watermark: persipubsub.queue. + _HighWaterMark = persipubsub.queue._HighWaterMark(), + strategy: persipubsub.queue._Strategy = persipubsub. + queue._Strategy.prune_first) -> None: + """ + Initialize queue. + + :param subscriber_ids: subscribers of the queue + :param max_readers: max number of reader of the lmdb + :param max_size: max size of the lmdb in bytes + :param high_watermark: high water mark limit of the queue + :return: + """ + if not self.path.exists(): + self.path.mkdir(parents=True, exist_ok=True) + + self.subscriber_ids = set(subscriber_ids) + # Databases needed for queue: + # 5 queues (main db, data db, meta db, pending db, queue db) + # + each subscriber has its own db + # TODO(snaji): should it be max(len(sub_ids), max_readers)? + max_db_num = 5 + len(self.subscriber_ids) + + env = persipubsub.queue._initialize_environment( + queue_dir=self.path, + max_db_num=max_db_num, + max_reader_num=max_readers, + max_db_size_bytes=max_size) + + set_queue_parameters( + max_reader_num=max_readers, + max_db_num=max_db_num, + max_db_size_bytes=max_size, + env=env) + + set_hwm(hwm=high_watermark, env=env) + set_strategy(strategy=strategy, env=env) + + for sub in self.subscriber_ids: + _add_sub(sub_id=sub, env=env) + + # load initialized queue + self.queue = persipubsub.queue._Queue() + self.queue.init(path=self.path) + + def check_queue_is_initialized(self) -> bool: + """ + Check if queue is initialized. + + :return is initialized when all values for the given keys are set + """ + keys = [ + persipubsub.MAX_DB_SIZE_BYTES_KEY, persipubsub.MAX_DB_NUM_KEY, + persipubsub.MSG_TIMEOUT_SECS_KEY, persipubsub.MAX_MSGS_NUM_KEY, + persipubsub.HWM_DB_SIZE_BYTES_KEY, persipubsub.STRATEGY_KEY, + persipubsub.SUBSCRIBER_IDS_KEY, persipubsub.MAX_READER_NUM_KEY + ] # type: List[bytes] + + for key in keys: + + try: + value = persipubsub.get_queue_data(path=self.path, key=key) + except lmdb.NotFoundError: + return False + + if value is None: + return False + + return True + + def clear_all_subscribers(self) -> None: + """Clear all subscriber and delete all messages for queue.""" + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.queue.env, lmdb.Environment) + with self.queue.env.begin(write=True) as txn: + for sub_id in self.subscriber_ids: + sub_db = self.queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + txn.drop(db=sub_db) + + pending_db = self.queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + txn.drop(db=pending_db, delete=False) + meta_db = self.queue.env.open_db( + key=persipubsub.META_DB, txn=txn, create=False) + txn.drop(db=meta_db, delete=False) + data_db = self.queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn, create=False) + txn.drop(db=data_db, delete=False) + + def prune_dangling_messages(self) -> None: + """Prune all dangling messages from the lmdb.""" + assert isinstance(self.queue, persipubsub.queue._Queue) + persipubsub.queue._prune_dangling_messages_for( + queue=self.queue, sub_list=list(self.subscriber_ids)) + + # pylint: disable=too-many-locals + def _prune_all_messages_for(self, sub_id: str) -> None: + """ + Prune all messages of a subscriber. + + :param sub_id: ID of the subscriber of which all messages should be + pruned + :param queue: of which some subscriber's messages should be pruned + """ + msg_of_sub = set() + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.queue.env, lmdb.Environment) + with self.queue.env.begin(write=True) as txn: + sub_db = self.queue.env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + for key in cursor.iternext(keys=True, values=False): + msg_of_sub.add(key) -def prune_dangling_messages(config_pth: Union[pathlib.Path, str]) -> None: - """ - Prune all dangling messages from the lmdb. + txn.drop(db=sub_db, delete=False) - :param config_pth: path to the JSON config file - """ - config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] - queues = config["queues"] # type: Dict[str, Any] - for values in queues.values(): - queue_dir = pathlib.Path(values["path"]) - if not queue_dir.exists(): - queue_dir.mkdir(parents=True, exist_ok=True) - sub_list = values["subscribers"] # type: List[str] - queue = persipubsub.queue._Queue() - queue.init( - config_pth=config_pth, - queue_dir=queue_dir, - max_reader_num=values["max_reader_num"], - max_db_num=values["max_db_num"], - max_db_size_bytes=values["max_db_size_bytes"]) - persipubsub.queue._prune_dangling_messages_for( - queue=queue, sub_list=sub_list) + pending_db = self.queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) + for key in msg_of_sub: + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + + def _remove_sub(self, sub_id: str, env: lmdb.Environment) -> None: + """ + Remove a subscriber and delete all its messages. + + :param sub_id: ID of the subscriber which should be removed + :param queue: from which the subscriber is removed + """ + msg_of_sub = set() + with env.begin(write=True) as txn: + sub_db = env.open_db( + key=persipubsub.encoding(sub_id), txn=txn, create=False) + cursor = txn.cursor(db=sub_db) + # check if database is not empty + if cursor.first(): + for key in cursor.iternext(keys=True, values=False): + msg_of_sub.add(key) + txn.drop(db=sub_db) -def _prune_all_messages_for(sub_id: str, - queue: persipubsub.queue._Queue) -> None: - """ - Prune all messages of a subscriber. + pending_db = env.open_db( + key=persipubsub.PENDING_DB, txn=txn, create=False) - :param sub_id: ID of the subscriber of which all messages should be pruned - :param queue: of which some subscriber's messages should be pruned - """ - msg_of_sub = set() - with queue.env.begin(write=True) as txn: - sub_db = queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) - - cursor = txn.cursor(db=sub_db) - # check if database is not empty - if cursor.first(): - for key in cursor.iternext(keys=True, values=False): - msg_of_sub.add(key) - - txn.drop(db=sub_db, delete=False) - - pending_db = queue.env.open_db( - key=persipubsub.PENDING_DB, txn=txn, create=False) - - for key in msg_of_sub: - pending_value = txn.get(key=key, db=pending_db) - pending_num = persipubsub.bytes_to_int(pending_value) - decreased_pending_num = pending_num - 1 - assert decreased_pending_num >= 0 + for key in msg_of_sub: + pending_value = txn.get(key=key, db=pending_db) + pending_num = persipubsub.bytes_to_int(pending_value) + decreased_pending_num = pending_num - 1 + assert decreased_pending_num >= 0 + txn.put( + key=key, + value=persipubsub.int_to_bytes(decreased_pending_num), + db=pending_db) + + queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=False) + subscriber_ids = txn.get( + key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) + + subscriber_list = subscriber_ids.decode( + persipubsub.ENCODING).split(' ') + subscriber_set = set(subscriber_list) + subscriber_set.remove(sub_id) + subscriber_str = " ".join(subscriber_set) txn.put( - key=key, - value=persipubsub.int_to_bytes(decreased_pending_num), - db=pending_db) + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=subscriber_str.encode(persipubsub.ENCODING), + db=queue_db) + + self.subscriber_ids.remove(sub_id) diff --git a/persipubsub/environment.py b/persipubsub/environment.py new file mode 100644 index 0000000..79ccef7 --- /dev/null +++ b/persipubsub/environment.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Fabricate new persipubsub components.""" + +import pathlib + +import persipubsub.control +import persipubsub.publisher +import persipubsub.subscriber + + +class Environment: + """Fabricate persipubsub components.""" + + def __init__(self, path: pathlib.Path): + """Initialize.""" + self.path = path + + def __enter__(self) -> 'Environment': + """Enter the context and give environment prepared to constructor.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + """Leave the context.""" + + def new_control(self) -> persipubsub.control.Control: + """ + Fabricate a new control. + + :return: Control to initialize queues + """ + control = persipubsub.control.Control(path=self.path) + control.init() + return control + + def new_publisher( + self, autosync: bool = False) -> persipubsub.publisher.Publisher: + """ + Fabricate a new publisher. + + :return: Publisher to send messages + """ + publisher = persipubsub.publisher.Publisher() + publisher.init(path=self.path, autosync=autosync) + return publisher + + def new_subscriber(self, + identifier: str) -> persipubsub.subscriber.Subscriber: + """ + Fabricate a new subscriber. + + :param identifier: of the subscriber + :return: Subscriber to receive messages + """ + subscriber = persipubsub.subscriber.Subscriber() + subscriber.init(sub_id=identifier, path=self.path) + return subscriber + + +def new_environment(path: pathlib.Path) -> Environment: + """ + Fabricate a new environment. + + :param path: path to the queue + :return Environment to create control, publisher and subscriber + """ + return Environment(path=path) diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index 0132290..4b82aa6 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -1,13 +1,15 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Distribute messages persistent from publisher to subscriber.""" import pathlib -from typing import List, Union +from typing import List, Optional, Union import persipubsub.queue +# pylint: disable=protected-access -class Pub: + +class Publisher: """ Create Publisher ready to send messages. @@ -21,37 +23,25 @@ class Pub: :vartype autosync: bool """ - pub_id = None # type: str - queue = None # type: persipubsub.queue._Queue - sub_list = None # type: List[str] - autosync = None # type: bool - def __init__(self) -> None: """Initialize class object.""" + self.queue = None # type: Optional[persipubsub.queue._Queue] + self.autosync = None # type: Optional[bool] - def init(self, - pub_id: str, - config_pth: Union[pathlib.Path, str], + def init(self, path: Union[pathlib.Path, str], autosync: bool = False) -> None: """ Initialize. :param pub_id: constant ID of publisher - :param config_pth: path to the JSON config file + :param path: path to the queue :param autosync: if True, store data automatically in lmdb """ - self.pub_id = pub_id - - config = persipubsub.get_config(path=config_pth) - publisher = config[pub_id] - queue_dir = publisher["out_queue"] self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access - self.queue.init(config_pth=config_pth, queue_dir=queue_dir) - assert isinstance(publisher["subscribers"], List) - self.sub_list = publisher["subscribers"] # type: List[str] + self.queue.init(path=path) self.autosync = autosync - def __enter__(self) -> 'Pub': + def __enter__(self) -> 'Publisher': """Enter the context and give the pub prepared in the constructor.""" return self @@ -64,7 +54,8 @@ def send(self, msg: bytes) -> None: :param msg: to send to all subscribers """ - self.queue.put(msg=msg, sub_list=self.sub_list) + assert isinstance(self.queue, persipubsub.queue._Queue) + self.queue.put(msg=msg) def send_many(self, msgs: List[bytes]) -> None: """ @@ -72,8 +63,9 @@ def send_many(self, msgs: List[bytes]) -> None: :param msgs: to send to all subscribers """ + assert isinstance(self.queue, persipubsub.queue._Queue) if self.autosync: for msg in msgs: - self.queue.put(msg=msg, sub_list=self.sub_list) + self.queue.put(msg=msg) else: - self.queue.put_many_flush_once(msgs=msgs, sub_list=self.sub_list) + self.queue.put_many_flush_once(msgs=msgs) diff --git a/persipubsub/queue.py b/persipubsub/queue.py index cb41d55..cc71913 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -1,12 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Store messages in a local LMDB queue.""" import datetime import enum import pathlib import uuid -from typing import Any, Dict, List, Optional, Union +from typing import List, Optional, Union -import icontract import lmdb import persipubsub @@ -85,7 +84,7 @@ def _initialize_environment( max_readers=max_reader_num, max_dbs=max_db_num, max_spare_txns=0) - + # TODO(snaji): doc return env @@ -96,6 +95,7 @@ def _prune_dangling_messages_for(queue: '_Queue', sub_list: List[str]) -> None: :param queue: of which dangling messages should be pruned :param sub_list: subscribers of which dangling messages should be pruned """ + assert isinstance(queue.env, lmdb.Environment) with queue.env.begin(write=True) as txn: pending_db = queue.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) @@ -118,6 +118,7 @@ def _prune_dangling_messages_for(queue: '_Queue', sub_list: List[str]) -> None: # subscriber might still await this messages after the timeout. # This messages needs also to be removed from all subscribers. msgs_to_delete_timeout = set() + assert isinstance(queue.hwm, _HighWaterMark) with queue.env.begin(db=meta_db) as txn: cursor = txn.cursor() @@ -160,47 +161,51 @@ class _Queue: :vartype sub_list: List[str] """ - config_pth = None # type: pathlib.Path - env = None # type: lmdb.Environment - hwm = None # type: _HighWaterMark - strategy = None # type: _Strategy - sub_list = None # type: List[str] - def __init__(self) -> None: """Initialize class object.""" + self.path = None # type: Optional[pathlib.Path] + self.env = None # type: Optional[lmdb.Environment] + self.hwm = None # type: Optional[_HighWaterMark] + self.strategy = None # type: Optional[_Strategy] + self.sub_list = None # type: Optional[List[str]] # pylint: disable=too-many-arguments - @icontract.require(lambda max_reader_num: max_reader_num >= 0) - # There have to be at least 4 databases. - # The main db, the data db, the pending db and the meta db. - @icontract.require(lambda max_db_num: max_db_num >= 4) - @icontract.require(lambda max_db_size_bytes: max_db_size_bytes > 0) + # pylint: disable=too-many-locals def init(self, - config_pth: Union[pathlib.Path, str], - queue_dir: Union[pathlib.Path, str], - max_reader_num: int = 1024, - max_db_num: int = 1024, - max_db_size_bytes: int = 32 * 1024**3) -> None: + path: Union[pathlib.Path, str], + max_reader_num: Optional[int] = None, + max_db_num: Optional[int] = None, + max_db_size_bytes: Optional[int] = None) -> None: """ Initialize the queue. - :param config_pth: path to the JSON config file - :param queue_dir: where the queue is stored + :param path: where the queue is stored :param max_reader_num: maximal number of reader :param max_db_num: maximal number of databases :param max_db_size_bytes: maximal size of database (bytes) """ - self.config_pth = config_pth if isinstance(config_pth, pathlib.Path) \ - else pathlib.Path(config_pth) - - queue_dir = queue_dir if isinstance(queue_dir, pathlib.Path) \ - else pathlib.Path(queue_dir) + self.path = path if isinstance(path, pathlib.Path) \ + else pathlib.Path(path) + + max_reader_num_queue = max_reader_num if max_reader_num is not None \ + else persipubsub.bytes_to_int( + persipubsub.get_queue_data( + path=self.path, key=persipubsub.MAX_READER_NUM_KEY)) + max_db_num_queue = max_db_num if max_db_num is not None else \ + persipubsub.bytes_to_int( + persipubsub.get_queue_data( + path=self.path, key=persipubsub.MAX_DB_NUM_KEY)) + max_db_size_bytes_queue = max_db_size_bytes if max_db_size_bytes \ + is not None else \ + persipubsub.bytes_to_int( + persipubsub.get_queue_data( + path=self.path, key=persipubsub.MAX_DB_SIZE_BYTES_KEY)) self.env = _initialize_environment( - queue_dir=queue_dir, - max_reader_num=max_reader_num, - max_db_num=max_db_num, - max_db_size_bytes=max_db_size_bytes) + queue_dir=self.path, + max_reader_num=max_reader_num_queue, + max_db_num=max_db_num_queue, + max_db_size_bytes=max_db_size_bytes_queue) with self.env.begin(write=True) as txn: _ = self.env.open_db(key=persipubsub.DATA_DB, txn=txn, create=True) @@ -208,23 +213,33 @@ def init(self, key=persipubsub.PENDING_DB, txn=txn, create=True) _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) - config = persipubsub.get_config(path=config_pth) # type: Dict[str, Any] - queues = config["queues"] # type: Dict[str, Any] - queue = queues[queue_dir.as_posix()] # type: Dict[str, Any] - hwm = queue["high-water-mark"] - - assert isinstance(hwm['MSG_TIMEOUT_SECS'], int) - assert isinstance(hwm['MAX_MSGS_NUM'], int) - assert isinstance(hwm['HWM_LMDB_SIZE_BYTES'], int) + msg_timeout_secs = persipubsub.bytes_to_int( + persipubsub.get_queue_data( + path=self.path, key=persipubsub.MSG_TIMEOUT_SECS_KEY)) + max_msgs_num = persipubsub.bytes_to_int( + persipubsub.get_queue_data( + path=self.path, key=persipubsub.MAX_MSGS_NUM_KEY)) + hwm_lmdb_size_bytes = persipubsub.bytes_to_int( + persipubsub.get_queue_data( + path=self.path, key=persipubsub.HWM_DB_SIZE_BYTES_KEY)) self.hwm = _HighWaterMark( - msg_timeout_secs=hwm['MSG_TIMEOUT_SECS'], - max_msgs_num=hwm['MAX_MSGS_NUM'], - hwm_lmdb_size_bytes=hwm['HWM_LMDB_SIZE_BYTES']) + msg_timeout_secs=msg_timeout_secs, + max_msgs_num=max_msgs_num, + hwm_lmdb_size_bytes=hwm_lmdb_size_bytes) - assert isinstance(hwm["strategy"], str) - self.strategy = _parse_strategy(hwm["strategy"]) + strategy = persipubsub.get_queue_data( + path=self.path, key=persipubsub.STRATEGY_KEY) + self.strategy = _parse_strategy(strategy.decode(persipubsub.ENCODING)) - self.sub_list = queue["subscribers"] + subscriber_list = persipubsub.get_queue_data( + path=self.path, key=persipubsub.SUBSCRIBER_IDS_KEY) + + if subscriber_list is None: + self.sub_list = [] + else: + # TODO(snaji): decode method + self.sub_list = subscriber_list.decode( + persipubsub.ENCODING).split(' ') def __enter__(self) -> '_Queue': """Enter the context and give the queue prepared in the constructor.""" @@ -233,7 +248,7 @@ def __enter__(self) -> '_Queue': def __exit__(self, exc_type, exc_val, exc_tb) -> None: """Leave the context.""" - def put(self, msg: bytes, sub_list: List[str]) -> None: + def put(self, msg: bytes) -> None: """ Put message to lmdb queue. @@ -244,12 +259,14 @@ def put(self, msg: bytes, sub_list: List[str]) -> None: # every publisher always prunes queue before sending a message. self.vacuum() msg_id = str(datetime.datetime.utcnow().timestamp()) + str(uuid.uuid4()) + assert isinstance(self.env, lmdb.Environment) + assert isinstance(self.sub_list, List) with self.env.begin(write=True) as txn: pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) txn.put( key=persipubsub.encoding(msg_id), - value=persipubsub.int_to_bytes(len(sub_list)), + value=persipubsub.int_to_bytes(len(self.sub_list)), db=pending_db) meta_db = self.env.open_db( @@ -264,13 +281,12 @@ def put(self, msg: bytes, sub_list: List[str]) -> None: key=persipubsub.DATA_DB, txn=txn, create=False) txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) - for sub in sub_list: + for sub in self.sub_list: sub_db = self.env.open_db( key=persipubsub.encoding(sub), txn=txn, create=False) txn.put(key=persipubsub.encoding(msg_id), db=sub_db) - def put_many_flush_once(self, msgs: List[bytes], - sub_list: List[str]) -> None: + def put_many_flush_once(self, msgs: List[bytes]) -> None: """ Put many message to lmdb queue. @@ -280,7 +296,8 @@ def put_many_flush_once(self, msgs: List[bytes], """ # every publisher always prunes queue before sending a message. self.vacuum() - + assert isinstance(self.env, lmdb.Environment) + assert isinstance(self.sub_list, List) with self.env.begin(write=True) as txn: pending_db = self.env.open_db( @@ -293,7 +310,7 @@ def put_many_flush_once(self, msgs: List[bytes], key=persipubsub.DATA_DB, txn=txn, create=False) sub_dbs = set() - for sub in sub_list: + for sub in self.sub_list: sub_dbs.add( self.env.open_db( key=persipubsub.encoding(sub), txn=txn, create=False)) @@ -304,7 +321,7 @@ def put_many_flush_once(self, msgs: List[bytes], txn.put( key=persipubsub.encoding(msg_id), - value=persipubsub.int_to_bytes(len(sub_list)), + value=persipubsub.int_to_bytes(len(self.sub_list)), db=pending_db) txn.put( @@ -318,8 +335,7 @@ def put_many_flush_once(self, msgs: List[bytes], for sub_db in sub_dbs: txn.put(key=persipubsub.encoding(msg_id), db=sub_db) - def front(self, sub_id: str) \ - -> Optional[bytes]: + def front(self, sub_id: str) -> Optional[bytes]: """ Peek at next message in lmdb queue. @@ -328,6 +344,7 @@ def front(self, sub_id: str) \ :param sub_id: Subscriber ID :return: """ + assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=False) as txn: sub_db = self.env.open_db( key=persipubsub.encoding(sub_id), txn=txn, create=False) @@ -338,10 +355,11 @@ def front(self, sub_id: str) \ # check if database is not empty if cursor.first(): key = cursor.key() + msg = txn.get(key=key, db=data_db) + else: + msg = None - return txn.get(key=key, db=data_db) - - return None + return msg def pop(self, sub_id: str) -> None: """ @@ -350,6 +368,7 @@ def pop(self, sub_id: str) -> None: :param sub_id: Subscriber ID :return: """ + assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=True) as txn: sub_db = self.env.open_db( key=persipubsub.encoding(sub_id), txn=txn, create=False) @@ -378,6 +397,7 @@ def prune_dangling_messages(self) -> None: :return: """ + assert isinstance(self.sub_list, List) _prune_dangling_messages_for(queue=self, sub_list=self.sub_list) def check_current_lmdb_size(self) -> int: @@ -386,10 +406,11 @@ def check_current_lmdb_size(self) -> int: Check size of data database by approximating size with multiplying page size with number of pages. + :return: data database size in bytes """ lmdb_size_bytes = 0 - + assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=False) as txn: data_db = self.env.open_db( key=persipubsub.DATA_DB, txn=txn, create=False) @@ -405,8 +426,10 @@ def count_msgs(self) -> int: Count number of messages in database. Count number of messages stored in meta database. + :return: number of messages in database """ + assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=False) as txn: meta_db = self.env.open_db( key=persipubsub.META_DB, txn=txn, create=False) @@ -420,6 +443,7 @@ def vacuum(self) -> None: :return: """ + assert isinstance(self.hwm, _HighWaterMark) self.prune_dangling_messages() msgs_num = self.count_msgs() if msgs_num >= self.hwm.max_msgs_num: @@ -433,11 +457,12 @@ def prune_messages(self) -> None: Prune one half of the messages stored. Depending on the strategy the first or the last will be deleted. + :return: """ # pylint: disable=too-many-locals messages_to_delete = set() - + assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=False) as txn: meta_db = self.env.open_db( key=persipubsub.META_DB, txn=txn, create=False) @@ -464,6 +489,7 @@ def prune_messages(self) -> None: else: raise RuntimeError("Pruning strategy not set.") + assert isinstance(self.sub_list, List) with self.env.begin(write=True) as txn: pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 566d265..35ca17f 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -1,18 +1,21 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Receive messages persistent from publisher to subscriber.""" import contextlib import datetime import pathlib import time -from typing import Generator, Union +from typing import Generator, Optional, Union import icontract +import lmdb # pylint: disable=unused-import import persipubsub.queue +# pylint: disable=protected-access -class Sub: + +class Subscriber: """ Create Subscriber ready to receive messages. @@ -22,28 +25,25 @@ class Sub: :vartype queue: persipubsub.queue.Queue """ - sub_id = None # type: str - queue = None # type: persipubsub.queue._Queue - def __init__(self) -> None: """Initialize class object.""" + self.sub_id = None # type: Optional[str] + self.queue = None # type: Optional[persipubsub.queue._Queue] - def init(self, sub_id: str, config_pth: Union[pathlib.Path, str]) -> None: + def init(self, sub_id: str, path: Union[pathlib.Path, str]) -> None: """ Initialize. :param sub_id: unique subscriber id - :param config_pth: path to the JSON config file + :param path: path to the queue """ self.sub_id = sub_id - - config = persipubsub.get_config(path=config_pth) - subscriber = config[sub_id] - queue_dir = subscriber["in_queue"] + assert isinstance(self.sub_id, str) self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access - self.queue.init(config_pth=config_pth, queue_dir=queue_dir) + self.queue.init(path=path) + assert isinstance(self.queue, persipubsub.queue._Queue) - def __enter__(self) -> 'Sub': + def __enter__(self) -> 'Subscriber': """Enter the context and give the sub prepared in the constructor.""" return self @@ -63,6 +63,8 @@ def receive(self, timeout: int = 60, retries: int = 10) -> Generator: """ msg = None end = int(datetime.datetime.utcnow().timestamp()) + timeout + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.sub_id, str) try: while int(datetime.datetime.utcnow().timestamp()) <= end: msg = self.queue.front(sub_id=self.sub_id) @@ -78,14 +80,27 @@ def receive(self, timeout: int = 60, retries: int = 10) -> Generator: def _pop(self) -> None: """Pop a message from the subscriber's lmdb.""" + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.sub_id, str) self.queue.pop(sub_id=self.sub_id) + # TODO(snaji): in one transaction? + # TODO: receive_to_top() -- akin to receive(), but removes all the messages + # except the top one (in the same transaction) + # TODO: clarify in docs; used in cases where this particular subscriber + # cares only about the very last message; other subscribers care about + # all the messages in the queue + # TODO: another use case: you only want to store the latest -- all the + # subscribers are interested only in the latest, use high watermark 1 def pop_to_top(self) -> None: """ Pops all messages until the most recent one. Used for slow processes to ensure realtime processing. """ + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.queue.env, lmdb.Environment) + assert isinstance(self.sub_id, str) with self.queue.env.begin(write=False) as txn: sub_db = self.queue.env.open_db( key=persipubsub.encoding(self.sub_id), txn=txn, create=False) diff --git a/tests/__init__.py b/tests/__init__.py index c30181b..8dcb92a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -6,6 +6,7 @@ import unittest from typing import Any, Dict +import lmdb import temppathlib import persipubsub @@ -18,6 +19,10 @@ TEST_HWM_MSG_NUM = 10 # type: int TEST_HWM_LMDB_SIZE = LMDB_PAGE_SIZE * 2 # type: int +TEST_MAX_DB_NUM = 1024 # type: int +TEST_MAX_DB_SIZE_BYTES = TEST_HWM_LMDB_SIZE * TEST_MAX_DB_NUM # type: int +TEST_MAX_READER_NUM = 1024 # type: int + # define all encodings here ENCODING = 'utf-8' BYTES_LENGTH = 8 @@ -27,53 +32,24 @@ DATA_DB = "data_db".encode(ENCODING) # msg_id | data PENDING_DB = "pending_db".encode(ENCODING) # msg_id | pending subscriber META_DB = "meta_db".encode(ENCODING) # msg_id | metadata - - -def generate_test_config(path: pathlib.Path) -> Dict[str, Any]: - queue_dir = path / "queue" - queue_dir.mkdir(parents=True, exist_ok=True) - - test_config = { - "pub": { - "out_queue": queue_dir.as_posix(), - "subscribers": ["sub"] - }, - "sub": { - "in_queue": queue_dir.as_posix() - }, - "queues": { - queue_dir.as_posix(): { - "path": queue_dir.as_posix(), - "max_reader_num": 1024, - "max_db_num": 1024, - "max_db_size_bytes": 34359738368, - "subscribers": ["sub"], - "high-water-mark": { - "MSG_TIMEOUT_SECS": TEST_MSG_TIMEOUT, - "MAX_MSGS_NUM": TEST_HWM_MSG_NUM, - "HWM_LMDB_SIZE_BYTES": TEST_HWM_LMDB_SIZE, - "strategy": "prune_first" - } - }, - } - } - - return test_config +QUEUE_DB = "queue_db".encode(ENCODING) # queue_pth | all queue data class TestPersiPubSub(unittest.TestCase): - def test_get_config(self): + def test_get_data(self): with temppathlib.TemporaryDirectory() as tmp_dir: - expected_config = generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(expected_config, file_object) - - config = persipubsub.get_config(path=file) - - self.assertDictEqual(expected_config, config) + env = lmdb.open(path=tmp_dir.path.as_posix(), max_dbs=2) + + with env.begin(write=True) as txn: + queue_db = env.open_db(key=QUEUE_DB, txn=txn) + txn.put( + key="key".encode(ENCODING), + value="value".encode(ENCODING), + db=queue_db) + + data = persipubsub.get_queue_data( + path=tmp_dir.path, key="key".encode(ENCODING)) + self.assertEqual(b'value', data) if __name__ == '__main__': diff --git a/tests/test_control.py b/tests/test_control.py index e6bd7c8..b6b9b12 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -2,7 +2,7 @@ """Test control unit.""" import datetime -import json +import pathlib import unittest from typing import List @@ -16,53 +16,35 @@ # pylint: disable=protected-access -class TestControl(unittest.TestCase): - def test_initialize_all(self): - with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) +def setup(path: pathlib.Path, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + control = persipubsub.control.Control(path=path) - persipubsub.control.initialize_all_dbs(config_pth=file) + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + control.init( + subscriber_ids=sub_list, + max_readers=tests.TEST_MAX_READER_NUM, + max_size=tests.TEST_MAX_DB_SIZE_BYTES, + high_watermark=hwm, + strategy=strategy) - expected_db_keys = [b'data_db', b'meta_db', b'pending_db', b'sub'] - db_keys = [] # type: List[bytes] - - with queue.env.begin() as txn: - cursor = txn.cursor() - cursor.first() - for key, value in cursor: # pylint: disable=unused-variable - db_keys.append(key) + return control - self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) - def test_add_sub(self): +class TestControl(unittest.TestCase): + def test_initialize_all(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - - persipubsub.control._add_sub(sub_id="sub1", queue=queue) - persipubsub.control._add_sub(sub_id="sub2", queue=queue) + control = setup(path=tmp_dir.path, sub_list=['sub']) expected_db_keys = [ - b'data_db', b'meta_db', b'pending_db', b'sub1', b'sub2' + b'data_db', b'meta_db', b'pending_db', b'queue_db', b'sub' ] db_keys = [] # type: List[bytes] - with queue.env.begin() as txn: + with control.queue.env.begin() as txn: cursor = txn.cursor() cursor.first() for key, value in cursor: # pylint: disable=unused-variable @@ -72,24 +54,16 @@ def test_add_sub(self): def test_del_sub(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + control = setup(path=tmp_dir.path, sub_list=["sub1", "sub2"]) - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + control._remove_sub(sub_id="sub2", env=control.queue.env) - persipubsub.control._add_sub(sub_id="sub1", queue=queue) - persipubsub.control._add_sub(sub_id="sub2", queue=queue) - persipubsub.control._remove_sub(sub_id="sub2", queue=queue) - - expected_db_keys = [b'data_db', b'meta_db', b'pending_db', b'sub1'] + expected_db_keys = [ + b'data_db', b'meta_db', b'pending_db', b'queue_db', b'sub1' + ] db_keys = [] # type: List[bytes] - with queue.env.begin() as txn: + with control.queue.env.begin() as txn: cursor = txn.cursor() cursor.first() for key, value in cursor: # pylint: disable=unused-variable @@ -99,23 +73,16 @@ def test_del_sub(self): def test_clear_all_subs(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + control = setup(path=tmp_dir.path, sub_list=["sub1", "sub2"]) - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + control.clear_all_subscribers() - persipubsub.control.initialize_all_dbs(config_pth=file) - persipubsub.control.clear_all_subs(config_pth=file) - - expected_db_keys = [b'data_db', b'meta_db', b'pending_db'] + expected_db_keys = [ + b'data_db', b'meta_db', b'pending_db', b'queue_db' + ] db_keys = [] # type: List[bytes] - with queue.env.begin() as txn: + with control.queue.env.begin() as txn: cursor = txn.cursor() cursor.first() for key, value in cursor: # pylint: disable=unused-variable @@ -126,27 +93,18 @@ def test_clear_all_subs(self): def test_prune_dangling_messages(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + control = setup(path=tmp_dir.path, sub_list=["sub"]) - persipubsub.control.initialize_all_dbs(config_pth=file) + control.queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT - - with queue.env.begin(write=True) as txn: - sub_db = queue.env.open_db( + with control.queue.env.begin(write=True) as txn: + sub_db = control.queue.env.open_db( key='sub'.encode(tests.ENCODING), txn=txn, create=False) txn.put(key="timeout_msg".encode(tests.ENCODING), db=sub_db) txn.put(key="valid_msg".encode(tests.ENCODING), db=sub_db) - with queue.env.begin(write=True) as txn: - data_db = queue.env.open_db( + with control.queue.env.begin(write=True) as txn: + data_db = control.queue.env.open_db( key=tests.DATA_DB, txn=txn, create=False) txn.put( @@ -161,8 +119,8 @@ def test_prune_dangling_messages(self): key="valid_msg".encode(tests.ENCODING), value="Free me!".encode(tests.ENCODING), db=data_db) - with queue.env.begin(write=True) as txn: - pending_db = queue.env.open_db( + with control.queue.env.begin(write=True) as txn: + pending_db = control.queue.env.open_db( key=tests.PENDING_DB, txn=txn, create=False) txn.put( @@ -181,8 +139,8 @@ def test_prune_dangling_messages(self): length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=pending_db) - with queue.env.begin(write=True) as txn: - meta_db = queue.env.open_db( + with control.queue.env.begin(write=True) as txn: + meta_db = control.queue.env.open_db( key=tests.META_DB, txn=txn, create=False) txn.put( @@ -203,7 +161,7 @@ def test_prune_dangling_messages(self): length=tests.BYTES_LENGTH, byteorder=tests.BYTES_ORDER), db=meta_db) - persipubsub.control.prune_dangling_messages(config_pth=file) + control.prune_dangling_messages() dbs = [sub_db, data_db, pending_db, meta_db] @@ -212,7 +170,7 @@ def test_prune_dangling_messages(self): # pylint: disable=invalid-name for db in dbs: - with queue.env.begin(db=db) as txn: + with control.queue.env.begin(db=db) as txn: cursor = txn.cursor() cursor.first() for key, value in cursor: # pylint: disable=unused-variable @@ -223,38 +181,38 @@ def test_prune_dangling_messages(self): def test_prune_all_messages_for_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - - persipubsub.control.initialize_all_dbs(config_pth=file) + control = setup(path=tmp_dir.path, sub_list=["sub"]) msg = persipubsub.encoding("hello world!") - queue.put(msg=msg, sub_list=["sub"]) - queue.put(msg=msg, sub_list=["sub"]) + control.queue.put(msg=msg) + control.queue.put(msg=msg) - with queue.env.begin(write=False) as txn: - sub_db = queue.env.open_db( + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( key=persipubsub.encoding('sub'), txn=txn, create=False) sub_stat = txn.stat(db=sub_db) self.assertEqual(2, sub_stat['entries']) - persipubsub.control._prune_all_messages_for( - sub_id="sub", queue=queue) + control._prune_all_messages_for(sub_id="sub") - with queue.env.begin(write=False) as txn: - sub_db = queue.env.open_db( + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( key=persipubsub.encoding('sub'), txn=txn, create=False) sub_stat = txn.stat(db=sub_db) self.assertEqual(0, sub_stat['entries']) + def test_is_initialized(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + control = setup(path=tmp_dir.path, sub_list=["sub"]) + + self.assertTrue(control.check_queue_is_initialized()) + + def test_is_not_initialized(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + control = persipubsub.control.Control(path=tmp_dir.path) + self.assertFalse(control.check_queue_is_initialized()) + if __name__ == '__main__': unittest.main() diff --git a/tests/test_environment.py b/tests/test_environment.py new file mode 100644 index 0000000..d505e94 --- /dev/null +++ b/tests/test_environment.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +"""Test environment.""" + +import unittest + +import temppathlib + +import persipubsub.control +import persipubsub.environment +import persipubsub.publisher +import persipubsub.queue +import persipubsub.subscriber + +# pylint: disable=missing-docstring +# pylint: disable=protected-access + + +class TestEnvironment(unittest.TestCase): + def test_new_environment(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + self.assertIsInstance(env, persipubsub.environment.Environment) + + def test_new_control(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + ctl = env.new_control() + self.assertIsInstance(ctl, persipubsub.control.Control) + + def test_new_publisher(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + ctl = env.new_control() + # TODO(snaji): icontract? optional, random number!!! + ctl.init() + pub = env.new_publisher() + self.assertIsInstance(pub, persipubsub.publisher.Publisher) + + def test_new_subscriber(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + ctl = env.new_control() + # TODO(snaji): icontract? optional + ctl.init() + sub = env.new_subscriber(identifier="sub") + self.assertIsInstance(sub, persipubsub.subscriber.Subscriber) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 6bc10ed..df5270b 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -1,10 +1,10 @@ #!/usr/bin/env python """Test publisher.""" -import json +import pathlib import unittest +from typing import List -import lmdb import temppathlib import persipubsub.control @@ -16,23 +16,35 @@ # pylint: disable=protected-access +def setup(path: pathlib.Path, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + control = persipubsub.control.Control(path=path) + + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first + + control.init( + subscriber_ids=sub_list, + max_readers=tests.TEST_MAX_READER_NUM, + max_size=tests.TEST_MAX_DB_SIZE_BYTES, + high_watermark=hwm, + strategy=strategy) + + return control + + class TestPublisher(unittest.TestCase): def test_send(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" + _ = setup(path=tmp_dir.path, sub_list=['sub']) - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) - pub = persipubsub.publisher.Pub() - pub.init(pub_id="pub", config_pth=file) + pub = persipubsub.publisher.Publisher() + pub.init(path=tmp_dir.path) msg = "Hello world!".encode(tests.ENCODING) pub.send(msg=msg) @@ -53,41 +65,14 @@ def test_send(self): self.assertIsNotNone(item) self.assertEqual(msg, item) - def test_send_to_nonexisting_db(self): - with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) - queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - - pub = persipubsub.publisher.Pub() - pub.init(pub_id="pub", config_pth=file) - - pub.sub_list.append('non-existent sub') - - msg = "message for every subscriber".encode(tests.ENCODING) - self.assertRaises(lmdb.NotFoundError, pub.send, msg=msg) - def test_send_many(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + subscriber = "sub" + _ = setup(path=tmp_dir.path, sub_list=[subscriber]) - pub = persipubsub.publisher.Pub() - pub.init(pub_id='pub', config_pth=file) + pub = persipubsub.publisher.Publisher() + pub.init(path=tmp_dir.path) msg = "I'm a message".encode(tests.ENCODING) msgs = [] @@ -95,8 +80,6 @@ def test_send_many(self): for _ in range(msg_num): msgs.append(msg) - subscriber = "sub" - pub.send_many(msgs=msgs) with pub.queue.env.begin(write=False) as txn: diff --git a/tests/test_queue.py b/tests/test_queue.py index 3c7c0aa..f9b1008 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Test database.""" -import json +import pathlib import time import unittest +from typing import List import temppathlib @@ -15,6 +16,27 @@ # pylint: disable=protected-access +def setup(path: pathlib.Path, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + control = persipubsub.control.Control(path=path) + + hwm = persipubsub.queue._HighWaterMark( + msg_timeout_secs=tests.TEST_MSG_TIMEOUT, + max_msgs_num=tests.TEST_HWM_MSG_NUM, + hwm_lmdb_size_bytes=tests.TEST_HWM_LMDB_SIZE) + strategy = persipubsub.queue._Strategy.prune_first + + control.init( + subscriber_ids=sub_list, + max_readers=tests.TEST_MAX_READER_NUM, + max_size=tests.TEST_MAX_DB_SIZE_BYTES, + high_watermark=hwm, + strategy=strategy) + + return control + + class TestQueue(unittest.TestCase): def test_initialize_environment(self): with temppathlib.TemporaryDirectory() as tmp_dir: @@ -44,17 +66,11 @@ def test_put_to_single_subscriber(self): subscriber = "sub" - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + _ = setup(path=tmp_dir.path, sub_list=['sub']) - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.put(msg=msg, sub_list=[subscriber]) + queue.init(path=tmp_dir.path) + queue.put(msg=msg) with queue.env.begin() as txn: self.assertIsNotNone( @@ -82,20 +98,12 @@ def test_put_multiple_subscriber(self): sub_list = ["sub", "another_sub"] - config = tests.generate_test_config(path=tmp_dir.path) + _ = setup(path=tmp_dir.path, sub_list=sub_list) - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - - persipubsub.control._add_sub('another_sub', queue=queue) + queue.init(path=tmp_dir.path) - queue.put(msg=msg, sub_list=sub_list) + queue.put(msg=msg) with queue.env.begin() as txn: self.assertIsNotNone( @@ -132,16 +140,11 @@ def test_put_multiple_subscriber(self): def test_put_many(self): # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + subscriber = "sub" + _ = setup(path=tmp_dir.path, sub_list=[subscriber]) - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) msg = "I'm a message".encode(tests.ENCODING) msgs = [] @@ -149,9 +152,7 @@ def test_put_many(self): for _ in range(msg_num): msgs.append(msg) - subscriber = "sub" - - queue.put_many_flush_once(msgs=msgs, sub_list=[subscriber]) + queue.put_many_flush_once(msgs=msgs) with queue.env.begin(write=False) as txn: self.assertIsNotNone( @@ -176,19 +177,11 @@ def test_front(self): msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" - - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=[subscriber]) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.put(msg=msg, sub_list=[subscriber]) + queue.init(path=tmp_dir.path) + queue.put(msg=msg) # pylint: disable=assignment-from-none # pylint: disable=assignment-from-no-return @@ -201,19 +194,11 @@ def test_pop(self): msg = "I'm a message.".encode(tests.ENCODING) subscriber = "sub" - - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=[subscriber]) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - queue.put(msg=msg, sub_list=[subscriber]) + queue.init(path=tmp_dir.path) + queue.put(msg=msg) # pylint: disable=assignment-from-none # pylint: disable=assignment-from-no-return @@ -248,62 +233,36 @@ def test_pop_queue_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: subscriber = "sub" - - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=[subscriber]) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) self.assertRaises(RuntimeError, queue.pop, sub_id=subscriber) def test_queue_initialisation(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() + queue.init(path=tmp_dir.path) - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") - - queue_config = config['queues'][(tmp_dir.path / "queue").as_posix()] self.assertIsNotNone(queue.env) - self.assertEqual(queue_config['path'], queue.env.path()) - hwm = queue_config['high-water-mark'] - self.assertEqual(hwm['HWM_LMDB_SIZE_BYTES'], + self.assertEqual(tmp_dir.path.as_posix(), queue.env.path()) + self.assertEqual(tests.TEST_HWM_LMDB_SIZE, queue.hwm.hwm_lmdb_size_bytes) - self.assertEqual(hwm['MAX_MSGS_NUM'], queue.hwm.max_msgs_num) - self.assertEqual(hwm['MSG_TIMEOUT_SECS'], - queue.hwm.msg_timeout_secs) - self.assertEqual(hwm['strategy'], queue.strategy.name) + self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.hwm.max_msgs_num) + self.assertEqual(tests.TEST_MSG_TIMEOUT, queue.hwm.msg_timeout_secs) + self.assertEqual(persipubsub.queue._Strategy.prune_first.name, + queue.strategy.name) self.assertEqual(['sub'], queue.sub_list) def test_overflow_msgs_limit(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() - - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM @@ -311,11 +270,11 @@ def test_overflow_msgs_limit(self): self.assertEqual(0, queue.count_msgs()) for _ in range(tests.TEST_HWM_MSG_NUM): - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.count_msgs()) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual( int(tests.TEST_HWM_MSG_NUM - int(tests.TEST_HWM_MSG_NUM / 2)), @@ -323,88 +282,64 @@ def test_overflow_msgs_limit(self): def test_overflow_limit_size(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() - - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE msg = ("a" * (int(tests.LMDB_PAGE_SIZE / 4))).encode(tests.ENCODING) while queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE: - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertTrue( queue.check_current_lmdb_size() > tests.TEST_HWM_LMDB_SIZE) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertTrue( queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE) def test_timeout(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() - - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT msg = "hello world".encode(tests.ENCODING) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual(1, queue.count_msgs()) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual(2, queue.count_msgs()) time.sleep(tests.TEST_MSG_TIMEOUT + 1) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual(1, queue.count_msgs()) def test_strategy_prune_first(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() - - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) queue.strategy = persipubsub.queue._Strategy.prune_first queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM for index in range(tests.TEST_HWM_MSG_NUM): msg = "secret message {}".format(index).encode(tests.ENCODING) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual("secret message 0".encode(tests.ENCODING), queue.front(sub_id='sub')) msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( tests.ENCODING) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual( "secret message {}".format( @@ -413,32 +348,24 @@ def test_strategy_prune_first(self): def test_strategy_prune_last(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) + _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() - - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) queue.strategy = persipubsub.queue._Strategy.prune_last queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM for index in range(tests.TEST_HWM_MSG_NUM): msg = "secret message {}".format(index).encode(tests.ENCODING) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual("secret message 0".encode(tests.ENCODING), queue.front(sub_id='sub')) msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( tests.ENCODING) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertEqual("secret message 0".encode(tests.ENCODING), queue.front(sub_id='sub')) diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index cc32a1b..8e08b85 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -1,8 +1,9 @@ #!/usr/bin/env python """Test subscriber.""" -import json +import pathlib import unittest +from typing import List import temppathlib @@ -15,25 +16,37 @@ # pylint: disable=protected-access +def setup(path: pathlib.Path, + sub_list: List[str]) -> persipubsub.control.Control: + """Create an initialized control""" + control = persipubsub.control.Control(path=path) + + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first + + control.init( + subscriber_ids=sub_list, + max_readers=tests.TEST_MAX_READER_NUM, + max_size=tests.TEST_MAX_DB_SIZE_BYTES, + high_watermark=hwm, + strategy=strategy) + + return control + + class TestSubscriber(unittest.TestCase): def test_receive_message(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + _ = setup(path=tmp_dir.path, sub_list=['sub']) - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) - sub = persipubsub.subscriber.Sub() - sub.init(sub_id="sub", config_pth=file) + sub = persipubsub.subscriber.Subscriber() + sub.init(sub_id="sub", path=tmp_dir.path) msg = "Hello World!".encode(tests.ENCODING) - queue.put(msg=msg, sub_list=["sub"]) + queue.put(msg=msg) with sub.receive(timeout=1) as received_msg: self.assertIsNotNone(received_msg) @@ -41,25 +54,19 @@ def test_receive_message(self): def test_timeout_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) + _ = setup(path=tmp_dir.path, sub_list=['sub']) - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) - sub = persipubsub.subscriber.Sub() - sub.init(sub_id='sub', config_pth=file) + sub = persipubsub.subscriber.Subscriber() + sub.init(sub_id='sub', path=tmp_dir.path) with sub.receive(timeout=1) as received_msg: self.assertIsNone(received_msg) msg = "message send after timeout and will not be popped" \ "".encode(tests.ENCODING) - queue.put(msg=msg, sub_list=queue.sub_list) + queue.put(msg=msg) self.assertIsNone(received_msg) self.assertEqual( @@ -76,25 +83,19 @@ def test_timeout_subscriber(self): def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" + _ = setup(path=tmp_dir.path, sub_list=['sub']) - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) - - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) - sub = persipubsub.subscriber.Sub() - sub.init(sub_id='sub', config_pth=file) + sub = persipubsub.subscriber.Subscriber() + sub.init(sub_id='sub', path=tmp_dir.path) msg1 = "I'm a message".encode(tests.ENCODING) - queue.put(msg=msg1, sub_list=queue.sub_list) + queue.put(msg=msg1) msg2 = "I'm a message too".encode(tests.ENCODING) - queue.put(msg=msg2, sub_list=queue.sub_list) + queue.put(msg=msg2) sub._pop() @@ -104,19 +105,13 @@ def test_pop(self): def test_pop_when_empty(self): with temppathlib.TemporaryDirectory() as tmp_dir: - config = tests.generate_test_config(path=tmp_dir.path) - - file = tmp_dir.path / "config.json" - - with open(file=file.as_posix(), mode='wt') as file_object: - json.dump(config, file_object) + _ = setup(path=tmp_dir.path, sub_list=['sub']) - persipubsub.control.initialize_all_dbs(config_pth=file) queue = persipubsub.queue._Queue() - queue.init(config_pth=file, queue_dir=tmp_dir.path / "queue") + queue.init(path=tmp_dir.path) - sub = persipubsub.subscriber.Sub() - sub.init(sub_id='sub', config_pth=file) + sub = persipubsub.subscriber.Subscriber() + sub.init(sub_id='sub', path=tmp_dir.path) self.assertRaises(RuntimeError, sub._pop) From 2a81ad0a8aab3dd31d0fccfef971ba6845ba44c9 Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 5 Feb 2019 11:34:09 +0100 Subject: [PATCH 22/35] code config free version --- docs/source/persipubsub/index.rst | 4 +- docs/source/persipubsub/persipubsub.env.rst | 5 +++ docs/source/persipubsub/persipubsub.rst | 5 +++ persipubsub/__init__.py | 36 ++++++----------- persipubsub/control.py | 39 +++++++++++-------- persipubsub/environment.py | 23 +++++++++-- persipubsub/queue.py | 9 ++--- persipubsub/subscriber.py | 43 +++++++++++++++------ tests/test_environment.py | 2 - tests/test_subscriber.py | 20 ++++++++++ 10 files changed, 122 insertions(+), 64 deletions(-) create mode 100644 docs/source/persipubsub/persipubsub.env.rst create mode 100644 docs/source/persipubsub/persipubsub.rst diff --git a/docs/source/persipubsub/index.rst b/docs/source/persipubsub/index.rst index dbf7647..e37145e 100644 --- a/docs/source/persipubsub/index.rst +++ b/docs/source/persipubsub/index.rst @@ -5,5 +5,7 @@ persipubsub .. toctree:: persipubsub.ctl + persipubsub.env persipubsub.publisher - persipubsub.subscriber \ No newline at end of file + persipubsub.subscriber + persipubsub \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.env.rst b/docs/source/persipubsub/persipubsub.env.rst new file mode 100644 index 0000000..a317995 --- /dev/null +++ b/docs/source/persipubsub/persipubsub.env.rst @@ -0,0 +1,5 @@ +Environment +*********** + +.. automodule:: persipubsub.environment + :members: \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.rst b/docs/source/persipubsub/persipubsub.rst new file mode 100644 index 0000000..fc3a069 --- /dev/null +++ b/docs/source/persipubsub/persipubsub.rst @@ -0,0 +1,5 @@ +Tools +***** + +.. automodule:: persipubsub + :members: \ No newline at end of file diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py index 5d73d8a..ad9a217 100644 --- a/persipubsub/__init__.py +++ b/persipubsub/__init__.py @@ -1,8 +1,6 @@ #!/usr/bin/env python """Distribute messages persistent from publisher to subscriber.""" -import json import pathlib -from typing import Any, Dict, Union import lmdb @@ -38,11 +36,21 @@ def encoding(string: str) -> bytes: Encode a string with utf-8 encoding. :param string: any string - :return: string encoded as utf-8 + :return: string encoded with utf-8 """ return string.encode(encoding=ENCODING) +def decoding(encoded_str: bytes) -> str: + """ + Decode bytes with utf-8 encoding. + + :param encoded_str: any bytes + :return: bytes decoded with utf-8 + """ + return encoded_str.decode(encoding=ENCODING) + + def int_to_bytes(value: int) -> bytes: """ Encode an integer to an array of bytes. @@ -84,25 +92,3 @@ def get_queue_data(path: pathlib.Path, key: bytes) -> bytes: data = txn.get(key=key, db=queue_db) return data - - -def put_queue_data(path: pathlib.Path, key: bytes, value: bytes) -> None: - """ - Get queue data. - - :param path: to the queue - :param key: of value to store - :param value: to store - :return: - """ - env = lmdb.open( - path=path, - map_size=MAX_DB_SIZE_BYTES, - subdir=True, - max_readers=MAX_READER_NUM, - max_dbs=MAX_READER_NUM, - max_spare_txns=0) - - with env.begin(write=True) as txn: - queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) - txn.put(key=key, value=value, db=queue_db) diff --git a/persipubsub/control.py b/persipubsub/control.py index af504ce..a2fd63a 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -95,8 +95,8 @@ def _add_sub(sub_id: str, env: lmdb.Environment) -> None: if subscriber_ids is None: subscriber_list = [] # type: List[str] else: - subscriber_list = subscriber_ids.decode( - persipubsub.ENCODING).split(' ') + subscriber_list = persipubsub.decoding( + encoded_str=subscriber_ids).split(' ') subscriber_set = set(subscriber_list) subscriber_set.add(sub_id) subscriber_str = " ".join(subscriber_set) @@ -138,19 +138,19 @@ def init(self, :return: """ if self.check_queue_is_initialized(): - self.reinitialize_queue() + self._reinitialize_queue() else: if subscriber_ids is None: subscriber_ids = [] assert isinstance(subscriber_ids, Sequence) - self.initialize_queue( + self._initialize_queue( subscriber_ids=subscriber_ids, max_readers=max_readers, max_size=max_size, high_watermark=high_watermark, strategy=strategy) - def reinitialize_queue(self) -> None: + def _reinitialize_queue(self) -> None: """Reinitialize the queue which is maintained by the control.""" self.queue = persipubsub.queue._Queue() self.queue.init(path=self.path) @@ -158,14 +158,20 @@ def reinitialize_queue(self) -> None: self.subscriber_ids = set(self.queue.sub_list) # pylint: disable=too-many-arguments - def initialize_queue(self, - subscriber_ids: Sequence[str], - max_readers: int = 1024, - max_size: int = 32 * 1024**3, - high_watermark: persipubsub.queue. - _HighWaterMark = persipubsub.queue._HighWaterMark(), - strategy: persipubsub.queue._Strategy = persipubsub. - queue._Strategy.prune_first) -> None: + @icontract.require(lambda max_readers: max_readers >= 0) + # yapf: disable + # Each named database needs at least one page of 4096 bytes. + @icontract.require(lambda max_size, subscriber_ids: + max_size >= (5 + len(subscriber_ids) * 4096)) + # yapf: enable + def _initialize_queue(self, + subscriber_ids: Sequence[str], + max_readers: int = 1024, + max_size: int = 32 * 1024**3, + high_watermark: persipubsub.queue. + _HighWaterMark = persipubsub.queue._HighWaterMark(), + strategy: persipubsub.queue._Strategy = persipubsub. + queue._Strategy.prune_first) -> None: """ Initialize queue. @@ -182,7 +188,6 @@ def initialize_queue(self, # Databases needed for queue: # 5 queues (main db, data db, meta db, pending db, queue db) # + each subscriber has its own db - # TODO(snaji): should it be max(len(sub_ids), max_readers)? max_db_num = 5 + len(self.subscriber_ids) env = persipubsub.queue._initialize_environment( @@ -211,7 +216,7 @@ def check_queue_is_initialized(self) -> bool: """ Check if queue is initialized. - :return is initialized when all values for the given keys are set + :return: is initialized when all values for the given keys are set """ keys = [ persipubsub.MAX_DB_SIZE_BYTES_KEY, persipubsub.MAX_DB_NUM_KEY, @@ -330,8 +335,8 @@ def _remove_sub(self, sub_id: str, env: lmdb.Environment) -> None: subscriber_ids = txn.get( key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) - subscriber_list = subscriber_ids.decode( - persipubsub.ENCODING).split(' ') + subscriber_list = persipubsub.decoding( + encoded_str=subscriber_ids).split(' ') subscriber_set = set(subscriber_list) subscriber_set.remove(sub_id) subscriber_str = " ".join(subscriber_set) diff --git a/persipubsub/environment.py b/persipubsub/environment.py index 79ccef7..7a60c40 100644 --- a/persipubsub/environment.py +++ b/persipubsub/environment.py @@ -2,11 +2,15 @@ """Fabricate new persipubsub components.""" import pathlib +from typing import Optional, Sequence import persipubsub.control import persipubsub.publisher +import persipubsub.queue import persipubsub.subscriber +# pylint: disable = protected-access + class Environment: """Fabricate persipubsub components.""" @@ -22,14 +26,27 @@ def __enter__(self) -> 'Environment': def __exit__(self, exc_type, exc_val, exc_tb) -> None: """Leave the context.""" - def new_control(self) -> persipubsub.control.Control: + # pylint: disable=too-many-arguments + def new_control(self, + subscriber_ids: Optional[Sequence[str]] = None, + max_readers: int = 1024, + max_size: int = 32 * 1024**3, + high_watermark: persipubsub.queue. + _HighWaterMark = persipubsub.queue._HighWaterMark(), + strategy: persipubsub.queue._Strategy = persipubsub.queue. + _Strategy.prune_first) -> persipubsub.control.Control: """ Fabricate a new control. :return: Control to initialize queues """ control = persipubsub.control.Control(path=self.path) - control.init() + control.init( + subscriber_ids=subscriber_ids, + max_readers=max_readers, + max_size=max_size, + high_watermark=high_watermark, + strategy=strategy) return control def new_publisher( @@ -61,6 +78,6 @@ def new_environment(path: pathlib.Path) -> Environment: Fabricate a new environment. :param path: path to the queue - :return Environment to create control, publisher and subscriber + :return: Environment to create control, publisher and subscriber """ return Environment(path=path) diff --git a/persipubsub/queue.py b/persipubsub/queue.py index cc71913..e28fbeb 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -84,7 +84,6 @@ def _initialize_environment( max_readers=max_reader_num, max_dbs=max_db_num, max_spare_txns=0) - # TODO(snaji): doc return env @@ -229,7 +228,8 @@ def init(self, strategy = persipubsub.get_queue_data( path=self.path, key=persipubsub.STRATEGY_KEY) - self.strategy = _parse_strategy(strategy.decode(persipubsub.ENCODING)) + self.strategy = _parse_strategy( + persipubsub.decoding(encoded_str=strategy)) subscriber_list = persipubsub.get_queue_data( path=self.path, key=persipubsub.SUBSCRIBER_IDS_KEY) @@ -237,9 +237,8 @@ def init(self, if subscriber_list is None: self.sub_list = [] else: - # TODO(snaji): decode method - self.sub_list = subscriber_list.decode( - persipubsub.ENCODING).split(' ') + self.sub_list = persipubsub.decoding( + encoded_str=subscriber_list).split(' ') def __enter__(self) -> '_Queue': """Enter the context and give the queue prepared in the constructor.""" diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 35ca17f..b0756ee 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -84,19 +84,23 @@ def _pop(self) -> None: assert isinstance(self.sub_id, str) self.queue.pop(sub_id=self.sub_id) - # TODO(snaji): in one transaction? - # TODO: receive_to_top() -- akin to receive(), but removes all the messages - # except the top one (in the same transaction) - # TODO: clarify in docs; used in cases where this particular subscriber - # cares only about the very last message; other subscribers care about - # all the messages in the queue - # TODO: another use case: you only want to store the latest -- all the - # subscribers are interested only in the latest, use high watermark 1 - def pop_to_top(self) -> None: + @icontract.require(lambda timeout: timeout > 0) + @icontract.require(lambda retries: retries > 0) + @contextlib.contextmanager + def receive_to_top(self, timeout: int = 60, retries: int = 10) -> Generator: """ - Pops all messages until the most recent one. + Pops all messages until the most recent one and receive the latest. - Used for slow processes to ensure realtime processing. + Used in the case that a particular subscriber cares only about the very + last message and other subscribers care about all the messages in the + queue. + For another use case, when you only want to store the latest message + and all subscribers are interested only in the latest, then use + high water mark max_msgs_num = 1. + + :param timeout: time waiting for a message. If none arrived until the + timeout then None will be returned. (secs) + :param retries: number of tries to check if a msg arrived in the queue """ assert isinstance(self.queue, persipubsub.queue._Queue) assert isinstance(self.queue.env, lmdb.Environment) @@ -110,3 +114,20 @@ def pop_to_top(self) -> None: for _ in range(msg_to_pop_num): self.queue.pop(sub_id=self.sub_id) + + msg = None + end = int(datetime.datetime.utcnow().timestamp()) + timeout + assert isinstance(self.queue, persipubsub.queue._Queue) + assert isinstance(self.sub_id, str) + try: + while int(datetime.datetime.utcnow().timestamp()) <= end: + msg = self.queue.front(sub_id=self.sub_id) + if msg is not None: + break + time.sleep(timeout / retries) + yield msg + finally: + pass + + if msg is not None: + self._pop() diff --git a/tests/test_environment.py b/tests/test_environment.py index d505e94..e1a6285 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -31,7 +31,6 @@ def test_new_publisher(self): with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.environment.new_environment(path=tmp_dir.path) ctl = env.new_control() - # TODO(snaji): icontract? optional, random number!!! ctl.init() pub = env.new_publisher() self.assertIsInstance(pub, persipubsub.publisher.Publisher) @@ -40,7 +39,6 @@ def test_new_subscriber(self): with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.environment.new_environment(path=tmp_dir.path) ctl = env.new_control() - # TODO(snaji): icontract? optional ctl.init() sub = env.new_subscriber(identifier="sub") self.assertIsInstance(sub, persipubsub.subscriber.Subscriber) diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index 8e08b85..e25be4d 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -115,6 +115,26 @@ def test_pop_when_empty(self): self.assertRaises(RuntimeError, sub._pop) + def test_receive_to_top(self): + with temppathlib.TemporaryDirectory() as tmp_dir: + _ = setup(path=tmp_dir.path, sub_list=['sub']) + + queue = persipubsub.queue._Queue() + queue.init(path=tmp_dir.path) + + sub = persipubsub.subscriber.Subscriber() + sub.init(sub_id='sub', path=tmp_dir.path) + + msg1 = "I'm a message".encode(tests.ENCODING) + queue.put(msg=msg1) + + msg2 = "I'm a message too".encode(tests.ENCODING) + queue.put(msg=msg2) + + with sub.receive_to_top() as msg: + self.assertIsNotNone(msg) + self.assertEqual(msg2, msg) + if __name__ == '__main__': unittest.main() From 271cead3fcd392d8b56b496978d6412c68ff480e Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 5 Feb 2019 14:12:01 +0100 Subject: [PATCH 23/35] updated Readme --- README.rst | 154 ++++++++++-------- docs/source/persipubsub/index.rst | 1 + docs/source/persipubsub/persipubsub.queue.rst | 5 + persipubsub/control.py | 24 +-- persipubsub/environment.py | 8 +- persipubsub/queue.py | 82 ++++++---- persipubsub/subscriber.py | 32 ++-- tests/test_control.py | 4 +- tests/test_publisher.py | 4 +- tests/test_queue.py | 30 ++-- tests/test_subscriber.py | 18 +- 11 files changed, 197 insertions(+), 165 deletions(-) create mode 100644 docs/source/persipubsub/persipubsub.queue.rst diff --git a/README.rst b/README.rst index a0c7b48..d54626b 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,17 @@ persipubsub =========== -.. image:: https://badges.frapsoft.com/os/mit/mit.png?v=103 - :target: https://opensource.org/licenses/mit-license.php - :alt: MIT License +.. image:: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :target: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :alt: Build Status + +.. image:: https://coveralls.io/repos/github/Parquery/persipubsub/badge.svg?branch=master + :target: https://coveralls.io/github/Parquery/persipubsub?branch=master + :alt: Coverage + +.. image:: https://readthedocs.org/projects/persipubsub/badge/?version=latest + :target: https://persipubsub.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status .. image:: https://badge.fury.io/py/persipubsub.svg :target: https://badge.fury.io/py/persipubsub @@ -12,9 +20,10 @@ persipubsub .. image:: https://img.shields.io/pypi/pyversions/persipubsub.svg :alt: PyPI - Python Version -.. image:: https://readthedocs.org/projects/persipubsub/badge/?version=latest - :target: https://persipubsub.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status +.. image:: https://badges.frapsoft.com/os/mit/mit.png?v=103 + :target: https://opensource.org/licenses/mit-license.php + :alt: MIT License + ``persipubsub`` implements a persistent, thread-safe and process-safe `lmdb @@ -80,17 +89,31 @@ Usage The usage of the library consists of two steps: deployment and runtime +Environment +----------- + +For improve the accessibility of the library, an environment class lets you +create and initialize any ``persipubsub`` component which you need in +deployment or runtime step. + +Initialize environment +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + Deployment ---------- -In the deployment stage the library sets up the queue structure which is saved -in a JSON file. +In the deployment stage the library sets up the queue structure with the control. -config.json -^^^^^^^^^^^ +Control +^^^^^^^ -Publishers, subscribers and queues need to be defined before runtime in the -config JSON file. +A control unit to initialize and maintain queues. .. note:: @@ -100,65 +123,50 @@ config JSON file. prune_first, which deletes the oldest messages, and prune_last, which deletes the latest messages. -.. code-block:: json - - { - "pub": { - "out_queue": "/home/user/queues/queue", - "subscribers": ["sub"] - }, - "sub": { - "in_queue": "/home/user/queues/queue" - }, - "queues": { - "/home/user/queues/queue": { - "path": "/home/user/queues/queue", - "max_reader_num": 1024, - "max_db_num": 1024, - "max_db_size_bytes": 34359738368, - "subscribers": ["sub"], - "high-water-mark": { - "MSG_TIMEOUT_SECS": 600, - "MAX_MSGS_NUM": 10000, - "HWM_LMDB_SIZE_BYTES": 1000000, - "strategy": "prune_first" - } - } - } - } - -Control -^^^^^^^ - -A control unit to initialize and maintain queues. - -Initialize all queues -""""""""""""""""""""" +Initialize queue +"""""""""""""""" .. code-block:: python - import persipubsub.control + import persipubsub.environment + import persipubsub.queue - persipubsub.control.initialize_all_dbs(config_pth="/home/user/config.json") + env = persipubsub.environment.new_environment(path="/home/user/new-queue/") + + # Initialize a queue with default values. + control = env.new_control() + # Define all optional parameters of the queue. + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first + control = env.new_control(subscriber_ids=["sub1", "sub2"], max_readers=2, + max_size=10*4096, high_watermark=hwm, + strategy=strategy) Prune all dangling messages """"""""""""""""""""""""""" .. code-block:: python - import persipubsub.control + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-dangling-messages/") + control = env.new_control() - persipubsub.control.prune_dangling_messages( - config_pth="/home/user/config.json") + control.prune_dangling_messages() Clear all messages """""""""""""""""" .. code-block:: python - import persipubsub.control + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-subscribers-and-messages/") + control = env.new_control() - persipubsub.control.clear_all_subs(config_pth="/home/user/config.json") + control.clear_all_subscribers() Runtime @@ -184,10 +192,11 @@ be initialized as following. .. code-block:: python - import persipubsub.publisher + import persipubsub.environment - pub = persipubsub.publisher.Pub() - pub.init(pub_id="pub", config_pth="/home/user/config.json") + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + pub = env.new_publisher() Send a message """""""""""""" @@ -197,7 +206,7 @@ Send a message msg = "Hello there!".encode('utf-8') pub.send(msg=msg) - # subscribers have now a message in the queue. + # Subscribers have now a message in the queue. Send many messages at once """""""""""""""""""""""""" @@ -222,38 +231,43 @@ be initialized as following. .. code-block:: python - import persipubsub.subscriber + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") - sub = persipubsub.subscriber.Sub() - sub.init(sub_id="sub", config_pth="/home/user/config.json") + sub = env.new_subscriber(identifier="sub") Receive a message """"""""""""""""" .. code-block:: python - # one message in queue + # One message in queue with sub.receive() as msg: # do something with the message print(msg) # b'Hello there!' - # sub queue is now empty + # This subscriber's queue is now empty Catch up with latest message """""""""""""""""""""""""""" -In case when the subscriber's loop is spinning slower then the publisher's, -there is a possibility receive the latest message and discard the others ones. +Used in the case that a particular subscriber cares only about the very last +message and other subscribers care about all the messages in the queue. + +.. note:: + For the other use case, when you only want to store the latest message and all + subscribers are interested only in the latest, then use high water mark + max_msgs_num = 1. .. code-block:: python - # many outdated messages in queue - sub.pop_to_top() - # most recent message left in queue - with sub.receive() as msg: - # do something with the message + # Many outdated messages in queue + + with sub.receive_to_top() as msg: + # do something with the latest message - # sub queue is now empty + # This subscriber's queue is now empty. Documentation ============= diff --git a/docs/source/persipubsub/index.rst b/docs/source/persipubsub/index.rst index e37145e..f9bce07 100644 --- a/docs/source/persipubsub/index.rst +++ b/docs/source/persipubsub/index.rst @@ -6,6 +6,7 @@ persipubsub persipubsub.ctl persipubsub.env + persipubsub.queue persipubsub.publisher persipubsub.subscriber persipubsub \ No newline at end of file diff --git a/docs/source/persipubsub/persipubsub.queue.rst b/docs/source/persipubsub/persipubsub.queue.rst new file mode 100644 index 0000000..3777523 --- /dev/null +++ b/docs/source/persipubsub/persipubsub.queue.rst @@ -0,0 +1,5 @@ +High Water Mark and Strategy +**************************** + +.. automodule:: persipubsub.queue + :members: \ No newline at end of file diff --git a/persipubsub/control.py b/persipubsub/control.py index a2fd63a..c2914b1 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -38,7 +38,7 @@ def set_queue_parameters(max_reader_num: int, max_db_num: int, db=queue_db) -def set_hwm(hwm: persipubsub.queue._HighWaterMark, +def set_hwm(hwm: persipubsub.queue.HighWaterMark, env: lmdb.Environment) -> None: """ Set high water mark values for queue. @@ -62,7 +62,7 @@ def set_hwm(hwm: persipubsub.queue._HighWaterMark, db=queue_db) -def set_strategy(strategy: persipubsub.queue._Strategy, +def set_strategy(strategy: persipubsub.queue.Strategy, env: lmdb.Environment) -> None: """ Set pruning strategy for queue. @@ -124,10 +124,10 @@ def init(self, subscriber_ids: Optional[Sequence[str]] = None, max_readers: int = 1024, max_size: int = 32 * 1024**3, - high_watermark: persipubsub.queue._HighWaterMark = persipubsub. - queue._HighWaterMark(), - strategy: persipubsub.queue._Strategy = persipubsub.queue. - _Strategy.prune_first) -> None: + high_watermark: persipubsub.queue.HighWaterMark = persipubsub. + queue.HighWaterMark(), + strategy: persipubsub.queue.Strategy = persipubsub.queue.Strategy. + prune_first) -> None: """ Initialize control with a (re)initialized queue. @@ -154,8 +154,8 @@ def _reinitialize_queue(self) -> None: """Reinitialize the queue which is maintained by the control.""" self.queue = persipubsub.queue._Queue() self.queue.init(path=self.path) - assert isinstance(self.queue.sub_list, List) - self.subscriber_ids = set(self.queue.sub_list) + assert isinstance(self.queue.subscriber_ids, List) + self.subscriber_ids = set(self.queue.subscriber_ids) # pylint: disable=too-many-arguments @icontract.require(lambda max_readers: max_readers >= 0) @@ -169,9 +169,9 @@ def _initialize_queue(self, max_readers: int = 1024, max_size: int = 32 * 1024**3, high_watermark: persipubsub.queue. - _HighWaterMark = persipubsub.queue._HighWaterMark(), - strategy: persipubsub.queue._Strategy = persipubsub. - queue._Strategy.prune_first) -> None: + HighWaterMark = persipubsub.queue.HighWaterMark(), + strategy: persipubsub.queue.Strategy = persipubsub. + queue.Strategy.prune_first) -> None: """ Initialize queue. @@ -261,7 +261,7 @@ def prune_dangling_messages(self) -> None: """Prune all dangling messages from the lmdb.""" assert isinstance(self.queue, persipubsub.queue._Queue) persipubsub.queue._prune_dangling_messages_for( - queue=self.queue, sub_list=list(self.subscriber_ids)) + queue=self.queue, subscriber_ids=list(self.subscriber_ids)) # pylint: disable=too-many-locals def _prune_all_messages_for(self, sub_id: str) -> None: diff --git a/persipubsub/environment.py b/persipubsub/environment.py index 7a60c40..a0e65ad 100644 --- a/persipubsub/environment.py +++ b/persipubsub/environment.py @@ -32,9 +32,9 @@ def new_control(self, max_readers: int = 1024, max_size: int = 32 * 1024**3, high_watermark: persipubsub.queue. - _HighWaterMark = persipubsub.queue._HighWaterMark(), - strategy: persipubsub.queue._Strategy = persipubsub.queue. - _Strategy.prune_first) -> persipubsub.control.Control: + HighWaterMark = persipubsub.queue.HighWaterMark(), + strategy: persipubsub.queue.Strategy = persipubsub.queue. + Strategy.prune_first) -> persipubsub.control.Control: """ Fabricate a new control. @@ -69,7 +69,7 @@ def new_subscriber(self, :return: Subscriber to receive messages """ subscriber = persipubsub.subscriber.Subscriber() - subscriber.init(sub_id=identifier, path=self.path) + subscriber.init(identifier=identifier, path=self.path) return subscriber diff --git a/persipubsub/queue.py b/persipubsub/queue.py index e28fbeb..01163fa 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -15,27 +15,27 @@ HWM_LMDB_SIZE_BYTES = 30 * 1024**3 # type: int -class _Strategy(enum.Enum): +class Strategy(enum.Enum): """Store possible strategies.""" prune_first = 0 prune_last = 1 -def _parse_strategy(strategy: str) -> _Strategy: +def _parse_strategy(strategy: str) -> Strategy: """ Parse overflow strategy. :param strategy: Strategy stored in config :return: set overflow strategy """ - if _Strategy.prune_first.name == strategy: - return _Strategy.prune_first + if Strategy.prune_first.name == strategy: + return Strategy.prune_first - return _Strategy.prune_last + return Strategy.prune_last -class _HighWaterMark: +class HighWaterMark: """Store high water mark limits.""" def __init__(self, @@ -77,6 +77,14 @@ def _initialize_environment( raise RuntimeError( "The queue directory does not exist: {}".format(queue_dir)) + # max_spare_txn: Read-only transactions to cache after becoming unused. + # Caching transactions avoids two allocations, one lock and linear scan of + # the shared environment per invocation of begin(), Transaction, get(), + # gets(), or cursor(). Should match the process’s maximum expected + # concurrent transactions (e.g. thread count). + # Setting max_spare_txn equals 0 doesn't allow any caching for read-only + # transactions to avoid the error: + # MDB_BAD_RSLOT: Invalid reuse of reader locktable slot. env = lmdb.open( path=queue_dir.as_posix(), map_size=max_db_size_bytes, @@ -87,12 +95,13 @@ def _initialize_environment( return env -def _prune_dangling_messages_for(queue: '_Queue', sub_list: List[str]) -> None: +def _prune_dangling_messages_for(queue: '_Queue', + subscriber_ids: List[str]) -> None: """ Prune all dangling messages for subscribers of a queue from lmdb. :param queue: of which dangling messages should be pruned - :param sub_list: subscribers of which dangling messages should be pruned + :param subscriber_ids: subscribers of which dangling msgs should be pruned """ assert isinstance(queue.env, lmdb.Environment) with queue.env.begin(write=True) as txn: @@ -117,7 +126,7 @@ def _prune_dangling_messages_for(queue: '_Queue', sub_list: List[str]) -> None: # subscriber might still await this messages after the timeout. # This messages needs also to be removed from all subscribers. msgs_to_delete_timeout = set() - assert isinstance(queue.hwm, _HighWaterMark) + assert isinstance(queue.hwm, HighWaterMark) with queue.env.begin(db=meta_db) as txn: cursor = txn.cursor() @@ -135,7 +144,7 @@ def _prune_dangling_messages_for(queue: '_Queue', sub_list: List[str]) -> None: txn.delete(key=key, db=data_db) with queue.env.begin(write=True) as txn: - for sub_id in sub_list: + for sub_id in subscriber_ids: sub_db = queue.env.open_db( key=persipubsub.encoding(sub_id), txn=txn, create=False) for key in msgs_to_delete_timeout: @@ -156,7 +165,7 @@ class _Queue: Strategy which will be used to remove messages when high water mark is reached. :vartype strategy: Strategy - :ivar sub_list: all subscribers of the queue + :ivar subscriber_ids: all subscribers of the queue :vartype sub_list: List[str] """ @@ -164,9 +173,9 @@ def __init__(self) -> None: """Initialize class object.""" self.path = None # type: Optional[pathlib.Path] self.env = None # type: Optional[lmdb.Environment] - self.hwm = None # type: Optional[_HighWaterMark] - self.strategy = None # type: Optional[_Strategy] - self.sub_list = None # type: Optional[List[str]] + self.hwm = None # type: Optional[HighWaterMark] + self.strategy = None # type: Optional[Strategy] + self.subscriber_ids = None # type: Optional[List[str]] # pylint: disable=too-many-arguments # pylint: disable=too-many-locals @@ -221,7 +230,7 @@ def init(self, hwm_lmdb_size_bytes = persipubsub.bytes_to_int( persipubsub.get_queue_data( path=self.path, key=persipubsub.HWM_DB_SIZE_BYTES_KEY)) - self.hwm = _HighWaterMark( + self.hwm = HighWaterMark( msg_timeout_secs=msg_timeout_secs, max_msgs_num=max_msgs_num, hwm_lmdb_size_bytes=hwm_lmdb_size_bytes) @@ -235,9 +244,9 @@ def init(self, path=self.path, key=persipubsub.SUBSCRIBER_IDS_KEY) if subscriber_list is None: - self.sub_list = [] + self.subscriber_ids = [] else: - self.sub_list = persipubsub.decoding( + self.subscriber_ids = persipubsub.decoding( encoded_str=subscriber_list).split(' ') def __enter__(self) -> '_Queue': @@ -259,13 +268,13 @@ def put(self, msg: bytes) -> None: self.vacuum() msg_id = str(datetime.datetime.utcnow().timestamp()) + str(uuid.uuid4()) assert isinstance(self.env, lmdb.Environment) - assert isinstance(self.sub_list, List) + assert isinstance(self.subscriber_ids, List) with self.env.begin(write=True) as txn: pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) txn.put( key=persipubsub.encoding(msg_id), - value=persipubsub.int_to_bytes(len(self.sub_list)), + value=persipubsub.int_to_bytes(len(self.subscriber_ids)), db=pending_db) meta_db = self.env.open_db( @@ -280,7 +289,7 @@ def put(self, msg: bytes) -> None: key=persipubsub.DATA_DB, txn=txn, create=False) txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) - for sub in self.sub_list: + for sub in self.subscriber_ids: sub_db = self.env.open_db( key=persipubsub.encoding(sub), txn=txn, create=False) txn.put(key=persipubsub.encoding(msg_id), db=sub_db) @@ -296,7 +305,7 @@ def put_many_flush_once(self, msgs: List[bytes]) -> None: # every publisher always prunes queue before sending a message. self.vacuum() assert isinstance(self.env, lmdb.Environment) - assert isinstance(self.sub_list, List) + assert isinstance(self.subscriber_ids, List) with self.env.begin(write=True) as txn: pending_db = self.env.open_db( @@ -309,7 +318,7 @@ def put_many_flush_once(self, msgs: List[bytes]) -> None: key=persipubsub.DATA_DB, txn=txn, create=False) sub_dbs = set() - for sub in self.sub_list: + for sub in self.subscriber_ids: sub_dbs.add( self.env.open_db( key=persipubsub.encoding(sub), txn=txn, create=False)) @@ -320,7 +329,7 @@ def put_many_flush_once(self, msgs: List[bytes]) -> None: txn.put( key=persipubsub.encoding(msg_id), - value=persipubsub.int_to_bytes(len(self.sub_list)), + value=persipubsub.int_to_bytes(len(self.subscriber_ids)), db=pending_db) txn.put( @@ -334,19 +343,19 @@ def put_many_flush_once(self, msgs: List[bytes]) -> None: for sub_db in sub_dbs: txn.put(key=persipubsub.encoding(msg_id), db=sub_db) - def front(self, sub_id: str) -> Optional[bytes]: + def front(self, identifier: str) -> Optional[bytes]: """ Peek at next message in lmdb queue. Load from LMDB queue into memory and process msg afterwards. - :param sub_id: Subscriber ID + :param identifier: Subscriber ID :return: """ assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=False) as txn: sub_db = self.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) + key=persipubsub.encoding(identifier), txn=txn, create=False) data_db = self.env.open_db( key=persipubsub.DATA_DB, txn=txn, create=False) @@ -360,17 +369,17 @@ def front(self, sub_id: str) -> Optional[bytes]: return msg - def pop(self, sub_id: str) -> None: + def pop(self, identifier: str) -> None: """ Remove msg from the subscriber's queue and reduce pending subscribers. - :param sub_id: Subscriber ID + :param identifier: Subscriber ID :return: """ assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=True) as txn: sub_db = self.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) + key=persipubsub.encoding(identifier), txn=txn, create=False) pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) @@ -396,8 +405,9 @@ def prune_dangling_messages(self) -> None: :return: """ - assert isinstance(self.sub_list, List) - _prune_dangling_messages_for(queue=self, sub_list=self.sub_list) + assert isinstance(self.subscriber_ids, List) + _prune_dangling_messages_for( + queue=self, subscriber_ids=self.subscriber_ids) def check_current_lmdb_size(self) -> int: """ @@ -442,7 +452,7 @@ def vacuum(self) -> None: :return: """ - assert isinstance(self.hwm, _HighWaterMark) + assert isinstance(self.hwm, HighWaterMark) self.prune_dangling_messages() msgs_num = self.count_msgs() if msgs_num >= self.hwm.max_msgs_num: @@ -469,7 +479,7 @@ def prune_messages(self) -> None: entries = meta_stat['entries'] cursor = txn.cursor(db=meta_db) - if self.strategy == _Strategy.prune_first: + if self.strategy == Strategy.prune_first: cursor.first() for index, key in enumerate( @@ -478,7 +488,7 @@ def prune_messages(self) -> None: if index >= int(entries / 2): break - elif self.strategy == _Strategy.prune_last: + elif self.strategy == Strategy.prune_last: cursor.last() for index, key in enumerate( cursor.iterprev(keys=True, values=False)): @@ -488,7 +498,7 @@ def prune_messages(self) -> None: else: raise RuntimeError("Pruning strategy not set.") - assert isinstance(self.sub_list, List) + assert isinstance(self.subscriber_ids, List) with self.env.begin(write=True) as txn: pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) @@ -499,7 +509,7 @@ def prune_messages(self) -> None: dbs = [pending_db, meta_db, data_db] - for sub in self.sub_list: + for sub in self.subscriber_ids: sub_db = self.env.open_db( key=persipubsub.encoding(sub), txn=txn, create=False) dbs.append(sub_db) diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index b0756ee..02896ef 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -19,7 +19,7 @@ class Subscriber: """ Create Subscriber ready to receive messages. - :ivar sub_id: subscriber ID + :ivar identifier: subscriber ID :vartype pub_id: str :ivar queue: from which messages are received :vartype queue: persipubsub.queue.Queue @@ -27,18 +27,18 @@ class Subscriber: def __init__(self) -> None: """Initialize class object.""" - self.sub_id = None # type: Optional[str] + self.identifier = None # type: Optional[str] self.queue = None # type: Optional[persipubsub.queue._Queue] - def init(self, sub_id: str, path: Union[pathlib.Path, str]) -> None: + def init(self, identifier: str, path: Union[pathlib.Path, str]) -> None: """ Initialize. - :param sub_id: unique subscriber id + :param identifier: unique subscriber id :param path: path to the queue """ - self.sub_id = sub_id - assert isinstance(self.sub_id, str) + self.identifier = identifier + assert isinstance(self.identifier, str) self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access self.queue.init(path=path) assert isinstance(self.queue, persipubsub.queue._Queue) @@ -64,10 +64,10 @@ def receive(self, timeout: int = 60, retries: int = 10) -> Generator: msg = None end = int(datetime.datetime.utcnow().timestamp()) + timeout assert isinstance(self.queue, persipubsub.queue._Queue) - assert isinstance(self.sub_id, str) + assert isinstance(self.identifier, str) try: while int(datetime.datetime.utcnow().timestamp()) <= end: - msg = self.queue.front(sub_id=self.sub_id) + msg = self.queue.front(identifier=self.identifier) if msg is not None: break time.sleep(timeout / retries) @@ -81,8 +81,8 @@ def receive(self, timeout: int = 60, retries: int = 10) -> Generator: def _pop(self) -> None: """Pop a message from the subscriber's lmdb.""" assert isinstance(self.queue, persipubsub.queue._Queue) - assert isinstance(self.sub_id, str) - self.queue.pop(sub_id=self.sub_id) + assert isinstance(self.identifier, str) + self.queue.pop(identifier=self.identifier) @icontract.require(lambda timeout: timeout > 0) @icontract.require(lambda retries: retries > 0) @@ -104,24 +104,26 @@ def receive_to_top(self, timeout: int = 60, retries: int = 10) -> Generator: """ assert isinstance(self.queue, persipubsub.queue._Queue) assert isinstance(self.queue.env, lmdb.Environment) - assert isinstance(self.sub_id, str) + assert isinstance(self.identifier, str) with self.queue.env.begin(write=False) as txn: sub_db = self.queue.env.open_db( - key=persipubsub.encoding(self.sub_id), txn=txn, create=False) + key=persipubsub.encoding(self.identifier), + txn=txn, + create=False) sub_stat = txn.stat(db=sub_db) # pop all message except the most recent one msg_to_pop_num = sub_stat['entries'] - 1 for _ in range(msg_to_pop_num): - self.queue.pop(sub_id=self.sub_id) + self.queue.pop(identifier=self.identifier) msg = None end = int(datetime.datetime.utcnow().timestamp()) + timeout assert isinstance(self.queue, persipubsub.queue._Queue) - assert isinstance(self.sub_id, str) + assert isinstance(self.identifier, str) try: while int(datetime.datetime.utcnow().timestamp()) <= end: - msg = self.queue.front(sub_id=self.sub_id) + msg = self.queue.front(identifier=self.identifier) if msg is not None: break time.sleep(timeout / retries) diff --git a/tests/test_control.py b/tests/test_control.py index b6b9b12..0feaadd 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -21,8 +21,8 @@ def setup(path: pathlib.Path, """Create an initialized control""" control = persipubsub.control.Control(path=path) - hwm = persipubsub.queue._HighWaterMark() - strategy = persipubsub.queue._Strategy.prune_first + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first control.init( subscriber_ids=sub_list, diff --git a/tests/test_publisher.py b/tests/test_publisher.py index df5270b..a32ab7a 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -21,8 +21,8 @@ def setup(path: pathlib.Path, """Create an initialized control""" control = persipubsub.control.Control(path=path) - hwm = persipubsub.queue._HighWaterMark() - strategy = persipubsub.queue._Strategy.prune_first + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first control.init( subscriber_ids=sub_list, diff --git a/tests/test_queue.py b/tests/test_queue.py index f9b1008..1342bee 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -21,11 +21,11 @@ def setup(path: pathlib.Path, """Create an initialized control""" control = persipubsub.control.Control(path=path) - hwm = persipubsub.queue._HighWaterMark( + hwm = persipubsub.queue.HighWaterMark( msg_timeout_secs=tests.TEST_MSG_TIMEOUT, max_msgs_num=tests.TEST_HWM_MSG_NUM, hwm_lmdb_size_bytes=tests.TEST_HWM_LMDB_SIZE) - strategy = persipubsub.queue._Strategy.prune_first + strategy = persipubsub.queue.Strategy.prune_first control.init( subscriber_ids=sub_list, @@ -185,7 +185,7 @@ def test_front(self): # pylint: disable=assignment-from-none # pylint: disable=assignment-from-no-return - received_msg = queue.front(sub_id=subscriber) + received_msg = queue.front(identifier=subscriber) self.assertIsNotNone(received_msg) self.assertEqual(msg, received_msg) @@ -202,7 +202,7 @@ def test_pop(self): # pylint: disable=assignment-from-none # pylint: disable=assignment-from-no-return - received_msg = queue.front(sub_id=subscriber) + received_msg = queue.front(identifier=subscriber) self.assertIsNotNone(received_msg) with queue.env.begin() as txn: @@ -213,9 +213,9 @@ def test_pop(self): self.assertTrue(cursor.first()) pending_before_pop = cursor.value() - queue.pop(sub_id=subscriber) + queue.pop(identifier=subscriber) - received_msg = queue.front(sub_id=subscriber) + received_msg = queue.front(identifier=subscriber) self.assertIsNone(received_msg) with queue.env.begin() as txn: @@ -238,7 +238,7 @@ def test_pop_queue_empty(self): queue = persipubsub.queue._Queue() queue.init(path=tmp_dir.path) - self.assertRaises(RuntimeError, queue.pop, sub_id=subscriber) + self.assertRaises(RuntimeError, queue.pop, identifier=subscriber) def test_queue_initialisation(self): with temppathlib.TemporaryDirectory() as tmp_dir: @@ -253,9 +253,9 @@ def test_queue_initialisation(self): queue.hwm.hwm_lmdb_size_bytes) self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.hwm.max_msgs_num) self.assertEqual(tests.TEST_MSG_TIMEOUT, queue.hwm.msg_timeout_secs) - self.assertEqual(persipubsub.queue._Strategy.prune_first.name, + self.assertEqual(persipubsub.queue.Strategy.prune_first.name, queue.strategy.name) - self.assertEqual(['sub'], queue.sub_list) + self.assertEqual(['sub'], queue.subscriber_ids) def test_overflow_msgs_limit(self): with temppathlib.TemporaryDirectory() as tmp_dir: @@ -326,7 +326,7 @@ def test_strategy_prune_first(self): queue = persipubsub.queue._Queue() queue.init(path=tmp_dir.path) - queue.strategy = persipubsub.queue._Strategy.prune_first + queue.strategy = persipubsub.queue.Strategy.prune_first queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM @@ -335,7 +335,7 @@ def test_strategy_prune_first(self): queue.put(msg=msg) self.assertEqual("secret message 0".encode(tests.ENCODING), - queue.front(sub_id='sub')) + queue.front(identifier='sub')) msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( tests.ENCODING) @@ -344,7 +344,7 @@ def test_strategy_prune_first(self): self.assertEqual( "secret message {}".format( int((tests.TEST_HWM_MSG_NUM / 2) + 1)).encode( - tests.ENCODING), queue.front(sub_id='sub')) + tests.ENCODING), queue.front(identifier='sub')) def test_strategy_prune_last(self): with temppathlib.TemporaryDirectory() as tmp_dir: @@ -352,7 +352,7 @@ def test_strategy_prune_last(self): queue = persipubsub.queue._Queue() queue.init(path=tmp_dir.path) - queue.strategy = persipubsub.queue._Strategy.prune_last + queue.strategy = persipubsub.queue.Strategy.prune_last queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM @@ -361,14 +361,14 @@ def test_strategy_prune_last(self): queue.put(msg=msg) self.assertEqual("secret message 0".encode(tests.ENCODING), - queue.front(sub_id='sub')) + queue.front(identifier='sub')) msg = "secret message {}".format(tests.TEST_HWM_MSG_NUM).encode( tests.ENCODING) queue.put(msg=msg) self.assertEqual("secret message 0".encode(tests.ENCODING), - queue.front(sub_id='sub')) + queue.front(identifier='sub')) if __name__ == '__main__': diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index e25be4d..03c2ff0 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -21,8 +21,8 @@ def setup(path: pathlib.Path, """Create an initialized control""" control = persipubsub.control.Control(path=path) - hwm = persipubsub.queue._HighWaterMark() - strategy = persipubsub.queue._Strategy.prune_first + hwm = persipubsub.queue.HighWaterMark() + strategy = persipubsub.queue.Strategy.prune_first control.init( subscriber_ids=sub_list, @@ -43,7 +43,7 @@ def test_receive_message(self): queue.init(path=tmp_dir.path) sub = persipubsub.subscriber.Subscriber() - sub.init(sub_id="sub", path=tmp_dir.path) + sub.init(identifier="sub", path=tmp_dir.path) msg = "Hello World!".encode(tests.ENCODING) queue.put(msg=msg) @@ -60,7 +60,7 @@ def test_timeout_subscriber(self): queue.init(path=tmp_dir.path) sub = persipubsub.subscriber.Subscriber() - sub.init(sub_id='sub', path=tmp_dir.path) + sub.init(identifier='sub', path=tmp_dir.path) with sub.receive(timeout=1) as received_msg: self.assertIsNone(received_msg) @@ -71,7 +71,7 @@ def test_timeout_subscriber(self): self.assertEqual( "message send after timeout and will not be popped".encode( - tests.ENCODING), queue.front(sub_id='sub')) + tests.ENCODING), queue.front(identifier='sub')) with sub.receive(timeout=1) as received_msg: self.assertIsNotNone(received_msg) @@ -79,7 +79,7 @@ def test_timeout_subscriber(self): "message send after timeout and will not be popped".encode( tests.ENCODING), received_msg) - self.assertIsNone(queue.front(sub_id='sub')) + self.assertIsNone(queue.front(identifier='sub')) def test_pop(self): with temppathlib.TemporaryDirectory() as tmp_dir: @@ -89,7 +89,7 @@ def test_pop(self): queue.init(path=tmp_dir.path) sub = persipubsub.subscriber.Subscriber() - sub.init(sub_id='sub', path=tmp_dir.path) + sub.init(identifier='sub', path=tmp_dir.path) msg1 = "I'm a message".encode(tests.ENCODING) queue.put(msg=msg1) @@ -111,7 +111,7 @@ def test_pop_when_empty(self): queue.init(path=tmp_dir.path) sub = persipubsub.subscriber.Subscriber() - sub.init(sub_id='sub', path=tmp_dir.path) + sub.init(identifier='sub', path=tmp_dir.path) self.assertRaises(RuntimeError, sub._pop) @@ -123,7 +123,7 @@ def test_receive_to_top(self): queue.init(path=tmp_dir.path) sub = persipubsub.subscriber.Subscriber() - sub.init(sub_id='sub', path=tmp_dir.path) + sub.init(identifier='sub', path=tmp_dir.path) msg1 = "I'm a message".encode(tests.ENCODING) queue.put(msg=msg1) From e41f3cdf07f29db4f38ff8b8a2652b9406514f70 Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 5 Feb 2019 16:16:37 +0100 Subject: [PATCH 24/35] docstring fix --- persipubsub/control.py | 2 ++ persipubsub/environment.py | 16 +++++++++++++--- persipubsub/publisher.py | 5 ----- persipubsub/queue.py | 2 +- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/persipubsub/control.py b/persipubsub/control.py index c2914b1..88869f1 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -135,6 +135,7 @@ def init(self, :param max_readers: max number of reader of the lmdb :param max_size: max size of the lmdb in bytes :param high_watermark: high water mark limit of the queue + :param strategy: used to prune queue :return: """ if self.check_queue_is_initialized(): @@ -179,6 +180,7 @@ def _initialize_queue(self, :param max_readers: max number of reader of the lmdb :param max_size: max size of the lmdb in bytes :param high_watermark: high water mark limit of the queue + :param strategy: used to prune queue :return: """ if not self.path.exists(): diff --git a/persipubsub/environment.py b/persipubsub/environment.py index a0e65ad..97cadb8 100644 --- a/persipubsub/environment.py +++ b/persipubsub/environment.py @@ -15,8 +15,12 @@ class Environment: """Fabricate persipubsub components.""" - def __init__(self, path: pathlib.Path): - """Initialize.""" + def __init__(self, path: pathlib.Path) -> None: + """ + Initialize. + + :param path: to the queue + """ self.path = path def __enter__(self) -> 'Environment': @@ -38,7 +42,12 @@ def new_control(self, """ Fabricate a new control. - :return: Control to initialize queues + :param subscriber_ids: subscribers of the queue + :param max_readers: max number of reader of the lmdb + :param max_size: max size of the lmdb in bytes + :param high_watermark: high water mark limit of the queue + :param strategy: used to prune queue + :return: Control to create and maintain queue """ control = persipubsub.control.Control(path=self.path) control.init( @@ -54,6 +63,7 @@ def new_publisher( """ Fabricate a new publisher. + :param autosync: if True, store data automatically in lmdb :return: Publisher to send messages """ publisher = persipubsub.publisher.Publisher() diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index 4b82aa6..11d2155 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -13,12 +13,8 @@ class Publisher: """ Create Publisher ready to send messages. - :ivar pub_id: publisher ID - :vartype pub_id: str :ivar queue: on which messages are published :vartype queue: persipubsub.queue.Queue - :ivar sub_list: list of subscriber to which messages are send - :vartype sub_list: List[str] :ivar autosync: sync after each message or after multiple messages :vartype autosync: bool """ @@ -33,7 +29,6 @@ def init(self, path: Union[pathlib.Path, str], """ Initialize. - :param pub_id: constant ID of publisher :param path: path to the queue :param autosync: if True, store data automatically in lmdb """ diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 01163fa..a55ac64 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -155,7 +155,7 @@ class _Queue: """ Queue messages persistently from many publishers for many subscribers. - :ivar config_pth: path to the JSON config file + :ivar path: to the queue :vartype config_pth: pathlib.Path :ivar env: LMDB environment on disk :vartype env: lmdb.Environment From 7c664110b29c936c0cf52c4396af8a54e2d5a776 Mon Sep 17 00:00:00 2001 From: slimeth Date: Mon, 11 Feb 2019 14:18:39 +0100 Subject: [PATCH 25/35] added mypy --strict flag and fixed errors --- persipubsub/__init__.py | 13 ++------ persipubsub/control.py | 12 ++++--- persipubsub/environment.py | 4 +-- persipubsub/publisher.py | 4 +-- persipubsub/queue.py | 67 +++++++++++++++++++++++--------------- persipubsub/subscriber.py | 10 +++--- precommit.py | 2 +- tests/__init__.py | 4 +-- tests/test_control.py | 29 ++++++++++++----- tests/test_environment.py | 8 ++--- tests/test_publisher.py | 9 +++-- tests/test_queue.py | 40 +++++++++++++++-------- tests/test_subscriber.py | 10 +++--- 13 files changed, 128 insertions(+), 84 deletions(-) diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py index ad9a217..f27e54b 100644 --- a/persipubsub/__init__.py +++ b/persipubsub/__init__.py @@ -71,24 +71,17 @@ def bytes_to_int(array_of_bytes: bytes) -> int: return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) -def get_queue_data(path: pathlib.Path, key: bytes) -> bytes: +def get_queue_data(key: bytes, env: lmdb.Environment) -> bytes: """ Get queue data. - :param path: to the queue :param key: for lookup + :param env: environment that stores queue data :return: lookup result """ - env = lmdb.open( - path=path.as_posix(), - map_size=MAX_DB_SIZE_BYTES, - subdir=True, - max_readers=MAX_READER_NUM, - max_dbs=MAX_READER_NUM, - max_spare_txns=0) - with env.begin(write=False) as txn: queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) data = txn.get(key=key, db=queue_db) + assert isinstance(data, bytes) return data diff --git a/persipubsub/control.py b/persipubsub/control.py index 88869f1..6dbe8ce 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -88,7 +88,7 @@ def _add_sub(sub_id: str, env: lmdb.Environment) -> None: with env.begin(write=True) as txn: _ = env.open_db(key=persipubsub.encoding(sub_id), txn=txn, create=True) - queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=False) + queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) subscriber_ids = txn.get( key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) @@ -118,6 +118,7 @@ def __init__(self, path: pathlib.Path): self.path = path self.queue = None # type: Optional[persipubsub.queue._Queue] self.subscriber_ids = set() # type: Set[str] + self.env = None # type: Optional[lmdb.Environment] # pylint: disable=too-many-arguments def init(self, @@ -160,10 +161,10 @@ def _reinitialize_queue(self) -> None: # pylint: disable=too-many-arguments @icontract.require(lambda max_readers: max_readers >= 0) - # yapf: disable # Each named database needs at least one page of 4096 bytes. - @icontract.require(lambda max_size, subscriber_ids: - max_size >= (5 + len(subscriber_ids) * 4096)) + # yapf: disable + @icontract.require(lambda max_size, subscriber_ids: max_size >= ( + 5 + len(subscriber_ids) * 4096)) # yapf: enable def _initialize_queue(self, subscriber_ids: Sequence[str], @@ -204,6 +205,7 @@ def _initialize_queue(self, max_db_size_bytes=max_size, env=env) + self.env = env set_hwm(hwm=high_watermark, env=env) set_strategy(strategy=strategy, env=env) @@ -230,7 +232,7 @@ def check_queue_is_initialized(self) -> bool: for key in keys: try: - value = persipubsub.get_queue_data(path=self.path, key=key) + value = persipubsub.get_queue_data(key=key, env=self.env) except lmdb.NotFoundError: return False diff --git a/persipubsub/environment.py b/persipubsub/environment.py index 97cadb8..8fe0293 100644 --- a/persipubsub/environment.py +++ b/persipubsub/environment.py @@ -2,7 +2,7 @@ """Fabricate new persipubsub components.""" import pathlib -from typing import Optional, Sequence +from typing import Any, Optional, Sequence import persipubsub.control import persipubsub.publisher @@ -27,7 +27,7 @@ def __enter__(self) -> 'Environment': """Enter the context and give environment prepared to constructor.""" return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Leave the context.""" # pylint: disable=too-many-arguments diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index 11d2155..38ebe63 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -2,7 +2,7 @@ """Distribute messages persistent from publisher to subscriber.""" import pathlib -from typing import List, Optional, Union +from typing import Any, List, Optional, Union import persipubsub.queue @@ -40,7 +40,7 @@ def __enter__(self) -> 'Publisher': """Enter the context and give the pub prepared in the constructor.""" return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Leave the context.""" def send(self, msg: bytes) -> None: diff --git a/persipubsub/queue.py b/persipubsub/queue.py index a55ac64..950a3ce 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -4,7 +4,7 @@ import enum import pathlib import uuid -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union import lmdb @@ -91,7 +91,7 @@ def _initialize_environment( subdir=True, max_readers=max_reader_num, max_dbs=max_db_num, - max_spare_txns=0) + max_spare_txns=1) return env @@ -195,19 +195,24 @@ def init(self, self.path = path if isinstance(path, pathlib.Path) \ else pathlib.Path(path) - max_reader_num_queue = max_reader_num if max_reader_num is not None \ - else persipubsub.bytes_to_int( - persipubsub.get_queue_data( - path=self.path, key=persipubsub.MAX_READER_NUM_KEY)) - max_db_num_queue = max_db_num if max_db_num is not None else \ - persipubsub.bytes_to_int( - persipubsub.get_queue_data( - path=self.path, key=persipubsub.MAX_DB_NUM_KEY)) - max_db_size_bytes_queue = max_db_size_bytes if max_db_size_bytes \ - is not None else \ - persipubsub.bytes_to_int( - persipubsub.get_queue_data( - path=self.path, key=persipubsub.MAX_DB_SIZE_BYTES_KEY)) + # max_reader_num_queue = max_reader_num if max_reader_num is not None \ + # else persipubsub.bytes_to_int( + # persipubsub.get_queue_data( + # path=self.path, key=persipubsub.MAX_READER_NUM_KEY)) + # max_db_num_queue = max_db_num if max_db_num is not None else \ + # persipubsub.bytes_to_int( + # persipubsub.get_queue_data( + # path=self.path, key=persipubsub.MAX_DB_NUM_KEY)) + # max_db_size_bytes_queue = max_db_size_bytes if max_db_size_bytes \ + # is not None else \ + # persipubsub.bytes_to_int( + # persipubsub.get_queue_data( + # path=self.path, key=persipubsub.MAX_DB_SIZE_BYTES_KEY)) + + # TODO(snaji): remove + max_reader_num_queue = persipubsub.MAX_READER_NUM + max_db_num_queue = persipubsub.MAX_DB_NUM + max_db_size_bytes_queue = persipubsub.MAX_DB_SIZE_BYTES self.env = _initialize_environment( queue_dir=self.path, @@ -220,29 +225,39 @@ def init(self, _ = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=True) _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) + _ = self.env.open_db(key=persipubsub.QUEUE_DB, txn=txn, create=True) msg_timeout_secs = persipubsub.bytes_to_int( persipubsub.get_queue_data( - path=self.path, key=persipubsub.MSG_TIMEOUT_SECS_KEY)) + key=persipubsub.MSG_TIMEOUT_SECS_KEY, env=self.env)) max_msgs_num = persipubsub.bytes_to_int( persipubsub.get_queue_data( - path=self.path, key=persipubsub.MAX_MSGS_NUM_KEY)) + key=persipubsub.MAX_MSGS_NUM_KEY, env=self.env)) hwm_lmdb_size_bytes = persipubsub.bytes_to_int( persipubsub.get_queue_data( - path=self.path, key=persipubsub.HWM_DB_SIZE_BYTES_KEY)) + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, env=self.env)) + + # TODO(snaji): remove + # msg_timeout_secs = 500 + # max_msgs_num = 10000000000000 + # hwm_lmdb_size_bytes = 30 * 1024**3 self.hwm = HighWaterMark( msg_timeout_secs=msg_timeout_secs, max_msgs_num=max_msgs_num, hwm_lmdb_size_bytes=hwm_lmdb_size_bytes) strategy = persipubsub.get_queue_data( - path=self.path, key=persipubsub.STRATEGY_KEY) + key=persipubsub.STRATEGY_KEY, env=self.env) + + # TODO(snaji): remove + # strategy = "prune_first".encode('utf-8') self.strategy = _parse_strategy( persipubsub.decoding(encoded_str=strategy)) subscriber_list = persipubsub.get_queue_data( - path=self.path, key=persipubsub.SUBSCRIBER_IDS_KEY) - + key=persipubsub.SUBSCRIBER_IDS_KEY, env=self.env) + # TODO(snaji): remove + # self.subscriber_ids = ['sub'] if subscriber_list is None: self.subscriber_ids = [] else: @@ -253,7 +268,7 @@ def __enter__(self) -> '_Queue': """Enter the context and give the queue prepared in the constructor.""" return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Leave the context.""" def put(self, msg: bytes) -> None: @@ -265,7 +280,7 @@ def put(self, msg: bytes) -> None: :return: """ # every publisher always prunes queue before sending a message. - self.vacuum() + # self.vacuum() msg_id = str(datetime.datetime.utcnow().timestamp()) + str(uuid.uuid4()) assert isinstance(self.env, lmdb.Environment) assert isinstance(self.subscriber_ids, List) @@ -367,7 +382,7 @@ def front(self, identifier: str) -> Optional[bytes]: else: msg = None - return msg + return msg # type: ignore def pop(self, identifier: str) -> None: """ @@ -442,9 +457,9 @@ def count_msgs(self) -> int: with self.env.begin(write=False) as txn: meta_db = self.env.open_db( key=persipubsub.META_DB, txn=txn, create=False) - meta_stat = txn.stat(db=meta_db) + meta_stat = txn.stat(db=meta_db) # type: Dict[str, int] - return meta_stat['entries'] + return meta_stat['entries'] def vacuum(self) -> None: """ diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 02896ef..55d3d57 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -5,7 +5,7 @@ import datetime import pathlib import time -from typing import Generator, Optional, Union +from typing import Any, Iterator, Optional, Union import icontract import lmdb # pylint: disable=unused-import @@ -47,13 +47,14 @@ def __enter__(self) -> 'Subscriber': """Enter the context and give the sub prepared in the constructor.""" return self - def __exit__(self, exc_type, exc_val, exc_tb) -> None: + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: """Leave the context.""" @icontract.require(lambda timeout: timeout > 0) @icontract.require(lambda retries: retries > 0) @contextlib.contextmanager - def receive(self, timeout: int = 60, retries: int = 10) -> Generator: + def receive(self, timeout: int = 60, + retries: int = 10) -> Iterator[Optional[bytes]]: """ Receive messages from the publisher. @@ -87,7 +88,8 @@ def _pop(self) -> None: @icontract.require(lambda timeout: timeout > 0) @icontract.require(lambda retries: retries > 0) @contextlib.contextmanager - def receive_to_top(self, timeout: int = 60, retries: int = 10) -> Generator: + def receive_to_top(self, timeout: int = 60, + retries: int = 10) -> Iterator[Optional[bytes]]: """ Pops all messages until the most recent one and receive the latest. diff --git a/precommit.py b/precommit.py index 28f6e16..6bbd70f 100644 --- a/precommit.py +++ b/precommit.py @@ -41,7 +41,7 @@ def main() -> int: cwd=repo_root.as_posix()) print("Mypy'ing...") - subprocess.check_call(["mypy", "persipubsub", "tests"], + subprocess.check_call(["mypy", "--strict", "persipubsub", "tests"], cwd=repo_root.as_posix()) print("Isort'ing...") diff --git a/tests/__init__.py b/tests/__init__.py index 8dcb92a..153e250 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -36,7 +36,7 @@ class TestPersiPubSub(unittest.TestCase): - def test_get_data(self): + def test_get_data(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: env = lmdb.open(path=tmp_dir.path.as_posix(), max_dbs=2) @@ -48,7 +48,7 @@ def test_get_data(self): db=queue_db) data = persipubsub.get_queue_data( - path=tmp_dir.path, key="key".encode(ENCODING)) + key="key".encode(ENCODING), env=env) self.assertEqual(b'value', data) diff --git a/tests/test_control.py b/tests/test_control.py index 0feaadd..2cdf780 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -6,6 +6,7 @@ import unittest from typing import List +import lmdb import temppathlib import persipubsub.control @@ -35,7 +36,7 @@ def setup(path: pathlib.Path, class TestControl(unittest.TestCase): - def test_initialize_all(self): + def test_initialize_all(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=['sub']) @@ -44,6 +45,8 @@ def test_initialize_all(self): ] db_keys = [] # type: List[bytes] + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) with control.queue.env.begin() as txn: cursor = txn.cursor() cursor.first() @@ -52,10 +55,13 @@ def test_initialize_all(self): self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) - def test_del_sub(self): + def test_del_sub(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub1", "sub2"]) + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + control._remove_sub(sub_id="sub2", env=control.queue.env) expected_db_keys = [ @@ -71,7 +77,7 @@ def test_del_sub(self): self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) - def test_clear_all_subs(self): + def test_clear_all_subs(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub1", "sub2"]) @@ -82,6 +88,8 @@ def test_clear_all_subs(self): ] db_keys = [] # type: List[bytes] + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) with control.queue.env.begin() as txn: cursor = txn.cursor() cursor.first() @@ -90,11 +98,15 @@ def test_clear_all_subs(self): self.assertListEqual(sorted(expected_db_keys), sorted(db_keys)) - def test_prune_dangling_messages(self): + def test_prune_dangling_messages(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub"]) + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) + assert isinstance(control.queue.hwm, + persipubsub.queue.HighWaterMark) control.queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT with control.queue.env.begin(write=True) as txn: @@ -179,12 +191,13 @@ def test_prune_dangling_messages(self): self.assertEqual(expected_remaining_entries, remaining_entries) - def test_prune_all_messages_for_subscriber(self): + def test_prune_all_messages_for_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub"]) msg = persipubsub.encoding("hello world!") - + assert isinstance(control.queue, persipubsub.queue._Queue) + assert isinstance(control.queue.env, lmdb.Environment) control.queue.put(msg=msg) control.queue.put(msg=msg) @@ -202,13 +215,13 @@ def test_prune_all_messages_for_subscriber(self): sub_stat = txn.stat(db=sub_db) self.assertEqual(0, sub_stat['entries']) - def test_is_initialized(self): + def test_is_initialized(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub"]) self.assertTrue(control.check_queue_is_initialized()) - def test_is_not_initialized(self): + def test_is_not_initialized(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = persipubsub.control.Control(path=tmp_dir.path) self.assertFalse(control.check_queue_is_initialized()) diff --git a/tests/test_environment.py b/tests/test_environment.py index e1a6285..aa487d3 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -16,18 +16,18 @@ class TestEnvironment(unittest.TestCase): - def test_new_environment(self): + def test_new_environment(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.environment.new_environment(path=tmp_dir.path) self.assertIsInstance(env, persipubsub.environment.Environment) - def test_new_control(self): + def test_new_control(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.environment.new_environment(path=tmp_dir.path) ctl = env.new_control() self.assertIsInstance(ctl, persipubsub.control.Control) - def test_new_publisher(self): + def test_new_publisher(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.environment.new_environment(path=tmp_dir.path) ctl = env.new_control() @@ -35,7 +35,7 @@ def test_new_publisher(self): pub = env.new_publisher() self.assertIsInstance(pub, persipubsub.publisher.Publisher) - def test_new_subscriber(self): + def test_new_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.environment.new_environment(path=tmp_dir.path) ctl = env.new_control() diff --git a/tests/test_publisher.py b/tests/test_publisher.py index a32ab7a..534e3f4 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -5,6 +5,7 @@ import unittest from typing import List +import lmdb import temppathlib import persipubsub.control @@ -35,7 +36,7 @@ def setup(path: pathlib.Path, class TestPublisher(unittest.TestCase): - def test_send(self): + def test_send(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -50,6 +51,8 @@ def test_send(self): pub.send(msg=msg) subscriber = "sub".encode(tests.ENCODING) + + assert isinstance(queue.env, lmdb.Environment) with queue.env.begin(write=False) as txn: self.assertIsNotNone(txn.get(key=subscriber)) sub_db = queue.env.open_db( @@ -65,7 +68,7 @@ def test_send(self): self.assertIsNotNone(item) self.assertEqual(msg, item) - def test_send_many(self): + def test_send_many(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: subscriber = "sub" @@ -82,6 +85,8 @@ def test_send_many(self): pub.send_many(msgs=msgs) + assert isinstance(pub.queue, persipubsub.queue._Queue) + assert isinstance(pub.queue.env, lmdb.Environment) with pub.queue.env.begin(write=False) as txn: self.assertIsNotNone( txn.get(key=subscriber.encode(tests.ENCODING))) diff --git a/tests/test_queue.py b/tests/test_queue.py index 1342bee..b234528 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -6,6 +6,7 @@ import unittest from typing import List +import lmdb import temppathlib import persipubsub.control @@ -38,7 +39,7 @@ def setup(path: pathlib.Path, class TestQueue(unittest.TestCase): - def test_initialize_environment(self): + def test_initialize_environment(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: env = persipubsub.queue._initialize_environment( queue_dir=tmp_dir.path) @@ -60,7 +61,7 @@ def test_initialize_environment(self): 'num_readers': 1 }, env.info()) - def test_put_to_single_subscriber(self): + def test_put_to_single_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) @@ -72,6 +73,7 @@ def test_put_to_single_subscriber(self): queue.init(path=tmp_dir.path) queue.put(msg=msg) + assert isinstance(queue.env, lmdb.Environment) with queue.env.begin() as txn: self.assertIsNotNone( txn.get(key=subscriber.encode(tests.ENCODING))) @@ -91,7 +93,7 @@ def test_put_to_single_subscriber(self): self.assertIsNotNone(value) self.assertEqual(msg, value) - def test_put_multiple_subscriber(self): + def test_put_multiple_subscriber(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) @@ -105,6 +107,7 @@ def test_put_multiple_subscriber(self): queue.put(msg=msg) + assert isinstance(queue.env, lmdb.Environment) with queue.env.begin() as txn: self.assertIsNotNone( txn.get(key=sub_list[0].encode(tests.ENCODING))) @@ -137,7 +140,7 @@ def test_put_multiple_subscriber(self): self.assertIsNotNone(value) self.assertEqual(msg, value) - def test_put_many(self): + def test_put_many(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: subscriber = "sub" @@ -154,6 +157,7 @@ def test_put_many(self): queue.put_many_flush_once(msgs=msgs) + assert isinstance(queue.env, lmdb.Environment) with queue.env.begin(write=False) as txn: self.assertIsNotNone( txn.get(key=subscriber.encode(tests.ENCODING))) @@ -172,7 +176,7 @@ def test_put_many(self): data_stat = txn.stat(db=data_db) self.assertEqual(msg_num, data_stat['entries']) - def test_front(self): + def test_front(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) @@ -189,7 +193,7 @@ def test_front(self): self.assertIsNotNone(received_msg) self.assertEqual(msg, received_msg) - def test_pop(self): + def test_pop(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) @@ -205,6 +209,7 @@ def test_pop(self): received_msg = queue.front(identifier=subscriber) self.assertIsNotNone(received_msg) + assert isinstance(queue.env, lmdb.Environment) with queue.env.begin() as txn: pending_db = queue.env.open_db( key=tests.PENDING_DB, txn=txn, create=False) @@ -229,7 +234,7 @@ def test_pop(self): int.from_bytes(pending_before_pop, tests.BYTES_ORDER) - 1, int.from_bytes(pending_after_pop, tests.BYTES_ORDER)) - def test_pop_queue_empty(self): + def test_pop_queue_empty(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: subscriber = "sub" @@ -240,7 +245,7 @@ def test_pop_queue_empty(self): self.assertRaises(RuntimeError, queue.pop, identifier=subscriber) - def test_queue_initialisation(self): + def test_queue_initialisation(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -248,22 +253,26 @@ def test_queue_initialisation(self): queue.init(path=tmp_dir.path) self.assertIsNotNone(queue.env) + assert isinstance(queue.env, lmdb.Environment) self.assertEqual(tmp_dir.path.as_posix(), queue.env.path()) + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) self.assertEqual(tests.TEST_HWM_LMDB_SIZE, queue.hwm.hwm_lmdb_size_bytes) self.assertEqual(tests.TEST_HWM_MSG_NUM, queue.hwm.max_msgs_num) self.assertEqual(tests.TEST_MSG_TIMEOUT, queue.hwm.msg_timeout_secs) + assert isinstance(queue.strategy, persipubsub.queue.Strategy) self.assertEqual(persipubsub.queue.Strategy.prune_first.name, queue.strategy.name) self.assertEqual(['sub'], queue.subscriber_ids) - def test_overflow_msgs_limit(self): + def test_overflow_msgs_limit(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() queue.init(path=tmp_dir.path) + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM msg = "hello world".encode(tests.ENCODING) @@ -280,13 +289,14 @@ def test_overflow_msgs_limit(self): int(tests.TEST_HWM_MSG_NUM - int(tests.TEST_HWM_MSG_NUM / 2)), queue.count_msgs()) - def test_overflow_limit_size(self): + def test_overflow_limit_size(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() queue.init(path=tmp_dir.path) + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE msg = ("a" * (int(tests.LMDB_PAGE_SIZE / 4))).encode(tests.ENCODING) @@ -302,12 +312,14 @@ def test_overflow_limit_size(self): self.assertTrue( queue.check_current_lmdb_size() <= tests.TEST_HWM_LMDB_SIZE) - def test_timeout(self): + def test_timeout(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) queue = persipubsub.queue._Queue() queue.init(path=tmp_dir.path) + + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT msg = "hello world".encode(tests.ENCODING) @@ -320,7 +332,7 @@ def test_timeout(self): queue.put(msg=msg) self.assertEqual(1, queue.count_msgs()) - def test_strategy_prune_first(self): + def test_strategy_prune_first(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -328,6 +340,7 @@ def test_strategy_prune_first(self): queue.init(path=tmp_dir.path) queue.strategy = persipubsub.queue.Strategy.prune_first + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM for index in range(tests.TEST_HWM_MSG_NUM): @@ -346,7 +359,7 @@ def test_strategy_prune_first(self): int((tests.TEST_HWM_MSG_NUM / 2) + 1)).encode( tests.ENCODING), queue.front(identifier='sub')) - def test_strategy_prune_last(self): + def test_strategy_prune_last(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -354,6 +367,7 @@ def test_strategy_prune_last(self): queue.init(path=tmp_dir.path) queue.strategy = persipubsub.queue.Strategy.prune_last + assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM for index in range(tests.TEST_HWM_MSG_NUM): diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index 03c2ff0..786e577 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -35,7 +35,7 @@ def setup(path: pathlib.Path, class TestSubscriber(unittest.TestCase): - def test_receive_message(self): + def test_receive_message(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -52,7 +52,7 @@ def test_receive_message(self): self.assertIsNotNone(received_msg) self.assertEqual(msg, received_msg) - def test_timeout_subscriber(self): + def test_timeout_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -81,7 +81,7 @@ def test_timeout_subscriber(self): self.assertIsNone(queue.front(identifier='sub')) - def test_pop(self): + def test_pop(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -103,7 +103,7 @@ def test_pop(self): self.assertIsNotNone(msg) self.assertEqual(msg2, msg) - def test_pop_when_empty(self): + def test_pop_when_empty(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) @@ -115,7 +115,7 @@ def test_pop_when_empty(self): self.assertRaises(RuntimeError, sub._pop) - def test_receive_to_top(self): + def test_receive_to_top(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: _ = setup(path=tmp_dir.path, sub_list=['sub']) From 911928841bbc4d39a50bb513c46a681f368c9224 Mon Sep 17 00:00:00 2001 From: slimeth Date: Mon, 11 Feb 2019 14:18:55 +0100 Subject: [PATCH 26/35] modified setup.py --- setup.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index a03922d..76b0e24 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires=[ # yapf: disable 'typing-extensions>=3.7.2', - 'icontract>=2.0.1,<3', + 'icontract>=2.0.2,<3', 'lmdb>=0.94,<1' # yapf: enable ], @@ -62,6 +62,7 @@ ] }, py_modules=['persipubsub', 'persipubsub_meta'], - include_package_data=True, - package_data={"persipubsub": ["py.typed"]}, - data_files=[('.', ['LICENSE', 'README.rst'])]) + package_data={ + "persipubsub": ["py.typed"], + '.': ['LICENSE', 'README.rst'] + }) From ee7d2b99400b57858edc2abb20aad93dafeeb59d Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 12 Feb 2019 12:21:10 +0100 Subject: [PATCH 27/35] first thread-safe and process-safe version --- persipubsub/__init__.py | 6 +- persipubsub/control.py | 100 +++----- persipubsub/environment.py | 15 +- persipubsub/publisher.py | 8 +- persipubsub/queue.py | 86 +++---- persipubsub/subscriber.py | 7 +- requirements-doc.txt | 2 +- setup.py | 11 +- tests/component_publisher.py | 30 +++ tests/component_subscriber.py | 73 ++++++ tests/test_control.py | 7 +- tests/test_live.py | 423 ++++++++++++++++++++++++++++++++++ tests/test_publisher.py | 37 ++- tests/test_queue.py | 138 ++++++----- tests/test_subscriber.py | 71 +++--- 15 files changed, 750 insertions(+), 264 deletions(-) create mode 100644 tests/component_publisher.py create mode 100644 tests/component_subscriber.py create mode 100644 tests/test_live.py diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py index f27e54b..2582202 100644 --- a/persipubsub/__init__.py +++ b/persipubsub/__init__.py @@ -1,6 +1,7 @@ #!/usr/bin/env python """Distribute messages persistent from publisher to subscriber.""" import pathlib +from typing import Optional import lmdb @@ -71,7 +72,7 @@ def bytes_to_int(array_of_bytes: bytes) -> int: return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) -def get_queue_data(key: bytes, env: lmdb.Environment) -> bytes: +def get_queue_data(key: bytes, env: lmdb.Environment) -> Optional[bytes]: """ Get queue data. @@ -83,5 +84,4 @@ def get_queue_data(key: bytes, env: lmdb.Environment) -> bytes: queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) data = txn.get(key=key, db=queue_db) - assert isinstance(data, bytes) - return data + return data # type: ignore diff --git a/persipubsub/control.py b/persipubsub/control.py index 6dbe8ce..6b2c067 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -12,38 +12,13 @@ # pylint: disable=protected-access -def set_queue_parameters(max_reader_num: int, max_db_num: int, - max_db_size_bytes: int, env: lmdb.Environment) -> None: - """ - Store queue parameter in lmdb. - - :param max_reader_num: max number of reader of the lmdb - :param max_db_num: max number of named databases - :param max_db_size_bytes: max size of the lmdb in bytes - :return: - """ - with env.begin(write=True) as txn: - queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) - txn.put( - key=persipubsub.MAX_READER_NUM_KEY, - value=persipubsub.int_to_bytes(max_reader_num), - db=queue_db) - txn.put( - key=persipubsub.MAX_DB_NUM_KEY, - value=persipubsub.int_to_bytes(max_db_num), - db=queue_db) - txn.put( - key=persipubsub.MAX_DB_SIZE_BYTES_KEY, - value=persipubsub.int_to_bytes(max_db_size_bytes), - db=queue_db) - - def set_hwm(hwm: persipubsub.queue.HighWaterMark, env: lmdb.Environment) -> None: """ Set high water mark values for queue. :param hwm: high water mark values + :param env: queue environment :return: """ with env.begin(write=True) as txn: @@ -68,6 +43,7 @@ def set_strategy(strategy: persipubsub.queue.Strategy, Set pruning strategy for queue. :param strategy: pruning strategy + :param env: queue environment :return: """ with env.begin(write=True) as txn: @@ -84,6 +60,7 @@ def _add_sub(sub_id: str, env: lmdb.Environment) -> None: Add a subscriber and create its lmdb. :param sub_id: ID of the subscriber which should be added + :param env: queue environment """ with env.begin(write=True) as txn: _ = env.open_db(key=persipubsub.encoding(sub_id), txn=txn, create=True) @@ -109,22 +86,32 @@ def _add_sub(sub_id: str, env: lmdb.Environment) -> None: class Control: """Control and maintain one queue.""" - def __init__(self, path: pathlib.Path): + def __init__(self, + path: pathlib.Path, + env: Optional[lmdb.Environment] = None): """ Initialize control class. :param path: to the queue. """ self.path = path + if not self.path.exists(): + self.path.mkdir(parents=True, exist_ok=True) self.queue = None # type: Optional[persipubsub.queue._Queue] self.subscriber_ids = set() # type: Set[str] - self.env = None # type: Optional[lmdb.Environment] + + if isinstance(env, lmdb.Environment): + self.env = env + else: + self.env = persipubsub.queue._initialize_environment( + queue_dir=self.path, + max_reader_num=persipubsub.MAX_READER_NUM, + max_db_num=persipubsub.MAX_DB_NUM, + max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) # pylint: disable=too-many-arguments def init(self, subscriber_ids: Optional[Sequence[str]] = None, - max_readers: int = 1024, - max_size: int = 32 * 1024**3, high_watermark: persipubsub.queue.HighWaterMark = persipubsub. queue.HighWaterMark(), strategy: persipubsub.queue.Strategy = persipubsub.queue.Strategy. @@ -133,8 +120,6 @@ def init(self, Initialize control with a (re)initialized queue. :param subscriber_ids: subscribers of the queue - :param max_readers: max number of reader of the lmdb - :param max_size: max size of the lmdb in bytes :param high_watermark: high water mark limit of the queue :param strategy: used to prune queue :return: @@ -147,29 +132,19 @@ def init(self, assert isinstance(subscriber_ids, Sequence) self._initialize_queue( subscriber_ids=subscriber_ids, - max_readers=max_readers, - max_size=max_size, high_watermark=high_watermark, strategy=strategy) def _reinitialize_queue(self) -> None: """Reinitialize the queue which is maintained by the control.""" - self.queue = persipubsub.queue._Queue() - self.queue.init(path=self.path) + self.queue = persipubsub.queue._Queue( + ) # type: persipubsub.queue._Queue + self.queue.init(path=self.path, env=self.env) assert isinstance(self.queue.subscriber_ids, List) self.subscriber_ids = set(self.queue.subscriber_ids) - # pylint: disable=too-many-arguments - @icontract.require(lambda max_readers: max_readers >= 0) - # Each named database needs at least one page of 4096 bytes. - # yapf: disable - @icontract.require(lambda max_size, subscriber_ids: max_size >= ( - 5 + len(subscriber_ids) * 4096)) - # yapf: enable def _initialize_queue(self, subscriber_ids: Sequence[str], - max_readers: int = 1024, - max_size: int = 32 * 1024**3, high_watermark: persipubsub.queue. HighWaterMark = persipubsub.queue.HighWaterMark(), strategy: persipubsub.queue.Strategy = persipubsub. @@ -178,43 +153,25 @@ def _initialize_queue(self, Initialize queue. :param subscriber_ids: subscribers of the queue - :param max_readers: max number of reader of the lmdb - :param max_size: max size of the lmdb in bytes :param high_watermark: high water mark limit of the queue :param strategy: used to prune queue :return: """ - if not self.path.exists(): - self.path.mkdir(parents=True, exist_ok=True) - self.subscriber_ids = set(subscriber_ids) # Databases needed for queue: - # 5 queues (main db, data db, meta db, pending db, queue db) + # 4 queues (data db, meta db, pending db, queue db) # + each subscriber has its own db - max_db_num = 5 + len(self.subscriber_ids) - - env = persipubsub.queue._initialize_environment( - queue_dir=self.path, - max_db_num=max_db_num, - max_reader_num=max_readers, - max_db_size_bytes=max_size) - - set_queue_parameters( - max_reader_num=max_readers, - max_db_num=max_db_num, - max_db_size_bytes=max_size, - env=env) - self.env = env - set_hwm(hwm=high_watermark, env=env) - set_strategy(strategy=strategy, env=env) + set_hwm(hwm=high_watermark, env=self.env) + set_strategy(strategy=strategy, env=self.env) for sub in self.subscriber_ids: - _add_sub(sub_id=sub, env=env) + _add_sub(sub_id=sub, env=self.env) # load initialized queue - self.queue = persipubsub.queue._Queue() - self.queue.init(path=self.path) + self.queue = persipubsub.queue._Queue( + ) # type: persipubsub.queue._Queue + self.queue.init(path=self.path, env=self.env) def check_queue_is_initialized(self) -> bool: """ @@ -223,10 +180,9 @@ def check_queue_is_initialized(self) -> bool: :return: is initialized when all values for the given keys are set """ keys = [ - persipubsub.MAX_DB_SIZE_BYTES_KEY, persipubsub.MAX_DB_NUM_KEY, persipubsub.MSG_TIMEOUT_SECS_KEY, persipubsub.MAX_MSGS_NUM_KEY, persipubsub.HWM_DB_SIZE_BYTES_KEY, persipubsub.STRATEGY_KEY, - persipubsub.SUBSCRIBER_IDS_KEY, persipubsub.MAX_READER_NUM_KEY + persipubsub.SUBSCRIBER_IDS_KEY ] # type: List[bytes] for key in keys: diff --git a/persipubsub/environment.py b/persipubsub/environment.py index 8fe0293..3fd31c6 100644 --- a/persipubsub/environment.py +++ b/persipubsub/environment.py @@ -22,6 +22,11 @@ def __init__(self, path: pathlib.Path) -> None: :param path: to the queue """ self.path = path + self.env = persipubsub.queue._initialize_environment( + queue_dir=self.path, + max_reader_num=persipubsub.MAX_READER_NUM, + max_db_num=persipubsub.MAX_DB_NUM, + max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) def __enter__(self) -> 'Environment': """Enter the context and give environment prepared to constructor.""" @@ -33,8 +38,6 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: # pylint: disable=too-many-arguments def new_control(self, subscriber_ids: Optional[Sequence[str]] = None, - max_readers: int = 1024, - max_size: int = 32 * 1024**3, high_watermark: persipubsub.queue. HighWaterMark = persipubsub.queue.HighWaterMark(), strategy: persipubsub.queue.Strategy = persipubsub.queue. @@ -49,11 +52,9 @@ def new_control(self, :param strategy: used to prune queue :return: Control to create and maintain queue """ - control = persipubsub.control.Control(path=self.path) + control = persipubsub.control.Control(path=self.path, env=self.env) control.init( subscriber_ids=subscriber_ids, - max_readers=max_readers, - max_size=max_size, high_watermark=high_watermark, strategy=strategy) return control @@ -67,7 +68,7 @@ def new_publisher( :return: Publisher to send messages """ publisher = persipubsub.publisher.Publisher() - publisher.init(path=self.path, autosync=autosync) + publisher.init(path=self.path, autosync=autosync, env=self.env) return publisher def new_subscriber(self, @@ -79,7 +80,7 @@ def new_subscriber(self, :return: Subscriber to receive messages """ subscriber = persipubsub.subscriber.Subscriber() - subscriber.init(identifier=identifier, path=self.path) + subscriber.init(identifier=identifier, path=self.path, env=self.env) return subscriber diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index 38ebe63..d5231b5 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -4,6 +4,8 @@ import pathlib from typing import Any, List, Optional, Union +import lmdb + import persipubsub.queue # pylint: disable=protected-access @@ -24,7 +26,9 @@ def __init__(self) -> None: self.queue = None # type: Optional[persipubsub.queue._Queue] self.autosync = None # type: Optional[bool] - def init(self, path: Union[pathlib.Path, str], + def init(self, + path: Union[pathlib.Path, str], + env: Optional[lmdb.Environment] = None, autosync: bool = False) -> None: """ Initialize. @@ -33,7 +37,7 @@ def init(self, path: Union[pathlib.Path, str], :param autosync: if True, store data automatically in lmdb """ self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access - self.queue.init(path=path) + self.queue.init(path=path, env=env) self.autosync = autosync def __enter__(self) -> 'Publisher': diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 950a3ce..2e5999a 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -85,6 +85,9 @@ def _initialize_environment( # Setting max_spare_txn equals 0 doesn't allow any caching for read-only # transactions to avoid the error: # MDB_BAD_RSLOT: Invalid reuse of reader locktable slot. + # Error only occures when one publisher/subscriber is used in multiple + # processes. So max_spare_txn equal 1 is fine as we expect a publisher and + # subscriber running on the same process. env = lmdb.open( path=queue_dir.as_posix(), map_size=max_db_size_bytes, @@ -181,9 +184,7 @@ def __init__(self) -> None: # pylint: disable=too-many-locals def init(self, path: Union[pathlib.Path, str], - max_reader_num: Optional[int] = None, - max_db_num: Optional[int] = None, - max_db_size_bytes: Optional[int] = None) -> None: + env: Optional[lmdb.Environment] = None) -> None: """ Initialize the queue. @@ -195,30 +196,14 @@ def init(self, self.path = path if isinstance(path, pathlib.Path) \ else pathlib.Path(path) - # max_reader_num_queue = max_reader_num if max_reader_num is not None \ - # else persipubsub.bytes_to_int( - # persipubsub.get_queue_data( - # path=self.path, key=persipubsub.MAX_READER_NUM_KEY)) - # max_db_num_queue = max_db_num if max_db_num is not None else \ - # persipubsub.bytes_to_int( - # persipubsub.get_queue_data( - # path=self.path, key=persipubsub.MAX_DB_NUM_KEY)) - # max_db_size_bytes_queue = max_db_size_bytes if max_db_size_bytes \ - # is not None else \ - # persipubsub.bytes_to_int( - # persipubsub.get_queue_data( - # path=self.path, key=persipubsub.MAX_DB_SIZE_BYTES_KEY)) - - # TODO(snaji): remove - max_reader_num_queue = persipubsub.MAX_READER_NUM - max_db_num_queue = persipubsub.MAX_DB_NUM - max_db_size_bytes_queue = persipubsub.MAX_DB_SIZE_BYTES - - self.env = _initialize_environment( - queue_dir=self.path, - max_reader_num=max_reader_num_queue, - max_db_num=max_db_num_queue, - max_db_size_bytes=max_db_size_bytes_queue) + if isinstance(env, lmdb.Environment): + self.env = env + else: + self.env = _initialize_environment( + queue_dir=self.path, + max_reader_num=persipubsub.MAX_READER_NUM, + max_db_num=persipubsub.MAX_DB_NUM, + max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) with self.env.begin(write=True) as txn: _ = self.env.open_db(key=persipubsub.DATA_DB, txn=txn, create=True) @@ -227,37 +212,40 @@ def init(self, _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) _ = self.env.open_db(key=persipubsub.QUEUE_DB, txn=txn, create=True) - msg_timeout_secs = persipubsub.bytes_to_int( - persipubsub.get_queue_data( - key=persipubsub.MSG_TIMEOUT_SECS_KEY, env=self.env)) - max_msgs_num = persipubsub.bytes_to_int( - persipubsub.get_queue_data( - key=persipubsub.MAX_MSGS_NUM_KEY, env=self.env)) - hwm_lmdb_size_bytes = persipubsub.bytes_to_int( - persipubsub.get_queue_data( - key=persipubsub.HWM_DB_SIZE_BYTES_KEY, env=self.env)) - - # TODO(snaji): remove - # msg_timeout_secs = 500 - # max_msgs_num = 10000000000000 - # hwm_lmdb_size_bytes = 30 * 1024**3 + msg_timeout_secs_bytes = persipubsub.get_queue_data( + key=persipubsub.MSG_TIMEOUT_SECS_KEY, env=self.env) + msg_timeout_secs = msg_timeout_secs_bytes \ + if msg_timeout_secs_bytes is None else persipubsub.bytes_to_int( + msg_timeout_secs_bytes) # type: Optional[int] + + max_msgs_num_bytes = persipubsub.get_queue_data( + key=persipubsub.MAX_MSGS_NUM_KEY, env=self.env) + max_msgs_num = max_msgs_num_bytes \ + if max_msgs_num_bytes is None else persipubsub.bytes_to_int( + max_msgs_num_bytes) # type: Optional[int] + + hwm_lmdb_size_bytes = persipubsub.get_queue_data( + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, env=self.env) + hwm_lmdb_size = hwm_lmdb_size_bytes \ + if hwm_lmdb_size_bytes is None else persipubsub.bytes_to_int( + hwm_lmdb_size_bytes) # type: Optional[int] + self.hwm = HighWaterMark( msg_timeout_secs=msg_timeout_secs, max_msgs_num=max_msgs_num, - hwm_lmdb_size_bytes=hwm_lmdb_size_bytes) + hwm_lmdb_size_bytes=hwm_lmdb_size) strategy = persipubsub.get_queue_data( key=persipubsub.STRATEGY_KEY, env=self.env) - # TODO(snaji): remove - # strategy = "prune_first".encode('utf-8') - self.strategy = _parse_strategy( - persipubsub.decoding(encoded_str=strategy)) + strategy_decoded = "" if strategy is None else persipubsub.decoding( + encoded_str=strategy) + + self.strategy = _parse_strategy(strategy=strategy_decoded) subscriber_list = persipubsub.get_queue_data( key=persipubsub.SUBSCRIBER_IDS_KEY, env=self.env) - # TODO(snaji): remove - # self.subscriber_ids = ['sub'] + if subscriber_list is None: self.subscriber_ids = [] else: @@ -280,7 +268,7 @@ def put(self, msg: bytes) -> None: :return: """ # every publisher always prunes queue before sending a message. - # self.vacuum() + self.vacuum() msg_id = str(datetime.datetime.utcnow().timestamp()) + str(uuid.uuid4()) assert isinstance(self.env, lmdb.Environment) assert isinstance(self.subscriber_ids, List) diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 55d3d57..868d520 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -30,7 +30,10 @@ def __init__(self) -> None: self.identifier = None # type: Optional[str] self.queue = None # type: Optional[persipubsub.queue._Queue] - def init(self, identifier: str, path: Union[pathlib.Path, str]) -> None: + def init(self, + identifier: str, + path: Union[pathlib.Path, str], + env: Optional[lmdb.Environment] = None) -> None: """ Initialize. @@ -40,7 +43,7 @@ def init(self, identifier: str, path: Union[pathlib.Path, str]) -> None: self.identifier = identifier assert isinstance(self.identifier, str) self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access - self.queue.init(path=path) + self.queue.init(path=path, env=env) assert isinstance(self.queue, persipubsub.queue._Queue) def __enter__(self) -> 'Subscriber': diff --git a/requirements-doc.txt b/requirements-doc.txt index 50039e6..eccd910 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -1,4 +1,4 @@ -sphinx>=1.8.3,<2 +sphinx>=1.8.4,<2 sphinx-autodoc-typehints>=1.6.0 sphinx-icontract>=2.0.0 sphinx-rtd-theme>=0.4.2 \ No newline at end of file diff --git a/setup.py b/setup.py index 76b0e24..137bc5f 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,8 @@ license='License :: OSI Approved :: MIT License', keywords= 'persistent publisher subscriber lmdb MQ message queue thread-safe process-safe', - packages=find_packages(exclude=['tests']), + packages=find_packages( + exclude=['tests', 'examples', 'messages', 'obsolete']), install_requires=[ # yapf: disable 'typing-extensions>=3.7.2', @@ -44,9 +45,9 @@ extras_require={ 'dev': [ # yapf: disable - 'mypy==0.660', + 'mypy==0.670', 'pylint==2.2.2', - 'yapf==0.25.0', + 'yapf==0.26.0', 'tox>=3.7.0', 'coverage>=4.5.2,<5', 'pydocstyle>=3.0.0,<4', @@ -55,8 +56,8 @@ 'isort>=4.3.4,<5', 'pygments>=2.3.1,<3', 'twine>=1.12.1,<2', - 'setuptools>=40.6.3,<41', - 'wheel>=0.32.3,<1', + 'setuptools>=40.8.0,<41', + 'wheel>=0.33.0,<1', 'temppathlib>=1.0.3,<2' # yapf: enable ] diff --git a/tests/component_publisher.py b/tests/component_publisher.py new file mode 100644 index 0000000..5abef79 --- /dev/null +++ b/tests/component_publisher.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +"""Publisher component for live test.""" + +import pathlib +import time + +import persipubsub.environment +import persipubsub.publisher + +# pylint: disable=missing-docstring + + +def send_thread(env: persipubsub.environment.Environment, + num_msg: int, + sleep_time: float = 0) -> None: + pub = env.new_publisher() + + for _ in range(num_msg): + pub.send(msg="hello subscriber".encode('utf-8')) + time.sleep(sleep_time) + + +def send_process(path: pathlib.Path, num_msg: int, + sleep_time: float = 0) -> None: + env = persipubsub.environment.Environment(path=path) + pub = env.new_publisher() + + for _ in range(num_msg): + pub.send(msg="hello subscriber".encode('utf-8')) + time.sleep(sleep_time) diff --git a/tests/component_subscriber.py b/tests/component_subscriber.py new file mode 100644 index 0000000..c7adfd1 --- /dev/null +++ b/tests/component_subscriber.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python +"""Subscriber component for live test.""" + +import pathlib +import time + +import persipubsub.environment + +# pylint: disable=missing-docstring + + +# pylint: disable=too-many-arguments +def receive_thread(path: pathlib.Path, + env: persipubsub.environment.Environment, + identifier: str, + num_msg: int, + timeout: int = 2, + retries: int = 10, + sleep_time: float = 0, + method_timeout: int = 10) -> None: + sub = env.new_subscriber(identifier=identifier) + + received_msg = 0 + + start = time.time() + while True: + if received_msg == num_msg: + break + elif time.time() - start >= method_timeout: + raise TimeoutError + else: + with sub.receive(timeout=timeout, retries=retries) as msg: + if msg is not None: + received_msg += 1 + time.sleep(sleep_time) + else: + time.sleep(sleep_time) + continue + + result = path / identifier + result.write_text('pass') + + +# pylint: disable=too-many-arguments +def receive_process(path: pathlib.Path, + identifier: str, + num_msg: int, + timeout: int = 2, + retries: int = 10, + sleep_time: float = 0, + method_timeout: int = 10) -> None: + env = persipubsub.environment.Environment(path=path) + sub = env.new_subscriber(identifier=identifier) + + received_msg = 0 + + start = time.time() + while True: + if received_msg == num_msg: + break + elif time.time() - start >= method_timeout: + raise TimeoutError + else: + with sub.receive(timeout=timeout, retries=retries) as msg: + if msg is not None: + received_msg += 1 + time.sleep(sleep_time) + else: + time.sleep(sleep_time) + continue + + result = path / identifier + result.write_text('pass') diff --git a/tests/test_control.py b/tests/test_control.py index 2cdf780..9afdd4a 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -25,12 +25,7 @@ def setup(path: pathlib.Path, hwm = persipubsub.queue.HighWaterMark() strategy = persipubsub.queue.Strategy.prune_first - control.init( - subscriber_ids=sub_list, - max_readers=tests.TEST_MAX_READER_NUM, - max_size=tests.TEST_MAX_DB_SIZE_BYTES, - high_watermark=hwm, - strategy=strategy) + control.init(subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) return control diff --git a/tests/test_live.py b/tests/test_live.py new file mode 100644 index 0000000..bd0a34e --- /dev/null +++ b/tests/test_live.py @@ -0,0 +1,423 @@ +#!/usr/bin/env python +"""Test persipubsub live.""" + +# pylint: disable=unused-import +import multiprocessing +import pathlib +import threading +import time +import unittest + +import lmdb +import temppathlib + +import persipubsub.environment +import persipubsub.publisher +import persipubsub.queue +import persipubsub.subscriber +import tests.component_publisher +import tests.component_subscriber + +# pylint: disable=missing-docstring + + +def receive(sub: persipubsub.subscriber.Subscriber, + num_msg: int, + timeout: int = 2, + sleep_time: float = 0, + method_timeout: int = 10) -> None: + received_msg = 0 + + start = time.time() + while True: + if received_msg == num_msg: + break + elif time.time() - start >= method_timeout: + raise TimeoutError + + with sub.receive(timeout=timeout) as msg: + if msg is not None: + received_msg += 1 + time.sleep(sleep_time) + + assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub.queue.env, lmdb.Environment) + assert isinstance(sub.queue.env.path(), str) + assert isinstance(sub.identifier, str) + result = pathlib.Path(sub.queue.env.path()) / sub.identifier + result.write_text('pass') + + +def send(pub: persipubsub.publisher.Publisher, + num_msg: int, + sleep_time: float = 0) -> None: + for _ in range(num_msg): + pub.send(msg="hello subscriber".encode('utf-8')) + time.sleep(sleep_time) + + +class TestLive(unittest.TestCase): + def test_multithreaded_communication_one_publisher_one_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub']) + + result = tmp_dir.path / "sub" + result.touch() + + pub = env.new_publisher() + sub = env.new_subscriber(identifier='sub') + + num_msg = 1000 + pub_thread = threading.Thread( + target=send, kwargs={ + 'pub': pub, + 'num_msg': num_msg + }) + sub_thread = threading.Thread( + target=receive, kwargs={ + 'sub': sub, + 'num_msg': num_msg + }) + + pub_thread.start() + sub_thread.start() + + for thread in [pub_thread, sub_thread]: + thread.join() + + self.assertEqual('pass', result.read_text()) + + # pylint: disable=too-many-locals + def test_multithreaded_communication_two_publisher_two_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub1', 'sub2']) + + result1 = tmp_dir.path / "sub1" + result1.touch() + result2 = tmp_dir.path / "sub2" + result2.touch() + + pub1 = env.new_publisher() + pub2 = env.new_publisher() + sub1 = env.new_subscriber(identifier='sub1') + sub2 = env.new_subscriber(identifier='sub2') + + num_msg = 1000 + + pub1_thread = threading.Thread( + target=send, kwargs={ + 'pub': pub1, + 'num_msg': num_msg + }) + sub1_thread = threading.Thread( + target=receive, kwargs={ + 'sub': sub1, + 'num_msg': 2 * num_msg + }) + pub2_thread = threading.Thread( + target=send, kwargs={ + 'pub': pub2, + 'num_msg': num_msg + }) + sub2_thread = threading.Thread( + target=receive, kwargs={ + 'sub': sub2, + 'num_msg': 2 * num_msg + }) + + pub1_thread.start() + sub1_thread.start() + pub2_thread.start() + sub2_thread.start() + + for thread in [pub1_thread, sub1_thread, pub2_thread, sub2_thread]: + thread.join() + + self.assertEqual('pass', result1.read_text()) + self.assertEqual('pass', result2.read_text()) + + def test_multithreaded_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub']) + + result = tmp_dir.path / "sub" + result.touch() + + num_msg = 1000 + pub_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + sub_thread = threading.Thread( + target=tests.component_subscriber.receive_thread, + kwargs={ + 'path': tmp_dir.path, + 'env': env, + 'identifier': 'sub', + 'num_msg': num_msg + }) + pub_thread.start() + sub_thread.start() + + for thread in [pub_thread, sub_thread]: + thread.join() + + self.assertEqual('pass', result.read_text()) + + def test_multithreaded_two_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub1', 'sub2']) + + result1 = tmp_dir.path / "sub1" + result1.touch() + result2 = tmp_dir.path / "sub2" + result2.touch() + + num_msg = 300 + pub1_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + sub1_thread = threading.Thread( + target=tests.component_subscriber.receive_thread, + kwargs={ + 'path': tmp_dir.path, + 'env': env, + 'identifier': 'sub1', + 'num_msg': 2 * num_msg, + 'method_timeout': 20 + }) + pub2_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + sub2_thread = threading.Thread( + target=tests.component_subscriber.receive_thread, + kwargs={ + 'path': tmp_dir.path, + 'env': env, + 'identifier': 'sub2', + 'num_msg': 2 * num_msg, + 'method_timeout': 20 + }) + pub1_thread.start() + sub1_thread.start() + pub2_thread.start() + sub2_thread.start() + + for thread in [pub1_thread, sub1_thread, pub2_thread, sub2_thread]: + thread.join() + + self.assertEqual('pass', result1.read_text()) + self.assertEqual('pass', result2.read_text()) + + def test_multiprocess_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub']) + env.env.close() + + result = tmp_dir.path / "sub" + result.touch() + + num_msg = 1000 + pub_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + sub_process = multiprocessing.Process( + target=tests.component_subscriber.receive_process, + kwargs={ + 'path': tmp_dir.path, + 'identifier': 'sub', + 'num_msg': num_msg + }) + pub_process.start() + sub_process.start() + + for process in [pub_process, sub_process]: + process.join() + + self.assertEqual('pass', result.read_text()) + + def test_multiprocess_two_component_publisher_component_subscriber( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(['sub1', 'sub2']) + + result1 = tmp_dir.path / "sub1" + result1.touch() + result2 = tmp_dir.path / "sub2" + result2.touch() + + num_msg = 300 + pub1_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + sub1_process = multiprocessing.Process( + target=tests.component_subscriber.receive_process, + kwargs={ + 'path': tmp_dir.path, + 'identifier': 'sub1', + 'num_msg': 2 * num_msg, + 'method_timeout': 20 + }) + pub2_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + sub2_process = multiprocessing.Process( + target=tests.component_subscriber.receive_process, + kwargs={ + 'path': tmp_dir.path, + 'identifier': 'sub2', + 'num_msg': 2 * num_msg, + 'method_timeout': 20 + }) + pub1_process.start() + sub1_process.start() + pub2_process.start() + sub2_process.start() + + for process in [ + pub1_process, sub1_process, pub2_process, sub2_process + ]: + process.join() + + self.assertEqual('pass', result1.read_text()) + self.assertEqual('pass', result2.read_text()) + + def test_multithreaded_race_condition_of_the_component_publisher( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + control = env.new_control(['sub']) + + num_msg = 50 + num_threads = 50 + + threads = [] + for _ in range(num_threads): + pub_thread = threading.Thread( + target=tests.component_publisher.send_thread, + kwargs={ + 'env': env, + 'num_msg': num_msg + }) + + pub_thread.start() + threads.append(pub_thread) + + for thread in threads: + thread.join() + + assert isinstance(control.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode('utf-8'), txn=txn) + self.assertEqual(num_msg * num_threads, + txn.stat(db=sub_db)['entries']) + + def test_multiprocess_race_condition_of_the_component_publisher( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + control = env.new_control(['sub']) + + num_msg = 50 + num_processes = 50 + + processes = [] + for _ in range(num_processes): + pub_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'path': tmp_dir.path, + 'num_msg': num_msg + }) + + pub_process.start() + processes.append(pub_process) + + for process in processes: + process.join() + + assert isinstance(control.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode('utf-8'), txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=sub_db)['entries']) + + def test_multiprocess_race_condition_of_the_component_publisher_one_env( + self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + control = env.new_control(subscriber_ids=['sub']) + + num_msg = 50 + num_processes = 50 + + processes = [] + for _ in range(num_processes): + pub_process = multiprocessing.Process( + target=tests.component_publisher.send_process, + kwargs={ + 'num_msg': num_msg, + 'path': tmp_dir.path, + }) + + pub_process.start() + processes.append(pub_process) + + for process in processes: + process.join() + + assert isinstance(control.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(control.queue.env, lmdb.Environment) + with control.queue.env.begin(write=False) as txn: + sub_db = control.queue.env.open_db( + key='sub'.encode('utf-8'), txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=sub_db)['entries']) + data_db = control.queue.env.open_db( + key=persipubsub.DATA_DB, txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=data_db)['entries']) + meta_db = control.queue.env.open_db( + key=persipubsub.META_DB, txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=meta_db)['entries']) + pending_db = control.queue.env.open_db( + key=persipubsub.PENDING_DB, txn=txn) + self.assertEqual(num_processes * num_msg, + txn.stat(db=pending_db)['entries']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 534e3f4..9226ca9 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -1,7 +1,6 @@ #!/usr/bin/env python """Test publisher.""" -import pathlib import unittest from typing import List @@ -9,6 +8,7 @@ import temppathlib import persipubsub.control +import persipubsub.environment import persipubsub.publisher import persipubsub.queue import tests @@ -17,20 +17,14 @@ # pylint: disable=protected-access -def setup(path: pathlib.Path, +def setup(env: persipubsub.environment.Environment, sub_list: List[str]) -> persipubsub.control.Control: """Create an initialized control""" - control = persipubsub.control.Control(path=path) - hwm = persipubsub.queue.HighWaterMark() strategy = persipubsub.queue.Strategy.prune_first - control.init( - subscriber_ids=sub_list, - max_readers=tests.TEST_MAX_READER_NUM, - max_size=tests.TEST_MAX_DB_SIZE_BYTES, - high_watermark=hwm, - strategy=strategy) + control = env.new_control( + subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) return control @@ -39,30 +33,25 @@ class TestPublisher(unittest.TestCase): def test_send(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) - - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + env = persipubsub.environment.Environment(path=tmp_dir.path) + _ = setup(env=env, sub_list=['sub']) - pub = persipubsub.publisher.Publisher() - pub.init(path=tmp_dir.path) + pub = env.new_publisher() msg = "Hello world!".encode(tests.ENCODING) pub.send(msg=msg) subscriber = "sub".encode(tests.ENCODING) - assert isinstance(queue.env, lmdb.Environment) - with queue.env.begin(write=False) as txn: + with env.env.begin(write=False) as txn: self.assertIsNotNone(txn.get(key=subscriber)) - sub_db = queue.env.open_db( - key=subscriber, txn=txn, create=False) + sub_db = env.env.open_db(key=subscriber, txn=txn, create=False) cursor = txn.cursor(db=sub_db) self.assertTrue(cursor.first()) key = cursor.key() - data_db = queue.env.open_db( + data_db = env.env.open_db( key=tests.DATA_DB, txn=txn, create=False) item = txn.get(key=key, db=data_db) self.assertIsNotNone(item) @@ -71,11 +60,11 @@ def test_send(self) -> None: def test_send_many(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.Environment(path=tmp_dir.path) subscriber = "sub" - _ = setup(path=tmp_dir.path, sub_list=[subscriber]) + _ = setup(env=env, sub_list=[subscriber]) - pub = persipubsub.publisher.Publisher() - pub.init(path=tmp_dir.path) + pub = env.new_publisher() msg = "I'm a message".encode(tests.ENCODING) msgs = [] diff --git a/tests/test_queue.py b/tests/test_queue.py index b234528..b4c03c5 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """Test database.""" -import pathlib import time import unittest from typing import List @@ -10,6 +9,7 @@ import temppathlib import persipubsub.control +import persipubsub.environment import persipubsub.queue import tests @@ -17,23 +17,14 @@ # pylint: disable=protected-access -def setup(path: pathlib.Path, +def setup(env: persipubsub.environment.Environment, sub_list: List[str]) -> persipubsub.control.Control: """Create an initialized control""" - control = persipubsub.control.Control(path=path) - - hwm = persipubsub.queue.HighWaterMark( - msg_timeout_secs=tests.TEST_MSG_TIMEOUT, - max_msgs_num=tests.TEST_HWM_MSG_NUM, - hwm_lmdb_size_bytes=tests.TEST_HWM_LMDB_SIZE) + hwm = persipubsub.queue.HighWaterMark() strategy = persipubsub.queue.Strategy.prune_first - control.init( - subscriber_ids=sub_list, - max_readers=tests.TEST_MAX_READER_NUM, - max_size=tests.TEST_MAX_DB_SIZE_BYTES, - high_watermark=hwm, - strategy=strategy) + control = env.new_control( + subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) return control @@ -65,12 +56,13 @@ def test_put_to_single_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) - subscriber = "sub" + env = persipubsub.environment.Environment(path=tmp_dir.path) - _ = setup(path=tmp_dir.path, sub_list=['sub']) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) queue.put(msg=msg) assert isinstance(queue.env, lmdb.Environment) @@ -100,11 +92,11 @@ def test_put_multiple_subscriber(self) -> None: sub_list = ["sub", "another_sub"] - _ = setup(path=tmp_dir.path, sub_list=sub_list) - - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + env = persipubsub.environment.Environment(path=tmp_dir.path) + _ = setup(env=env, sub_list=sub_list) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) queue.put(msg=msg) assert isinstance(queue.env, lmdb.Environment) @@ -143,11 +135,13 @@ def test_put_multiple_subscriber(self) -> None: def test_put_many(self) -> None: # pylint: disable=too-many-locals with temppathlib.TemporaryDirectory() as tmp_dir: - subscriber = "sub" - _ = setup(path=tmp_dir.path, sub_list=[subscriber]) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) msg = "I'm a message".encode(tests.ENCODING) msgs = [] @@ -180,11 +174,13 @@ def test_front(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) - subscriber = "sub" - _ = setup(path=tmp_dir.path, sub_list=[subscriber]) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) queue.put(msg=msg) # pylint: disable=assignment-from-none @@ -197,11 +193,13 @@ def test_pop(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: msg = "I'm a message.".encode(tests.ENCODING) - subscriber = "sub" - _ = setup(path=tmp_dir.path, sub_list=[subscriber]) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) queue.put(msg=msg) # pylint: disable=assignment-from-none @@ -237,20 +235,33 @@ def test_pop(self) -> None: def test_pop_queue_empty(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - subscriber = "sub" - _ = setup(path=tmp_dir.path, sub_list=[subscriber]) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) self.assertRaises(RuntimeError, queue.pop, identifier=subscriber) def test_queue_initialisation(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + hwm = persipubsub.queue.HighWaterMark( + msg_timeout_secs=tests.TEST_MSG_TIMEOUT, + max_msgs_num=tests.TEST_HWM_MSG_NUM, + hwm_lmdb_size_bytes=tests.TEST_HWM_LMDB_SIZE) + + persipubsub.control.set_hwm(hwm=hwm, env=env.env) + persipubsub.control.set_strategy( + strategy=persipubsub.queue.Strategy.prune_first, env=env.env) + queue = env.new_control().queue + assert isinstance(queue, persipubsub.queue._Queue) self.assertIsNotNone(queue.env) assert isinstance(queue.env, lmdb.Environment) @@ -267,10 +278,13 @@ def test_queue_initialisation(self) -> None: def test_overflow_msgs_limit(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.max_msgs_num = tests.TEST_HWM_MSG_NUM @@ -291,10 +305,13 @@ def test_overflow_msgs_limit(self) -> None: def test_overflow_limit_size(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.hwm_lmdb_size_bytes = tests.TEST_HWM_LMDB_SIZE @@ -314,10 +331,13 @@ def test_overflow_limit_size(self) -> None: def test_timeout(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) + + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) queue.hwm.msg_timeout_secs = tests.TEST_MSG_TIMEOUT @@ -334,10 +354,13 @@ def test_timeout(self) -> None: def test_strategy_prune_first(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) queue.strategy = persipubsub.queue.Strategy.prune_first assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) @@ -361,10 +384,13 @@ def test_strategy_prune_first(self) -> None: def test_strategy_prune_last(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) + + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) queue.strategy = persipubsub.queue.Strategy.prune_last assert isinstance(queue.hwm, persipubsub.queue.HighWaterMark) diff --git a/tests/test_subscriber.py b/tests/test_subscriber.py index 786e577..b0144e4 100644 --- a/tests/test_subscriber.py +++ b/tests/test_subscriber.py @@ -1,13 +1,13 @@ #!/usr/bin/env python """Test subscriber.""" -import pathlib import unittest from typing import List import temppathlib import persipubsub.control +import persipubsub.environment import persipubsub.queue import persipubsub.subscriber import tests @@ -16,20 +16,14 @@ # pylint: disable=protected-access -def setup(path: pathlib.Path, +def setup(env: persipubsub.environment.Environment, sub_list: List[str]) -> persipubsub.control.Control: """Create an initialized control""" - control = persipubsub.control.Control(path=path) - hwm = persipubsub.queue.HighWaterMark() strategy = persipubsub.queue.Strategy.prune_first - control.init( - subscriber_ids=sub_list, - max_readers=tests.TEST_MAX_READER_NUM, - max_size=tests.TEST_MAX_DB_SIZE_BYTES, - high_watermark=hwm, - strategy=strategy) + control = env.new_control( + subscriber_ids=sub_list, high_watermark=hwm, strategy=strategy) return control @@ -37,13 +31,14 @@ def setup(path: pathlib.Path, class TestSubscriber(unittest.TestCase): def test_receive_message(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - sub = persipubsub.subscriber.Subscriber() - sub.init(identifier="sub", path=tmp_dir.path) + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) msg = "Hello World!".encode(tests.ENCODING) queue.put(msg=msg) @@ -54,13 +49,14 @@ def test_receive_message(self) -> None: def test_timeout_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - sub = persipubsub.subscriber.Subscriber() - sub.init(identifier='sub', path=tmp_dir.path) + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) with sub.receive(timeout=1) as received_msg: self.assertIsNone(received_msg) @@ -71,7 +67,7 @@ def test_timeout_subscriber(self) -> None: self.assertEqual( "message send after timeout and will not be popped".encode( - tests.ENCODING), queue.front(identifier='sub')) + tests.ENCODING), queue.front(identifier=subscriber)) with sub.receive(timeout=1) as received_msg: self.assertIsNotNone(received_msg) @@ -79,17 +75,18 @@ def test_timeout_subscriber(self) -> None: "message send after timeout and will not be popped".encode( tests.ENCODING), received_msg) - self.assertIsNone(queue.front(identifier='sub')) + self.assertIsNone(queue.front(identifier=subscriber)) def test_pop(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - sub = persipubsub.subscriber.Subscriber() - sub.init(identifier='sub', path=tmp_dir.path) + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) msg1 = "I'm a message".encode(tests.ENCODING) queue.put(msg=msg1) @@ -105,25 +102,25 @@ def test_pop(self) -> None: def test_pop_when_empty(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - sub = persipubsub.subscriber.Subscriber() - sub.init(identifier='sub', path=tmp_dir.path) + sub = env.new_subscriber(identifier=subscriber) self.assertRaises(RuntimeError, sub._pop) def test_receive_to_top(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: - _ = setup(path=tmp_dir.path, sub_list=['sub']) + env = persipubsub.environment.Environment(path=tmp_dir.path) - queue = persipubsub.queue._Queue() - queue.init(path=tmp_dir.path) + subscriber = 'sub' + _ = setup(env=env, sub_list=[subscriber]) - sub = persipubsub.subscriber.Subscriber() - sub.init(identifier='sub', path=tmp_dir.path) + sub = env.new_subscriber(identifier=subscriber) + queue = env.new_publisher().queue + assert isinstance(queue, persipubsub.queue._Queue) msg1 = "I'm a message".encode(tests.ENCODING) queue.put(msg=msg1) From 92e2b0aadc5fd4bfa4dd3fc1eb26c01853df88c4 Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 12 Feb 2019 12:29:49 +0100 Subject: [PATCH 28/35] adjust timeout time --- tests/component_subscriber.py | 2 +- tests/test_live.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/component_subscriber.py b/tests/component_subscriber.py index c7adfd1..64b989e 100644 --- a/tests/component_subscriber.py +++ b/tests/component_subscriber.py @@ -48,7 +48,7 @@ def receive_process(path: pathlib.Path, timeout: int = 2, retries: int = 10, sleep_time: float = 0, - method_timeout: int = 10) -> None: + method_timeout: int = 60) -> None: env = persipubsub.environment.Environment(path=path) sub = env.new_subscriber(identifier=identifier) diff --git a/tests/test_live.py b/tests/test_live.py index bd0a34e..a692056 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -25,7 +25,7 @@ def receive(sub: persipubsub.subscriber.Subscriber, num_msg: int, timeout: int = 2, sleep_time: float = 0, - method_timeout: int = 10) -> None: + method_timeout: int = 60) -> None: received_msg = 0 start = time.time() @@ -197,7 +197,7 @@ def test_multithreaded_two_component_publisher_component_subscriber( 'env': env, 'identifier': 'sub1', 'num_msg': 2 * num_msg, - 'method_timeout': 20 + 'method_timeout': 60 }) pub2_thread = threading.Thread( target=tests.component_publisher.send_thread, @@ -212,7 +212,7 @@ def test_multithreaded_two_component_publisher_component_subscriber( 'env': env, 'identifier': 'sub2', 'num_msg': 2 * num_msg, - 'method_timeout': 20 + 'method_timeout': 60 }) pub1_thread.start() sub1_thread.start() @@ -281,7 +281,7 @@ def test_multiprocess_two_component_publisher_component_subscriber( 'path': tmp_dir.path, 'identifier': 'sub1', 'num_msg': 2 * num_msg, - 'method_timeout': 20 + 'method_timeout': 60 }) pub2_process = multiprocessing.Process( target=tests.component_publisher.send_process, @@ -295,7 +295,7 @@ def test_multiprocess_two_component_publisher_component_subscriber( 'path': tmp_dir.path, 'identifier': 'sub2', 'num_msg': 2 * num_msg, - 'method_timeout': 20 + 'method_timeout': 60 }) pub1_process.start() sub1_process.start() From 56ba77e7c1a1828fbdc8c6712f68e443ce3e278b Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 12 Feb 2019 14:20:34 +0100 Subject: [PATCH 29/35] added concurrent reading test --- tests/test_live.py | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/test_live.py b/tests/test_live.py index a692056..75ff7b0 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -9,6 +9,7 @@ import unittest import lmdb +import logthis import temppathlib import persipubsub.environment @@ -20,6 +21,41 @@ # pylint: disable=missing-docstring +FIRST_SUB_STARTED_READING = False + + +def subscriber_receive_first(sub: persipubsub.subscriber.Subscriber) -> None: + with sub.receive() as _: + global FIRST_SUB_STARTED_READING # pylint: disable=global-statement + FIRST_SUB_STARTED_READING = True + time.sleep(2) + + +def subscriber_receive_second(sub: persipubsub.subscriber.Subscriber) -> None: + global FIRST_SUB_STARTED_READING # pylint: disable=global-statement + + start = time.time() + while not FIRST_SUB_STARTED_READING: + if time.time() - start > 10: + raise TimeoutError + time.sleep(0.1) + + with sub.receive() as _: + if time.time() - start < 2: + assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub.queue.env, lmdb.Environment) + assert isinstance(sub.queue.env.path(), str) + assert isinstance(sub.identifier, str) + result = pathlib.Path(sub.queue.env.path()) / sub.identifier + result.write_text('pass') + else: + assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub.queue.env, lmdb.Environment) + assert isinstance(sub.queue.env.path(), str) + assert isinstance(sub.identifier, str) + result = pathlib.Path(sub.queue.env.path()) / sub.identifier + result.write_text('fail') + def receive(sub: persipubsub.subscriber.Subscriber, num_msg: int, @@ -418,6 +454,38 @@ def test_multiprocess_race_condition_of_the_component_publisher_one_env( self.assertEqual(num_processes * num_msg, txn.stat(db=pending_db)['entries']) + def test_2_subscriber_non_blocking(self) -> None: + with temppathlib.TemporaryDirectory() as tmp_dir: + env = persipubsub.environment.new_environment(path=tmp_dir.path) + _ = env.new_control(subscriber_ids=['sub1', 'sub2']) + + pub = env.new_publisher() + sub1 = env.new_subscriber(identifier='sub1') + sub2 = env.new_subscriber(identifier='sub2') + + pub.send(msg='msg for two subscriber'.encode('utf-8')) + + sub1_thread = threading.Thread( + target=subscriber_receive_first, kwargs={ + 'sub': sub1, + }) + sub2_thread = threading.Thread( + target=subscriber_receive_second, kwargs={ + 'sub': sub2, + }) + sub1_thread.start() + sub2_thread.start() + + for thread in [sub1_thread, sub2_thread]: + thread.join() + + assert isinstance(sub2.queue, persipubsub.queue._Queue) # pylint: disable=protected-access + assert isinstance(sub2.queue.env, lmdb.Environment) + assert isinstance(sub2.queue.env.path(), str) + assert isinstance(sub2.identifier, str) + result = pathlib.Path(sub2.queue.env.path()) / sub2.identifier + self.assertEqual('pass', result.read_text()) + if __name__ == '__main__': unittest.main() From 7b90576a1c58d095a485cc2929b84637d2f1928f Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 12 Feb 2019 16:06:42 +0100 Subject: [PATCH 30/35] Fixed review complains --- persipubsub/__init__.py | 13 ++--- persipubsub/control.py | 53 +++++++++---------- persipubsub/environment.py | 28 +++++++--- persipubsub/publisher.py | 13 ++--- persipubsub/queue.py | 104 ++++++++++++++++++++----------------- persipubsub/subscriber.py | 9 ++-- tests/__init__.py | 2 +- tests/test_control.py | 12 ++--- tests/test_live.py | 3 -- 9 files changed, 129 insertions(+), 108 deletions(-) diff --git a/persipubsub/__init__.py b/persipubsub/__init__.py index 2582202..1173490 100644 --- a/persipubsub/__init__.py +++ b/persipubsub/__init__.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -"""Distribute messages persistent from publisher to subscriber.""" -import pathlib +"""Set default values of persipubsub and offers encoding tools.""" from typing import Optional import lmdb @@ -32,7 +31,7 @@ SUBSCRIBER_IDS_KEY = "subscriber_ids".encode(ENCODING) -def encoding(string: str) -> bytes: +def str_to_bytes(string: str) -> bytes: """ Encode a string with utf-8 encoding. @@ -42,7 +41,7 @@ def encoding(string: str) -> bytes: return string.encode(encoding=ENCODING) -def decoding(encoded_str: bytes) -> str: +def bytes_to_str(encoded_str: bytes) -> str: """ Decode bytes with utf-8 encoding. @@ -72,9 +71,11 @@ def bytes_to_int(array_of_bytes: bytes) -> int: return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) -def get_queue_data(key: bytes, env: lmdb.Environment) -> Optional[bytes]: +def lookup_queue_data(key: bytes, env: lmdb.Environment) -> Optional[bytes]: """ - Get queue data. + Lookup set value for a given key in named database 'queue_db'. + + Value stored in 'queue_db' are high water mark values and pruning strategy. :param key: for lookup :param env: environment that stores queue data diff --git a/persipubsub/control.py b/persipubsub/control.py index 6b2c067..2990ec9 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Setup and control persistent distribution of messages.""" +"""Setup and control persistent queue.""" import pathlib from typing import List, Optional, Sequence, Set @@ -18,7 +18,7 @@ def set_hwm(hwm: persipubsub.queue.HighWaterMark, Set high water mark values for queue. :param hwm: high water mark values - :param env: queue environment + :param env: open LMDB environment :return: """ with env.begin(write=True) as txn: @@ -43,27 +43,28 @@ def set_strategy(strategy: persipubsub.queue.Strategy, Set pruning strategy for queue. :param strategy: pruning strategy - :param env: queue environment + :param env: open LMDB environment :return: """ with env.begin(write=True) as txn: queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) txn.put( key=persipubsub.STRATEGY_KEY, - value=persipubsub.encoding(str(strategy.name)), + value=persipubsub.str_to_bytes(str(strategy.name)), db=queue_db) -@icontract.require(lambda sub_id: sub_id.find(' ') == -1) +@icontract.require(lambda sub_id: ' ' not in sub_id) def _add_sub(sub_id: str, env: lmdb.Environment) -> None: """ - Add a subscriber and create its lmdb. + Add a subscriber and create its LMDB. :param sub_id: ID of the subscriber which should be added - :param env: queue environment + :param env: open LMDB environment """ with env.begin(write=True) as txn: - _ = env.open_db(key=persipubsub.encoding(sub_id), txn=txn, create=True) + _ = env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=True) queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) subscriber_ids = txn.get( @@ -72,7 +73,7 @@ def _add_sub(sub_id: str, env: lmdb.Environment) -> None: if subscriber_ids is None: subscriber_list = [] # type: List[str] else: - subscriber_list = persipubsub.decoding( + subscriber_list = persipubsub.bytes_to_str( encoded_str=subscriber_ids).split(' ') subscriber_set = set(subscriber_list) subscriber_set.add(sub_id) @@ -84,7 +85,7 @@ def _add_sub(sub_id: str, env: lmdb.Environment) -> None: class Control: - """Control and maintain one queue.""" + """Control and maintain a queue.""" def __init__(self, path: pathlib.Path, @@ -93,6 +94,7 @@ def __init__(self, Initialize control class. :param path: to the queue. + :param env: open LMDB environment """ self.path = path if not self.path.exists(): @@ -109,7 +111,6 @@ def __init__(self, max_db_num=persipubsub.MAX_DB_NUM, max_db_size_bytes=persipubsub.MAX_DB_SIZE_BYTES) - # pylint: disable=too-many-arguments def init(self, subscriber_ids: Optional[Sequence[str]] = None, high_watermark: persipubsub.queue.HighWaterMark = persipubsub. @@ -124,7 +125,8 @@ def init(self, :param strategy: used to prune queue :return: """ - if self.check_queue_is_initialized(): + # pylint: disable=too-many-arguments + if self.is_initialized(): self._reinitialize_queue() else: if subscriber_ids is None: @@ -173,7 +175,7 @@ def _initialize_queue(self, ) # type: persipubsub.queue._Queue self.queue.init(path=self.path, env=self.env) - def check_queue_is_initialized(self) -> bool: + def is_initialized(self) -> bool: """ Check if queue is initialized. @@ -188,7 +190,7 @@ def check_queue_is_initialized(self) -> bool: for key in keys: try: - value = persipubsub.get_queue_data(key=key, env=self.env) + value = persipubsub.lookup_queue_data(key=key, env=self.env) except lmdb.NotFoundError: return False @@ -204,7 +206,7 @@ def clear_all_subscribers(self) -> None: with self.queue.env.begin(write=True) as txn: for sub_id in self.subscriber_ids: sub_db = self.queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) txn.drop(db=sub_db) pending_db = self.queue.env.open_db( @@ -218,7 +220,7 @@ def clear_all_subscribers(self) -> None: txn.drop(db=data_db, delete=False) def prune_dangling_messages(self) -> None: - """Prune all dangling messages from the lmdb.""" + """Prune all dangling messages from the LMDB.""" assert isinstance(self.queue, persipubsub.queue._Queue) persipubsub.queue._prune_dangling_messages_for( queue=self.queue, subscriber_ids=list(self.subscriber_ids)) @@ -230,14 +232,13 @@ def _prune_all_messages_for(self, sub_id: str) -> None: :param sub_id: ID of the subscriber of which all messages should be pruned - :param queue: of which some subscriber's messages should be pruned """ msg_of_sub = set() assert isinstance(self.queue, persipubsub.queue._Queue) assert isinstance(self.queue.env, lmdb.Environment) with self.queue.env.begin(write=True) as txn: sub_db = self.queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) cursor = txn.cursor(db=sub_db) # check if database is not empty @@ -260,17 +261,16 @@ def _prune_all_messages_for(self, sub_id: str) -> None: value=persipubsub.int_to_bytes(decreased_pending_num), db=pending_db) - def _remove_sub(self, sub_id: str, env: lmdb.Environment) -> None: + def _remove_sub(self, sub_id: str) -> None: """ Remove a subscriber and delete all its messages. :param sub_id: ID of the subscriber which should be removed - :param queue: from which the subscriber is removed """ msg_of_sub = set() - with env.begin(write=True) as txn: - sub_db = env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) + with self.env.begin(write=True) as txn: + sub_db = self.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) cursor = txn.cursor(db=sub_db) # check if database is not empty if cursor.first(): @@ -278,7 +278,7 @@ def _remove_sub(self, sub_id: str, env: lmdb.Environment) -> None: msg_of_sub.add(key) txn.drop(db=sub_db) - pending_db = env.open_db( + pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) for key in msg_of_sub: @@ -291,11 +291,12 @@ def _remove_sub(self, sub_id: str, env: lmdb.Environment) -> None: value=persipubsub.int_to_bytes(decreased_pending_num), db=pending_db) - queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=False) + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=False) subscriber_ids = txn.get( key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) - subscriber_list = persipubsub.decoding( + subscriber_list = persipubsub.bytes_to_str( encoded_str=subscriber_ids).split(' ') subscriber_set = set(subscriber_list) subscriber_set.remove(sub_id) diff --git a/persipubsub/environment.py b/persipubsub/environment.py index 3fd31c6..3f3e3c2 100644 --- a/persipubsub/environment.py +++ b/persipubsub/environment.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Fabricate new persipubsub components.""" +"""Create new persipubsub components.""" import pathlib from typing import Any, Optional, Sequence @@ -13,7 +13,21 @@ class Environment: - """Fabricate persipubsub components.""" + """ + Create persipubsub components. + + Only one environment of each queue per process allowed and it's forbidden to + fork environment or any child components to multiple processes. + In that case, persipubsub is multi-threading and multi-process safe. + If multiple environments of the same queue are active on the same process, + or environment is forked to multiple processes the lock is broken and + correctness can't be guaranteed. + + :ivar path: to the queue + :vartype path: pathlib.Path + :ivar env: only open lmdb environment of process + :vartype env: lmdb.Environment + """ def __init__(self, path: pathlib.Path) -> None: """ @@ -43,11 +57,9 @@ def new_control(self, strategy: persipubsub.queue.Strategy = persipubsub.queue. Strategy.prune_first) -> persipubsub.control.Control: """ - Fabricate a new control. + Create a new control. :param subscriber_ids: subscribers of the queue - :param max_readers: max number of reader of the lmdb - :param max_size: max size of the lmdb in bytes :param high_watermark: high water mark limit of the queue :param strategy: used to prune queue :return: Control to create and maintain queue @@ -62,7 +74,7 @@ def new_control(self, def new_publisher( self, autosync: bool = False) -> persipubsub.publisher.Publisher: """ - Fabricate a new publisher. + Create a new publisher. :param autosync: if True, store data automatically in lmdb :return: Publisher to send messages @@ -74,7 +86,7 @@ def new_publisher( def new_subscriber(self, identifier: str) -> persipubsub.subscriber.Subscriber: """ - Fabricate a new subscriber. + Create a new subscriber. :param identifier: of the subscriber :return: Subscriber to receive messages @@ -86,7 +98,7 @@ def new_subscriber(self, def new_environment(path: pathlib.Path) -> Environment: """ - Fabricate a new environment. + Create a new environment. :param path: path to the queue :return: Environment to create control, publisher and subscriber diff --git a/persipubsub/publisher.py b/persipubsub/publisher.py index d5231b5..22c77de 100644 --- a/persipubsub/publisher.py +++ b/persipubsub/publisher.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Distribute messages persistent from publisher to subscriber.""" +"""Publish messages to a queue and save them persistently.""" import pathlib from typing import Any, List, Optional, Union @@ -13,7 +13,7 @@ class Publisher: """ - Create Publisher ready to send messages. + Handle publishing messages to the queue. :ivar queue: on which messages are published :vartype queue: persipubsub.queue.Queue @@ -34,6 +34,7 @@ def init(self, Initialize. :param path: path to the queue + :param env: open lmdb environment :param autosync: if True, store data automatically in lmdb """ self.queue = persipubsub.queue._Queue() # pylint: disable=protected-access @@ -49,18 +50,18 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: def send(self, msg: bytes) -> None: """ - Send message to subscribers. + Write one message to queue in one transaction. - :param msg: to send to all subscribers + :param msg: to queue that all subscribers can read it """ assert isinstance(self.queue, persipubsub.queue._Queue) self.queue.put(msg=msg) def send_many(self, msgs: List[bytes]) -> None: """ - Send messages to subscribers. + Write multiple messages to queue in one transaction. - :param msgs: to send to all subscribers + :param msgs: to queue that all subscribers can read them """ assert isinstance(self.queue, persipubsub.queue._Queue) if self.autosync: diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 2e5999a..1aaed25 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Store messages in a local LMDB queue.""" +"""Store messages in a local LMDB.""" import datetime import enum import pathlib @@ -16,7 +16,7 @@ class Strategy(enum.Enum): - """Store possible strategies.""" + """Hold possible strategies.""" prune_first = 0 prune_last = 1 @@ -26,17 +26,20 @@ def _parse_strategy(strategy: str) -> Strategy: """ Parse overflow strategy. - :param strategy: Strategy stored in config + :param strategy: Strategy stored in LMDB :return: set overflow strategy """ if Strategy.prune_first.name == strategy: return Strategy.prune_first - return Strategy.prune_last + if Strategy.prune_last.name == strategy: + return Strategy.prune_last + + raise ValueError("Unknown strategy: '{}' choosen.".format(strategy)) class HighWaterMark: - """Store high water mark limits.""" + """Hold high water mark limits.""" def __init__(self, msg_timeout_secs: Optional[int] = None, @@ -48,7 +51,7 @@ def __init__(self, :param msg_timeout_secs: time after which msg is classified as dangling msg (secs) :param max_msgs_num: maximal amount of msg - :param hwm_lmdb_size_bytes: high water mark for total size of lmdb + :param hwm_lmdb_size_bytes: high water mark for total size of LMDB (bytes) """ self.msg_timeout_secs = MSG_TIMEOUT_SECS \ @@ -68,10 +71,10 @@ def _initialize_environment( Initialize the queue; the queue directory is assumed to exist. :param queue_dir: where the queue is stored - :param max_reader_num: maximal number of reader + :param max_reader_num: maximal number of readers :param max_db_num: maximal number of databases :param max_db_size_bytes: maximal size of database (bytes) - :return: Load or if needed create LMDB queue from directory + :return: Load or if needed create LMDB from directory """ if not queue_dir.exists(): raise RuntimeError( @@ -101,7 +104,7 @@ def _initialize_environment( def _prune_dangling_messages_for(queue: '_Queue', subscriber_ids: List[str]) -> None: """ - Prune all dangling messages for subscribers of a queue from lmdb. + Prune all dangling messages for subscribers of a queue from LMDB. :param queue: of which dangling messages should be pruned :param subscriber_ids: subscribers of which dangling msgs should be pruned @@ -126,8 +129,8 @@ def _prune_dangling_messages_for(queue: '_Queue', if persipubsub.bytes_to_int(pending_subscribers_num) == 0: msgs_to_delete.add(key) - # subscriber might still await this messages after the timeout. - # This messages needs also to be removed from all subscribers. + # subscriber might still await these messages after the timeout. + # These messages needs also to be removed from all subscribers. msgs_to_delete_timeout = set() assert isinstance(queue.hwm, HighWaterMark) with queue.env.begin(db=meta_db) as txn: @@ -149,14 +152,17 @@ def _prune_dangling_messages_for(queue: '_Queue', with queue.env.begin(write=True) as txn: for sub_id in subscriber_ids: sub_db = queue.env.open_db( - key=persipubsub.encoding(sub_id), txn=txn, create=False) + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=False) for key in msgs_to_delete_timeout: txn.delete(key=key, db=sub_db) class _Queue: """ - Queue messages persistently from many publishers for many subscribers. + Represent a message queue. + + The messages come from many publishers and many subscribers. + They are stored persistently in an LMDB database. :ivar path: to the queue :vartype config_pth: pathlib.Path @@ -180,8 +186,6 @@ def __init__(self) -> None: self.strategy = None # type: Optional[Strategy] self.subscriber_ids = None # type: Optional[List[str]] - # pylint: disable=too-many-arguments - # pylint: disable=too-many-locals def init(self, path: Union[pathlib.Path, str], env: Optional[lmdb.Environment] = None) -> None: @@ -189,10 +193,10 @@ def init(self, Initialize the queue. :param path: where the queue is stored - :param max_reader_num: maximal number of reader - :param max_db_num: maximal number of databases - :param max_db_size_bytes: maximal size of database (bytes) + :param env: open LMDB environment """ + # pylint: disable=too-many-arguments + # pylint: disable=too-many-locals self.path = path if isinstance(path, pathlib.Path) \ else pathlib.Path(path) @@ -212,19 +216,19 @@ def init(self, _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) _ = self.env.open_db(key=persipubsub.QUEUE_DB, txn=txn, create=True) - msg_timeout_secs_bytes = persipubsub.get_queue_data( + msg_timeout_secs_bytes = persipubsub.lookup_queue_data( key=persipubsub.MSG_TIMEOUT_SECS_KEY, env=self.env) msg_timeout_secs = msg_timeout_secs_bytes \ if msg_timeout_secs_bytes is None else persipubsub.bytes_to_int( msg_timeout_secs_bytes) # type: Optional[int] - max_msgs_num_bytes = persipubsub.get_queue_data( + max_msgs_num_bytes = persipubsub.lookup_queue_data( key=persipubsub.MAX_MSGS_NUM_KEY, env=self.env) max_msgs_num = max_msgs_num_bytes \ if max_msgs_num_bytes is None else persipubsub.bytes_to_int( max_msgs_num_bytes) # type: Optional[int] - hwm_lmdb_size_bytes = persipubsub.get_queue_data( + hwm_lmdb_size_bytes = persipubsub.lookup_queue_data( key=persipubsub.HWM_DB_SIZE_BYTES_KEY, env=self.env) hwm_lmdb_size = hwm_lmdb_size_bytes \ if hwm_lmdb_size_bytes is None else persipubsub.bytes_to_int( @@ -235,21 +239,21 @@ def init(self, max_msgs_num=max_msgs_num, hwm_lmdb_size_bytes=hwm_lmdb_size) - strategy = persipubsub.get_queue_data( + strategy = persipubsub.lookup_queue_data( key=persipubsub.STRATEGY_KEY, env=self.env) - strategy_decoded = "" if strategy is None else persipubsub.decoding( + strategy_decoded = "" if strategy is None else persipubsub.bytes_to_str( encoded_str=strategy) self.strategy = _parse_strategy(strategy=strategy_decoded) - subscriber_list = persipubsub.get_queue_data( + subscriber_list = persipubsub.lookup_queue_data( key=persipubsub.SUBSCRIBER_IDS_KEY, env=self.env) if subscriber_list is None: self.subscriber_ids = [] else: - self.subscriber_ids = persipubsub.decoding( + self.subscriber_ids = persipubsub.bytes_to_str( encoded_str=subscriber_list).split(' ') def __enter__(self) -> '_Queue': @@ -261,10 +265,9 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: def put(self, msg: bytes) -> None: """ - Put message to lmdb queue. + Put message to LMDB in one transaction. - :param msg: proto message send from publisher to subscribers - :param sub_list: List of subscribers encoded in 'utf-8' + :param msg: message in bytes :return: """ # every publisher always prunes queue before sending a message. @@ -276,33 +279,32 @@ def put(self, msg: bytes) -> None: pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) txn.put( - key=persipubsub.encoding(msg_id), + key=persipubsub.str_to_bytes(msg_id), value=persipubsub.int_to_bytes(len(self.subscriber_ids)), db=pending_db) meta_db = self.env.open_db( key=persipubsub.META_DB, txn=txn, create=False) txn.put( - key=persipubsub.encoding(msg_id), + key=persipubsub.str_to_bytes(msg_id), value=persipubsub.int_to_bytes( int(datetime.datetime.utcnow().timestamp())), db=meta_db) data_db = self.env.open_db( key=persipubsub.DATA_DB, txn=txn, create=False) - txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) + txn.put(key=persipubsub.str_to_bytes(msg_id), value=msg, db=data_db) for sub in self.subscriber_ids: sub_db = self.env.open_db( - key=persipubsub.encoding(sub), txn=txn, create=False) - txn.put(key=persipubsub.encoding(msg_id), db=sub_db) + key=persipubsub.str_to_bytes(sub), txn=txn, create=False) + txn.put(key=persipubsub.str_to_bytes(msg_id), db=sub_db) def put_many_flush_once(self, msgs: List[bytes]) -> None: """ - Put many message to lmdb queue. + Put multiple message to LMDB in one transaction. - :param msgs: proto messages send from publisher to subscribers - :param sub_list: List of subscribers encoded in 'utf-8' + :param msgs: messages in bytes :return: """ # every publisher always prunes queue before sending a message. @@ -324,33 +326,36 @@ def put_many_flush_once(self, msgs: List[bytes]) -> None: for sub in self.subscriber_ids: sub_dbs.add( self.env.open_db( - key=persipubsub.encoding(sub), txn=txn, create=False)) + key=persipubsub.str_to_bytes(sub), + txn=txn, + create=False)) for msg in msgs: msg_id = str(datetime.datetime.utcnow().timestamp()) + str( uuid.uuid4()) txn.put( - key=persipubsub.encoding(msg_id), + key=persipubsub.str_to_bytes(msg_id), value=persipubsub.int_to_bytes(len(self.subscriber_ids)), db=pending_db) txn.put( - key=persipubsub.encoding(msg_id), + key=persipubsub.str_to_bytes(msg_id), value=persipubsub.int_to_bytes( int(datetime.datetime.utcnow().timestamp())), db=meta_db) - txn.put(key=persipubsub.encoding(msg_id), value=msg, db=data_db) + txn.put( + key=persipubsub.str_to_bytes(msg_id), value=msg, db=data_db) for sub_db in sub_dbs: - txn.put(key=persipubsub.encoding(msg_id), db=sub_db) + txn.put(key=persipubsub.str_to_bytes(msg_id), db=sub_db) def front(self, identifier: str) -> Optional[bytes]: """ - Peek at next message in lmdb queue. + Peek at next message in LMDB. - Load from LMDB queue into memory and process msg afterwards. + Load from LMDB into memory and process msg afterwards. :param identifier: Subscriber ID :return: @@ -358,7 +363,7 @@ def front(self, identifier: str) -> Optional[bytes]: assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=False) as txn: sub_db = self.env.open_db( - key=persipubsub.encoding(identifier), txn=txn, create=False) + key=persipubsub.str_to_bytes(identifier), txn=txn, create=False) data_db = self.env.open_db( key=persipubsub.DATA_DB, txn=txn, create=False) @@ -382,7 +387,7 @@ def pop(self, identifier: str) -> None: assert isinstance(self.env, lmdb.Environment) with self.env.begin(write=True) as txn: sub_db = self.env.open_db( - key=persipubsub.encoding(identifier), txn=txn, create=False) + key=persipubsub.str_to_bytes(identifier), txn=txn, create=False) pending_db = self.env.open_db( key=persipubsub.PENDING_DB, txn=txn, create=False) @@ -406,6 +411,9 @@ def prune_dangling_messages(self) -> None: """ Prune dangling messages in the queue. + Definition of dangling messages: + - having no pending subscribers + - exists longer than timeout allows :return: """ assert isinstance(self.subscriber_ids, List) @@ -414,7 +422,7 @@ def prune_dangling_messages(self) -> None: def check_current_lmdb_size(self) -> int: """ - Check current lmdb size in bytes. + Check current LMDB size in bytes. Check size of data database by approximating size with multiplying page size with number of pages. @@ -437,7 +445,7 @@ def count_msgs(self) -> int: """ Count number of messages in database. - Count number of messages stored in meta database. + Count number of messages stored in named database 'meta_db'. :return: number of messages in database """ @@ -514,7 +522,7 @@ def prune_messages(self) -> None: for sub in self.subscriber_ids: sub_db = self.env.open_db( - key=persipubsub.encoding(sub), txn=txn, create=False) + key=persipubsub.str_to_bytes(sub), txn=txn, create=False) dbs.append(sub_db) for key in messages_to_delete: diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index 868d520..b9852c5 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Receive messages persistent from publisher to subscriber.""" +"""Receive messages persistently from the queue.""" import contextlib import datetime @@ -17,7 +17,7 @@ class Subscriber: """ - Create Subscriber ready to receive messages. + Handle receiving messages stored in the queue. :ivar identifier: subscriber ID :vartype pub_id: str @@ -39,6 +39,7 @@ def init(self, :param identifier: unique subscriber id :param path: path to the queue + :param env: open lmdb environment """ self.identifier = identifier assert isinstance(self.identifier, str) @@ -59,7 +60,7 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: def receive(self, timeout: int = 60, retries: int = 10) -> Iterator[Optional[bytes]]: """ - Receive messages from the publisher. + Receive messages from the queue. :param timeout: time waiting for a message. If none arrived until the timeout then None will be returned. (secs) @@ -112,7 +113,7 @@ def receive_to_top(self, timeout: int = 60, assert isinstance(self.identifier, str) with self.queue.env.begin(write=False) as txn: sub_db = self.queue.env.open_db( - key=persipubsub.encoding(self.identifier), + key=persipubsub.str_to_bytes(self.identifier), txn=txn, create=False) sub_stat = txn.stat(db=sub_db) diff --git a/tests/__init__.py b/tests/__init__.py index 153e250..f30c5e8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -47,7 +47,7 @@ def test_get_data(self) -> None: value="value".encode(ENCODING), db=queue_db) - data = persipubsub.get_queue_data( + data = persipubsub.lookup_queue_data( key="key".encode(ENCODING), env=env) self.assertEqual(b'value', data) diff --git a/tests/test_control.py b/tests/test_control.py index 9afdd4a..2a5cb50 100644 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -57,7 +57,7 @@ def test_del_sub(self) -> None: assert isinstance(control.queue, persipubsub.queue._Queue) assert isinstance(control.queue.env, lmdb.Environment) - control._remove_sub(sub_id="sub2", env=control.queue.env) + control._remove_sub(sub_id="sub2") expected_db_keys = [ b'data_db', b'meta_db', b'pending_db', b'queue_db', b'sub1' @@ -190,7 +190,7 @@ def test_prune_all_messages_for_subscriber(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub"]) - msg = persipubsub.encoding("hello world!") + msg = persipubsub.str_to_bytes("hello world!") assert isinstance(control.queue, persipubsub.queue._Queue) assert isinstance(control.queue.env, lmdb.Environment) control.queue.put(msg=msg) @@ -198,7 +198,7 @@ def test_prune_all_messages_for_subscriber(self) -> None: with control.queue.env.begin(write=False) as txn: sub_db = control.queue.env.open_db( - key=persipubsub.encoding('sub'), txn=txn, create=False) + key=persipubsub.str_to_bytes('sub'), txn=txn, create=False) sub_stat = txn.stat(db=sub_db) self.assertEqual(2, sub_stat['entries']) @@ -206,7 +206,7 @@ def test_prune_all_messages_for_subscriber(self) -> None: with control.queue.env.begin(write=False) as txn: sub_db = control.queue.env.open_db( - key=persipubsub.encoding('sub'), txn=txn, create=False) + key=persipubsub.str_to_bytes('sub'), txn=txn, create=False) sub_stat = txn.stat(db=sub_db) self.assertEqual(0, sub_stat['entries']) @@ -214,12 +214,12 @@ def test_is_initialized(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = setup(path=tmp_dir.path, sub_list=["sub"]) - self.assertTrue(control.check_queue_is_initialized()) + self.assertTrue(control.is_initialized()) def test_is_not_initialized(self) -> None: with temppathlib.TemporaryDirectory() as tmp_dir: control = persipubsub.control.Control(path=tmp_dir.path) - self.assertFalse(control.check_queue_is_initialized()) + self.assertFalse(control.is_initialized()) if __name__ == '__main__': diff --git a/tests/test_live.py b/tests/test_live.py index 75ff7b0..e076b8f 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -1,7 +1,5 @@ #!/usr/bin/env python """Test persipubsub live.""" - -# pylint: disable=unused-import import multiprocessing import pathlib import threading @@ -9,7 +7,6 @@ import unittest import lmdb -import logthis import temppathlib import persipubsub.environment From 2ec61f5fedfb8043a6533f4263204f9640860334 Mon Sep 17 00:00:00 2001 From: slimeth Date: Mon, 18 Feb 2019 10:26:37 +0100 Subject: [PATCH 31/35] 2nd review change --- README.rst | 33 +++++---- persipubsub/control.py | 146 ++++++++++++++++++-------------------- persipubsub/queue.py | 14 +--- persipubsub/subscriber.py | 14 ++-- persipubsub_meta.py | 4 +- tests/test_queue.py | 8 +-- 6 files changed, 104 insertions(+), 115 deletions(-) diff --git a/README.rst b/README.rst index d54626b..8c88e5b 100644 --- a/README.rst +++ b/README.rst @@ -24,10 +24,8 @@ persipubsub :target: https://opensource.org/licenses/mit-license.php :alt: MIT License - - -``persipubsub`` implements a persistent, thread-safe and process-safe `lmdb -`_-queue for inter-process communication. +``persipubsub`` implements a persistent, thread-safe and process-safe queue for +inter-process communication, based on `lmdb `_. Primarily, we used `zeromq `_ for inter-process communication with a slight improvement through `persizmq @@ -92,10 +90,19 @@ The usage of the library consists of two steps: deployment and runtime Environment ----------- -For improve the accessibility of the library, an environment class lets you +To improve the accessibility of the library, an environment class lets you create and initialize any ``persipubsub`` component which you need in deployment or runtime step. +.. warning:: + + Only one environment of each queue per process allowed and it's forbidden to + fork environment or any child components to multiple processes. + In that case, persipubsub is multi-threading and multi-process safe. + If multiple environments of the same queue are active on the same process, + or environment is forked to multiple processes the lock is broken and + correctness can't be guaranteed. + Initialize environment ^^^^^^^^^^^^^^^^^^^^^^ @@ -135,11 +142,11 @@ Initialize queue # Initialize a queue with default values. control = env.new_control() - # Define all optional parameters of the queue. + # Or define all optional parameters of the queue. hwm = persipubsub.queue._HighWaterMark() strategy = persipubsub.queue._Strategy.prune_first - control = env.new_control(subscriber_ids=["sub1", "sub2"], max_readers=2, - max_size=10*4096, high_watermark=hwm, + control = env.new_control(subscriber_ids=["sub1", "sub2"], + high_watermark=hwm, strategy=strategy) Prune all dangling messages @@ -252,13 +259,13 @@ Receive a message Catch up with latest message """""""""""""""""""""""""""" -Used in the case that a particular subscriber cares only about the very last -message and other subscribers care about all the messages in the queue. +Can be used in the case when a particular subscriber cares only about the very +last message. The messages are not popped for other subscribers. .. note:: - For the other use case, when you only want to store the latest message and all - subscribers are interested only in the latest, then use high water mark - max_msgs_num = 1. + If you want to store only the latest message for all subscribers, then use + high water mark max_msgs_num = 1. + .. code-block:: python diff --git a/persipubsub/control.py b/persipubsub/control.py index 2990ec9..83a3927 100644 --- a/persipubsub/control.py +++ b/persipubsub/control.py @@ -12,78 +12,6 @@ # pylint: disable=protected-access -def set_hwm(hwm: persipubsub.queue.HighWaterMark, - env: lmdb.Environment) -> None: - """ - Set high water mark values for queue. - - :param hwm: high water mark values - :param env: open LMDB environment - :return: - """ - with env.begin(write=True) as txn: - queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) - txn.put( - key=persipubsub.HWM_DB_SIZE_BYTES_KEY, - value=persipubsub.int_to_bytes(hwm.hwm_lmdb_size_bytes), - db=queue_db) - txn.put( - key=persipubsub.MAX_MSGS_NUM_KEY, - value=persipubsub.int_to_bytes(hwm.max_msgs_num), - db=queue_db) - txn.put( - key=persipubsub.MSG_TIMEOUT_SECS_KEY, - value=persipubsub.int_to_bytes(hwm.msg_timeout_secs), - db=queue_db) - - -def set_strategy(strategy: persipubsub.queue.Strategy, - env: lmdb.Environment) -> None: - """ - Set pruning strategy for queue. - - :param strategy: pruning strategy - :param env: open LMDB environment - :return: - """ - with env.begin(write=True) as txn: - queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) - txn.put( - key=persipubsub.STRATEGY_KEY, - value=persipubsub.str_to_bytes(str(strategy.name)), - db=queue_db) - - -@icontract.require(lambda sub_id: ' ' not in sub_id) -def _add_sub(sub_id: str, env: lmdb.Environment) -> None: - """ - Add a subscriber and create its LMDB. - - :param sub_id: ID of the subscriber which should be added - :param env: open LMDB environment - """ - with env.begin(write=True) as txn: - _ = env.open_db( - key=persipubsub.str_to_bytes(sub_id), txn=txn, create=True) - - queue_db = env.open_db(persipubsub.QUEUE_DB, txn=txn, create=True) - subscriber_ids = txn.get( - key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) - - if subscriber_ids is None: - subscriber_list = [] # type: List[str] - else: - subscriber_list = persipubsub.bytes_to_str( - encoded_str=subscriber_ids).split(' ') - subscriber_set = set(subscriber_list) - subscriber_set.add(sub_id) - subscriber_str = " ".join(subscriber_set) - txn.put( - key=persipubsub.SUBSCRIBER_IDS_KEY, - value=subscriber_str.encode(persipubsub.ENCODING), - db=queue_db) - - class Control: """Control and maintain a queue.""" @@ -164,11 +92,11 @@ def _initialize_queue(self, # 4 queues (data db, meta db, pending db, queue db) # + each subscriber has its own db - set_hwm(hwm=high_watermark, env=self.env) - set_strategy(strategy=strategy, env=self.env) + self.set_hwm(hwm=high_watermark) + self.set_strategy(strategy=strategy) for sub in self.subscriber_ids: - _add_sub(sub_id=sub, env=self.env) + self._add_sub(sub_id=sub) # load initialized queue self.queue = persipubsub.queue._Queue( @@ -261,6 +189,35 @@ def _prune_all_messages_for(self, sub_id: str) -> None: value=persipubsub.int_to_bytes(decreased_pending_num), db=pending_db) + @icontract.require(lambda sub_id: ' ' not in sub_id) + def _add_sub(self, sub_id: str) -> None: + """ + Add a subscriber and create its LMDB. + + :param sub_id: ID of the subscriber which should be added + """ + with self.env.begin(write=True) as txn: + _ = self.env.open_db( + key=persipubsub.str_to_bytes(sub_id), txn=txn, create=True) + + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=True) + subscriber_ids = txn.get( + key=persipubsub.SUBSCRIBER_IDS_KEY, db=queue_db) + + if subscriber_ids is None: + subscriber_list = [] # type: List[str] + else: + subscriber_list = persipubsub.bytes_to_str( + encoded_str=subscriber_ids).split(' ') + subscriber_set = set(subscriber_list) + subscriber_set.add(sub_id) + subscriber_str = " ".join(subscriber_set) + txn.put( + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=subscriber_str.encode(persipubsub.ENCODING), + db=queue_db) + def _remove_sub(self, sub_id: str) -> None: """ Remove a subscriber and delete all its messages. @@ -307,3 +264,42 @@ def _remove_sub(self, sub_id: str) -> None: db=queue_db) self.subscriber_ids.remove(sub_id) + + def set_hwm(self, hwm: persipubsub.queue.HighWaterMark) -> None: + """ + Set high water mark values for queue. + + :param hwm: high water mark values + :return: + """ + with self.env.begin(write=True) as txn: + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, + value=persipubsub.int_to_bytes(hwm.hwm_lmdb_size_bytes), + db=queue_db) + txn.put( + key=persipubsub.MAX_MSGS_NUM_KEY, + value=persipubsub.int_to_bytes(hwm.max_msgs_num), + db=queue_db) + txn.put( + key=persipubsub.MSG_TIMEOUT_SECS_KEY, + value=persipubsub.int_to_bytes(hwm.msg_timeout_secs), + db=queue_db) + + def set_strategy(self, strategy: persipubsub.queue.Strategy) -> None: + """ + Set pruning strategy for queue. + + :param strategy: pruning strategy + :param env: open LMDB environment + :return: + """ + with self.env.begin(write=True) as txn: + queue_db = self.env.open_db( + persipubsub.QUEUE_DB, txn=txn, create=True) + txn.put( + key=persipubsub.STRATEGY_KEY, + value=persipubsub.str_to_bytes(str(strategy.name)), + db=queue_db) diff --git a/persipubsub/queue.py b/persipubsub/queue.py index 1aaed25..68ccb3d 100644 --- a/persipubsub/queue.py +++ b/persipubsub/queue.py @@ -80,24 +80,12 @@ def _initialize_environment( raise RuntimeError( "The queue directory does not exist: {}".format(queue_dir)) - # max_spare_txn: Read-only transactions to cache after becoming unused. - # Caching transactions avoids two allocations, one lock and linear scan of - # the shared environment per invocation of begin(), Transaction, get(), - # gets(), or cursor(). Should match the process’s maximum expected - # concurrent transactions (e.g. thread count). - # Setting max_spare_txn equals 0 doesn't allow any caching for read-only - # transactions to avoid the error: - # MDB_BAD_RSLOT: Invalid reuse of reader locktable slot. - # Error only occures when one publisher/subscriber is used in multiple - # processes. So max_spare_txn equal 1 is fine as we expect a publisher and - # subscriber running on the same process. env = lmdb.open( path=queue_dir.as_posix(), map_size=max_db_size_bytes, subdir=True, max_readers=max_reader_num, - max_dbs=max_db_num, - max_spare_txns=1) + max_dbs=max_db_num) return env diff --git a/persipubsub/subscriber.py b/persipubsub/subscriber.py index b9852c5..53b298d 100644 --- a/persipubsub/subscriber.py +++ b/persipubsub/subscriber.py @@ -97,12 +97,12 @@ def receive_to_top(self, timeout: int = 60, """ Pops all messages until the most recent one and receive the latest. - Used in the case that a particular subscriber cares only about the very - last message and other subscribers care about all the messages in the - queue. - For another use case, when you only want to store the latest message - and all subscribers are interested only in the latest, then use - high water mark max_msgs_num = 1. + Can be used in the case when a particular subscriber cares only about + the very last message. The messages are not popped for other + subscribers. + + If you want to store only the latest message for all subscribers, then + use high water mark max_msgs_num = 1. :param timeout: time waiting for a message. If none arrived until the timeout then None will be returned. (secs) @@ -125,8 +125,6 @@ def receive_to_top(self, timeout: int = 60, msg = None end = int(datetime.datetime.utcnow().timestamp()) + timeout - assert isinstance(self.queue, persipubsub.queue._Queue) - assert isinstance(self.identifier, str) try: while int(datetime.datetime.utcnow().timestamp()) <= end: msg = self.queue.front(identifier=self.identifier) diff --git a/persipubsub_meta.py b/persipubsub_meta.py index 5be4657..0ed375a 100644 --- a/persipubsub_meta.py +++ b/persipubsub_meta.py @@ -1,10 +1,10 @@ """Define meta information about persipubsub package.""" __title__ = 'persipubsub' -__description__ = 'Send messages persistent from publishers to subscribers.' +__description__ = 'Implements a persistent, thread-safe and process-safe queue for inter-process communication.' __url__ = 'https://github.com/Parquery/persipubsub' __version__ = '1.0.0' __author__ = 'Selim Naji, Adam Radomski and Marko Ristin' __author_email__ = 'selim.naji@parquery.com, adam.radomski@parquery.com, marko.ristin@gmail.com' __license__ = 'MIT' -__copyright__ = 'Copyright 2018 Parquery AG' \ No newline at end of file +__copyright__ = 'Copyright 2019 Parquery AG' \ No newline at end of file diff --git a/tests/test_queue.py b/tests/test_queue.py index b4c03c5..b49a605 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -250,16 +250,16 @@ def test_queue_initialisation(self) -> None: env = persipubsub.environment.Environment(path=tmp_dir.path) subscriber = 'sub' - _ = setup(env=env, sub_list=[subscriber]) + control = setup(env=env, sub_list=[subscriber]) hwm = persipubsub.queue.HighWaterMark( msg_timeout_secs=tests.TEST_MSG_TIMEOUT, max_msgs_num=tests.TEST_HWM_MSG_NUM, hwm_lmdb_size_bytes=tests.TEST_HWM_LMDB_SIZE) - persipubsub.control.set_hwm(hwm=hwm, env=env.env) - persipubsub.control.set_strategy( - strategy=persipubsub.queue.Strategy.prune_first, env=env.env) + control.set_hwm(hwm=hwm) + control.set_strategy( + strategy=persipubsub.queue.Strategy.prune_first) queue = env.new_control().queue assert isinstance(queue, persipubsub.queue._Queue) From 4edf2e549f48137eac41a4ec3e8f3e8a76f0a034 Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 19 Feb 2019 10:40:19 +0100 Subject: [PATCH 32/35] move python module to py folder --- .gitignore | 7 +- .travis.yml | 53 ++- README.rst | 3 + py/.gitignore | 65 ++++ .isort.cfg => py/.isort.cfg | 0 py/README.rst | 364 ++++++++++++++++++ {docs => py/docs}/Makefile | 0 {docs => py/docs}/source/changelog.rst | 0 {docs => py/docs}/source/conf.py | 1 - {docs => py/docs}/source/index.rst | 0 .../docs}/source/persipubsub/index.rst | 0 .../source/persipubsub/persipubsub.ctl.rst | 0 .../source/persipubsub/persipubsub.env.rst | 0 .../persipubsub/persipubsub.publisher.rst | 0 .../source/persipubsub/persipubsub.queue.rst | 0 .../docs}/source/persipubsub/persipubsub.rst | 0 .../persipubsub/persipubsub.subscriber.rst | 0 {docs => py/docs}/source/readme.rst | 0 mypy.ini => py/mypy.ini | 0 {persipubsub => py/persipubsub}/__init__.py | 0 {persipubsub => py/persipubsub}/control.py | 0 .../persipubsub}/environment.py | 0 {persipubsub => py/persipubsub}/publisher.py | 0 {persipubsub => py/persipubsub}/queue.py | 0 {persipubsub => py/persipubsub}/subscriber.py | 0 persipubsub_meta.py => py/persipubsub_meta.py | 0 precommit.py => py/precommit.py | 0 pylint.rc => py/pylint.rc | 0 .../requirements-doc.txt | 0 setup.py => py/setup.py | 2 +- style.yapf => py/style.yapf | 0 {tests => py/tests}/__init__.py | 3 - {tests => py/tests}/component_publisher.py | 1 - {tests => py/tests}/component_subscriber.py | 0 {tests => py/tests}/test_control.py | 0 {tests => py/tests}/test_environment.py | 0 {tests => py/tests}/test_live.py | 15 +- {tests => py/tests}/test_publisher.py | 1 - {tests => py/tests}/test_queue.py | 0 {tests => py/tests}/test_subscriber.py | 0 tox.ini => py/tox.ini | 0 41 files changed, 488 insertions(+), 27 deletions(-) create mode 100644 py/.gitignore rename .isort.cfg => py/.isort.cfg (100%) create mode 100644 py/README.rst rename {docs => py/docs}/Makefile (100%) rename {docs => py/docs}/source/changelog.rst (100%) rename {docs => py/docs}/source/conf.py (99%) rename {docs => py/docs}/source/index.rst (100%) rename {docs => py/docs}/source/persipubsub/index.rst (100%) rename {docs => py/docs}/source/persipubsub/persipubsub.ctl.rst (100%) rename {docs => py/docs}/source/persipubsub/persipubsub.env.rst (100%) rename {docs => py/docs}/source/persipubsub/persipubsub.publisher.rst (100%) rename {docs => py/docs}/source/persipubsub/persipubsub.queue.rst (100%) rename {docs => py/docs}/source/persipubsub/persipubsub.rst (100%) rename {docs => py/docs}/source/persipubsub/persipubsub.subscriber.rst (100%) rename {docs => py/docs}/source/readme.rst (100%) rename mypy.ini => py/mypy.ini (100%) rename {persipubsub => py/persipubsub}/__init__.py (100%) rename {persipubsub => py/persipubsub}/control.py (100%) rename {persipubsub => py/persipubsub}/environment.py (100%) rename {persipubsub => py/persipubsub}/publisher.py (100%) rename {persipubsub => py/persipubsub}/queue.py (100%) rename {persipubsub => py/persipubsub}/subscriber.py (100%) rename persipubsub_meta.py => py/persipubsub_meta.py (100%) rename precommit.py => py/precommit.py (100%) rename pylint.rc => py/pylint.rc (100%) rename requirements-doc.txt => py/requirements-doc.txt (100%) rename setup.py => py/setup.py (98%) rename style.yapf => py/style.yapf (100%) rename {tests => py/tests}/__init__.py (96%) rename {tests => py/tests}/component_publisher.py (96%) rename {tests => py/tests}/component_subscriber.py (100%) rename {tests => py/tests}/test_control.py (100%) rename {tests => py/tests}/test_environment.py (100%) rename {tests => py/tests}/test_live.py (98%) rename {tests => py/tests}/test_publisher.py (99%) rename {tests => py/tests}/test_queue.py (100%) rename {tests => py/tests}/test_subscriber.py (100%) rename tox.ini => py/tox.ini (100%) diff --git a/.gitignore b/.gitignore index b534413..e950645 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,9 @@ venv-doc/ .idea # Archive -obsolete/ \ No newline at end of file +obsolete/ + +# Project specific +messages/ +example/ +benchmarks/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index fd0e703..8992dbe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,40 @@ -language: python -python: - - "3.5" - - "3.6" -addons: - apt: - packages: -install: - - pip3 install -e .[dev] - - pip3 install coveralls -script: - - python3 precommit.py - - coveralls \ No newline at end of file +matrix: + include: + - language: python + python: + - "3.5" + - "3.6" + addons: + apt: + packages: + before_script: + - cd py + install: + - pip3 install -e .[dev] + - pip3 install coveralls + script: + - python3 precommit.py + - coveralls + - language: cpp + os: linux + dist: xenial + compiler: + - clang + - gcc + addons: + apt: + sources: + - ubuntu-toolchain-r-test + - llvm-toolchain-precise-3.8 + packages: + - g++-6 + - clang-3.8 + install: + - make get-deps + - "[ $CXX = g++ ] && export CXX=g++-6 || true" + - "[ $CXX = clang++ ] && export CXX=clang++-3.8 || true" + before_script: + - cd cpp + script: + - make + - make test \ No newline at end of file diff --git a/README.rst b/README.rst index 8c88e5b..b794c08 100644 --- a/README.rst +++ b/README.rst @@ -87,6 +87,9 @@ Usage The usage of the library consists of two steps: deployment and runtime +Python +====== + Environment ----------- diff --git a/py/.gitignore b/py/.gitignore new file mode 100644 index 0000000..0563776 --- /dev/null +++ b/py/.gitignore @@ -0,0 +1,65 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$class + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Sphinx documentation +docs/build/ + +# pyenv +.python-version + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# mypy +.mypy_cache/ + +# IDE +.idea + +# Archive +obsolete/ \ No newline at end of file diff --git a/.isort.cfg b/py/.isort.cfg similarity index 100% rename from .isort.cfg rename to py/.isort.cfg diff --git a/py/README.rst b/py/README.rst new file mode 100644 index 0000000..8c88e5b --- /dev/null +++ b/py/README.rst @@ -0,0 +1,364 @@ +persipubsub +=========== + +.. image:: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :target: https://api.travis-ci.com/Parquery/persipubsub.svg?branch=master + :alt: Build Status + +.. image:: https://coveralls.io/repos/github/Parquery/persipubsub/badge.svg?branch=master + :target: https://coveralls.io/github/Parquery/persipubsub?branch=master + :alt: Coverage + +.. image:: https://readthedocs.org/projects/persipubsub/badge/?version=latest + :target: https://persipubsub.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. image:: https://badge.fury.io/py/persipubsub.svg + :target: https://badge.fury.io/py/persipubsub + :alt: PyPI - version + +.. image:: https://img.shields.io/pypi/pyversions/persipubsub.svg + :alt: PyPI - Python Version + +.. image:: https://badges.frapsoft.com/os/mit/mit.png?v=103 + :target: https://opensource.org/licenses/mit-license.php + :alt: MIT License + +``persipubsub`` implements a persistent, thread-safe and process-safe queue for +inter-process communication, based on `lmdb `_. + +Primarily, we used `zeromq `_ for inter-process +communication with a slight improvement through `persizmq +`_. This still did not fulfill the level +of persistence we wanted. + +Our motivation was to replace our previous library with a one which is +similarly easy to setup and to use. Additionally, it should make it possible to +send `protobuf `_ messages +(bytes) thread-safely and process-safely from many publishers to many +subscribers. + +Besides basic publisher and subscriber classes the library offers control +methods for easy deployment from a config JSON file and maintenance in case +needed. + +Related projects +================ + +persist-queue +------------- + +* The library offers not all functionality expected from a queue. It has put + and get function which are basically only push and pop. Therefore ``front`` + functionality is missing. In consequence neither can the queue have multiple + subscribers nor can be guaranteed that no data is lost when a thread fails. +* All messages in queues are serialized by ``pickle`` which was for us the main + reason not to use this library. + +Kafka +----- + +* Hence we only need Inter Process Communication, the TCP ability of `Kafka + `_ is an unnecessary overhead. +* Integration of ``Kafka`` written in Scala and Java in our C++/Python/Go + codebase is challenging. +* Hard to setup and maintain ``Kafka``. +* broker system eventually a bottleneck. + +RabbitMQ +-------- + +* Hence we only need Inter Process Communication, the TCP ability of `RabbitMQ + `_ is an unnecessary overhead. +* broker system eventually a bottleneck. +* ``RabbitMQ`` is less scalable than ``Kafka``, but + is supported officially by more languages. + +zeromq persistence pattern +-------------------------- + +* `Titanic `_ is the only persistence + pattern of `zeromq `_ which is also a broker system. + This takes away the purpose and advantage of ``zeromq`` to be a + lightweight library which requires no broker. + +Usage +===== + +The usage of the library consists of two steps: deployment and runtime + +Environment +----------- + +To improve the accessibility of the library, an environment class lets you +create and initialize any ``persipubsub`` component which you need in +deployment or runtime step. + +.. warning:: + + Only one environment of each queue per process allowed and it's forbidden to + fork environment or any child components to multiple processes. + In that case, persipubsub is multi-threading and multi-process safe. + If multiple environments of the same queue are active on the same process, + or environment is forked to multiple processes the lock is broken and + correctness can't be guaranteed. + +Initialize environment +^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + +Deployment +---------- + +In the deployment stage the library sets up the queue structure with the control. + +Control +^^^^^^^ + +A control unit to initialize and maintain queues. + +.. note:: + + The high water marks are limits for the queue. The message is deleted in + case that it reaches the timeout. In the other case of an overflow one + of two strategies is used to prune half of the queue. The choice is between + prune_first, which deletes the oldest messages, and prune_last, which + deletes the latest messages. + +Initialize queue +"""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + import persipubsub.queue + + env = persipubsub.environment.new_environment(path="/home/user/new-queue/") + + # Initialize a queue with default values. + control = env.new_control() + # Or define all optional parameters of the queue. + hwm = persipubsub.queue._HighWaterMark() + strategy = persipubsub.queue._Strategy.prune_first + control = env.new_control(subscriber_ids=["sub1", "sub2"], + high_watermark=hwm, + strategy=strategy) + +Prune all dangling messages +""""""""""""""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-dangling-messages/") + control = env.new_control() + + control.prune_dangling_messages() + +Clear all messages +"""""""""""""""""" + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment( + path="/home/user/queue-with-subscribers-and-messages/") + control = env.new_control() + + control.clear_all_subscribers() + + +Runtime +------- + +During runtime only publisher and subscriber are needed. + +.. note:: + + Control can be optionally be used for pruning although the queues prune + itself on a regular basis when high water mark is reached. The high water + mark includes a timeout, maximum number of messages and the maximum bytes + size of the queue. + +Publisher +^^^^^^^^^ + +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the publisher can +be initialized as following. + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + pub = env.new_publisher() + +Send a message +"""""""""""""" + +.. code-block:: python + + msg = "Hello there!".encode('utf-8') + pub.send(msg=msg) + + # Subscribers have now a message in the queue. + +Send many messages at once +"""""""""""""""""""""""""" + +.. code-block:: python + + msgs = ["What's up?".encode('utf-8'), + "Do you like the README?".encode('utf-8')] + pub.send_many(msgs=msgs) + + # Both messages are now available for the subscribers. Note that the order + # of the messages are not necessarily kept. + +Subscriber +^^^^^^^^^^ + +Initialization +"""""""""""""" + +Assuming that all queues were initialized during deployment the subscriber can +be initialized as following. + +.. code-block:: python + + import persipubsub.environment + + env = persipubsub.environment.new_environment(path="/home/user/queue/") + + sub = env.new_subscriber(identifier="sub") + +Receive a message +""""""""""""""""" + +.. code-block:: python + + # One message in queue + with sub.receive() as msg: + # do something with the message + print(msg) # b'Hello there!' + + # This subscriber's queue is now empty + +Catch up with latest message +"""""""""""""""""""""""""""" + +Can be used in the case when a particular subscriber cares only about the very +last message. The messages are not popped for other subscribers. + +.. note:: + If you want to store only the latest message for all subscribers, then use + high water mark max_msgs_num = 1. + + +.. code-block:: python + + # Many outdated messages in queue + + with sub.receive_to_top() as msg: + # do something with the latest message + + # This subscriber's queue is now empty. + +Documentation +============= + +The documentation is available on `readthedocs +`_. + +Installation +============ + +* Install persipubsub with pip: + +.. code-block:: bash + + pip3 install persipubsub + +Development +=========== + +* Check out the repository. + +* In the repository root, create the virtual environment: + +.. code-block:: bash + + python3 -m venv venv3 + +* Activate the virtual environment: + +.. code-block:: bash + + source venv3/bin/activate + +* Install the development dependencies: + +.. code-block:: bash + + pip3 install -e .[dev] + +We use tox for testing and packaging the distribution. Assuming that the virtual +environment has been activated and the development dependencies have been +installed, run: + +.. code-block:: bash + + tox + +Pre-commit Checks +----------------- + +We provide a set of pre-commit checks that lint and check code for formatting. + +Namely, we use: + +* `yapf `_ to check the formatting. +* The style of the docstrings is checked with `pydocstyle `_. +* Static type analysis is performed with `mypy `_. +* `isort `_ to sort your imports for you. +* Various linter checks are done with `pylint `_. +* Doctests are executed using the Python `doctest module `_. +* `pyicontract-lint `_ lints contracts + in Python code defined with `icontract library `_. +* `twine `_ to check the README for invalid markup + which prevents it from rendering correctly on PyPI. + +Run the pre-commit checks locally from an activated virtual environment with +development dependencies: + +.. code-block:: bash + + ./precommit.py + +* The pre-commit script can also automatically format the code: + +.. code-block:: bash + + ./precommit.py --overwrite + +Versioning +========== + +We follow `Semantic Versioning `_. +The version X.Y.Z indicates: + +* X is the major version (backward-incompatible), +* Y is the minor version (backward-compatible), and +* Z is the patch version (backward-compatible bug fix). \ No newline at end of file diff --git a/docs/Makefile b/py/docs/Makefile similarity index 100% rename from docs/Makefile rename to py/docs/Makefile diff --git a/docs/source/changelog.rst b/py/docs/source/changelog.rst similarity index 100% rename from docs/source/changelog.rst rename to py/docs/source/changelog.rst diff --git a/docs/source/conf.py b/py/docs/source/conf.py similarity index 99% rename from docs/source/conf.py rename to py/docs/source/conf.py index ec249a8..b90d8c9 100644 --- a/docs/source/conf.py +++ b/py/docs/source/conf.py @@ -18,7 +18,6 @@ import persipubsub_meta - # -- Project information ----------------------------------------------------- project = persipubsub_meta.__title__ diff --git a/docs/source/index.rst b/py/docs/source/index.rst similarity index 100% rename from docs/source/index.rst rename to py/docs/source/index.rst diff --git a/docs/source/persipubsub/index.rst b/py/docs/source/persipubsub/index.rst similarity index 100% rename from docs/source/persipubsub/index.rst rename to py/docs/source/persipubsub/index.rst diff --git a/docs/source/persipubsub/persipubsub.ctl.rst b/py/docs/source/persipubsub/persipubsub.ctl.rst similarity index 100% rename from docs/source/persipubsub/persipubsub.ctl.rst rename to py/docs/source/persipubsub/persipubsub.ctl.rst diff --git a/docs/source/persipubsub/persipubsub.env.rst b/py/docs/source/persipubsub/persipubsub.env.rst similarity index 100% rename from docs/source/persipubsub/persipubsub.env.rst rename to py/docs/source/persipubsub/persipubsub.env.rst diff --git a/docs/source/persipubsub/persipubsub.publisher.rst b/py/docs/source/persipubsub/persipubsub.publisher.rst similarity index 100% rename from docs/source/persipubsub/persipubsub.publisher.rst rename to py/docs/source/persipubsub/persipubsub.publisher.rst diff --git a/docs/source/persipubsub/persipubsub.queue.rst b/py/docs/source/persipubsub/persipubsub.queue.rst similarity index 100% rename from docs/source/persipubsub/persipubsub.queue.rst rename to py/docs/source/persipubsub/persipubsub.queue.rst diff --git a/docs/source/persipubsub/persipubsub.rst b/py/docs/source/persipubsub/persipubsub.rst similarity index 100% rename from docs/source/persipubsub/persipubsub.rst rename to py/docs/source/persipubsub/persipubsub.rst diff --git a/docs/source/persipubsub/persipubsub.subscriber.rst b/py/docs/source/persipubsub/persipubsub.subscriber.rst similarity index 100% rename from docs/source/persipubsub/persipubsub.subscriber.rst rename to py/docs/source/persipubsub/persipubsub.subscriber.rst diff --git a/docs/source/readme.rst b/py/docs/source/readme.rst similarity index 100% rename from docs/source/readme.rst rename to py/docs/source/readme.rst diff --git a/mypy.ini b/py/mypy.ini similarity index 100% rename from mypy.ini rename to py/mypy.ini diff --git a/persipubsub/__init__.py b/py/persipubsub/__init__.py similarity index 100% rename from persipubsub/__init__.py rename to py/persipubsub/__init__.py diff --git a/persipubsub/control.py b/py/persipubsub/control.py similarity index 100% rename from persipubsub/control.py rename to py/persipubsub/control.py diff --git a/persipubsub/environment.py b/py/persipubsub/environment.py similarity index 100% rename from persipubsub/environment.py rename to py/persipubsub/environment.py diff --git a/persipubsub/publisher.py b/py/persipubsub/publisher.py similarity index 100% rename from persipubsub/publisher.py rename to py/persipubsub/publisher.py diff --git a/persipubsub/queue.py b/py/persipubsub/queue.py similarity index 100% rename from persipubsub/queue.py rename to py/persipubsub/queue.py diff --git a/persipubsub/subscriber.py b/py/persipubsub/subscriber.py similarity index 100% rename from persipubsub/subscriber.py rename to py/persipubsub/subscriber.py diff --git a/persipubsub_meta.py b/py/persipubsub_meta.py similarity index 100% rename from persipubsub_meta.py rename to py/persipubsub_meta.py diff --git a/precommit.py b/py/precommit.py similarity index 100% rename from precommit.py rename to py/precommit.py diff --git a/pylint.rc b/py/pylint.rc similarity index 100% rename from pylint.rc rename to py/pylint.rc diff --git a/requirements-doc.txt b/py/requirements-doc.txt similarity index 100% rename from requirements-doc.txt rename to py/requirements-doc.txt diff --git a/setup.py b/py/setup.py similarity index 98% rename from setup.py rename to py/setup.py index 137bc5f..6284d85 100644 --- a/setup.py +++ b/py/setup.py @@ -64,6 +64,6 @@ }, py_modules=['persipubsub', 'persipubsub_meta'], package_data={ - "persipubsub": ["py.typed"], + "persipubsub": ["typed"], '.': ['LICENSE', 'README.rst'] }) diff --git a/style.yapf b/py/style.yapf similarity index 100% rename from style.yapf rename to py/style.yapf diff --git a/tests/__init__.py b/py/tests/__init__.py similarity index 96% rename from tests/__init__.py rename to py/tests/__init__.py index f30c5e8..9744cf7 100644 --- a/tests/__init__.py +++ b/py/tests/__init__.py @@ -1,10 +1,7 @@ #!/usr/bin/env python3 """Test persipubsub.""" -import json -import pathlib import unittest -from typing import Any, Dict import lmdb import temppathlib diff --git a/tests/component_publisher.py b/py/tests/component_publisher.py similarity index 96% rename from tests/component_publisher.py rename to py/tests/component_publisher.py index 5abef79..43b27b7 100644 --- a/tests/component_publisher.py +++ b/py/tests/component_publisher.py @@ -5,7 +5,6 @@ import time import persipubsub.environment -import persipubsub.publisher # pylint: disable=missing-docstring diff --git a/tests/component_subscriber.py b/py/tests/component_subscriber.py similarity index 100% rename from tests/component_subscriber.py rename to py/tests/component_subscriber.py diff --git a/tests/test_control.py b/py/tests/test_control.py similarity index 100% rename from tests/test_control.py rename to py/tests/test_control.py diff --git a/tests/test_environment.py b/py/tests/test_environment.py similarity index 100% rename from tests/test_environment.py rename to py/tests/test_environment.py diff --git a/tests/test_live.py b/py/tests/test_live.py similarity index 98% rename from tests/test_live.py rename to py/tests/test_live.py index e076b8f..23462c3 100644 --- a/tests/test_live.py +++ b/py/tests/test_live.py @@ -13,6 +13,7 @@ import persipubsub.publisher import persipubsub.queue import persipubsub.subscriber +import tests import tests.component_publisher import tests.component_subscriber @@ -54,11 +55,13 @@ def subscriber_receive_second(sub: persipubsub.subscriber.Subscriber) -> None: result.write_text('fail') -def receive(sub: persipubsub.subscriber.Subscriber, - num_msg: int, - timeout: int = 2, - sleep_time: float = 0, - method_timeout: int = 60) -> None: +def receive( + sub: persipubsub.subscriber.Subscriber, + num_msg: int, + timeout: int = 2, + # TODO: remove + sleep_time: float = 0, + method_timeout: int = 60) -> None: received_msg = 0 start = time.time() @@ -71,7 +74,7 @@ def receive(sub: persipubsub.subscriber.Subscriber, with sub.receive(timeout=timeout) as msg: if msg is not None: received_msg += 1 - time.sleep(sleep_time) + time.sleep(sleep_time) assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access assert isinstance(sub.queue.env, lmdb.Environment) diff --git a/tests/test_publisher.py b/py/tests/test_publisher.py similarity index 99% rename from tests/test_publisher.py rename to py/tests/test_publisher.py index 9226ca9..19ee2a6 100644 --- a/tests/test_publisher.py +++ b/py/tests/test_publisher.py @@ -9,7 +9,6 @@ import persipubsub.control import persipubsub.environment -import persipubsub.publisher import persipubsub.queue import tests diff --git a/tests/test_queue.py b/py/tests/test_queue.py similarity index 100% rename from tests/test_queue.py rename to py/tests/test_queue.py diff --git a/tests/test_subscriber.py b/py/tests/test_subscriber.py similarity index 100% rename from tests/test_subscriber.py rename to py/tests/test_subscriber.py diff --git a/tox.ini b/py/tox.ini similarity index 100% rename from tox.ini rename to py/tox.ini From 1e01e2a1442ede79946b879edeacbe2d72a521ce Mon Sep 17 00:00:00 2001 From: slimeth Date: Tue, 19 Feb 2019 10:49:29 +0100 Subject: [PATCH 33/35] added cpp to persipubsub --- cpp/.gitignore | 22 + cpp/CMakeLists.txt | 11 + cpp/include/filesystem.h | 153 +++ cpp/include/lmdb++.h | 1913 +++++++++++++++++++++++++++++++ cpp/include/prettyprint.hpp | 446 +++++++ cpp/include/scope_exit.hpp | 39 + cpp/include/tinyformat.h | 1003 ++++++++++++++++ cpp/include/unique_resource.hpp | 78 ++ cpp/src/CMakeLists.txt | 6 + cpp/src/control.h | 135 +++ cpp/src/environment.h | 75 ++ cpp/src/filesystem.cpp | 317 +++++ cpp/src/library.cpp | 19 + cpp/src/library.h | 21 + cpp/src/publisher.h | 52 + cpp/src/queue.cpp | 404 +++++++ cpp/src/queue.h | 172 +++ cpp/src/subscriber.h | 70 ++ cpp/test/CMakeLists.txt | 13 + cpp/test/test_queue.cpp | 106 ++ 20 files changed, 5055 insertions(+) create mode 100644 cpp/.gitignore create mode 100644 cpp/CMakeLists.txt create mode 100755 cpp/include/filesystem.h create mode 100644 cpp/include/lmdb++.h create mode 100644 cpp/include/prettyprint.hpp create mode 100644 cpp/include/scope_exit.hpp create mode 100644 cpp/include/tinyformat.h create mode 100644 cpp/include/unique_resource.hpp create mode 100644 cpp/src/CMakeLists.txt create mode 100644 cpp/src/control.h create mode 100644 cpp/src/environment.h create mode 100644 cpp/src/filesystem.cpp create mode 100644 cpp/src/library.cpp create mode 100644 cpp/src/library.h create mode 100644 cpp/src/publisher.h create mode 100644 cpp/src/queue.cpp create mode 100644 cpp/src/queue.h create mode 100644 cpp/src/subscriber.h create mode 100644 cpp/test/CMakeLists.txt create mode 100644 cpp/test/test_queue.cpp diff --git a/cpp/.gitignore b/cpp/.gitignore new file mode 100644 index 0000000..05d2a2a --- /dev/null +++ b/cpp/.gitignore @@ -0,0 +1,22 @@ +# Distribution / packaging +cmake-build-debug/ +TODO + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +venv3/ +venv-doc/ + +# IDE +.idea + +# Archive +obsolete/ + +# Project specific \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..17aab6e --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.12) +project(persipubsub) + +set(CMAKE_CXX_STANDARD 17) + +include_directories(include) + +enable_testing() +add_subdirectory(src) +add_subdirectory(test) +add_test(NAME myTest COMMAND Test) \ No newline at end of file diff --git a/cpp/include/filesystem.h b/cpp/include/filesystem.h new file mode 100755 index 0000000..039266c --- /dev/null +++ b/cpp/include/filesystem.h @@ -0,0 +1,153 @@ +// Copyright (c) 2016 Parquery AG. All rights reserved. +// Created by mristin on 10/12/16. + +#pragma once + +#include +#include + +#include +#include +#include + +#include +#include + +namespace pqry { +namespace filesystem { + +/** + * Simulates Unix ls -lt + * @param directory to list + * @return absolute paths sorted by modification time + */ +std::vector ls_lt(const boost::filesystem::path& directory); + +/** + * @param directory to be listed + * @return list of absolute paths to files contained in the directory + */ +std::vector listdir(const boost::filesystem::path& directory); + +/** + * @param path to the file + * @return modified time in nanoseconds since epoch + */ +int64_t modified_time(const boost::filesystem::path& path); + +/** + * simulates `mkdtemp` command + * @return path to the temporary directory + */ +boost::filesystem::path mkdtemp(); + +/** + * Reads a file to the string + * @param path to the file + * @return content of the file + */ +std::string read(const boost::filesystem::path& path); + +/** + * Writes the whole text to the file + * @param path to the file + * @param text to be written + */ +void write(const boost::filesystem::path& path, const std::string& text); + +/** + * Writes the `bytes` to the file + * @param path to write to + * @param bytes that are to be written + * @param size of the `bytes` + */ +void write(const boost::filesystem::path& path, const char* bytes, int size); + +/** + * Copies a source directory with all contents into a new target directory recursively + * @param source directory that gets copied + * @param dest directory where source gets copied to. Will be created if non-existent + */ +void copy_directory(const boost::filesystem::path& source, const boost::filesystem::path& dest); + +/** + * Unzips an archive. + * @param path to the archive + * @param dest_dir destination where the archive will be unzipped. Expects it to exist + */ +void unzip_archive(const boost::filesystem::path& path, const boost::filesystem::path& dest_dir); + +/** + * Uses POSIX flock() for named mutexes. Use an instance of this class to prevent multiple processes running at the + * same time. + * + * Mind that the instance keeps a file descriptor open during its lifetime. + * The implementation is crash-safe, i.e. if the program crashes, the system automatically releases the lock. + * Unlike boost::interprocess::file_lock, we do not have to create the file manually first. + * boost::interprocess:named_mutex did not work on Linux with boost 1.62 (marko tested it manually). + */ +class LockFileGuard { + public: + LockFileGuard() : locked_(false) {} + + /** + * Creates the lock file and writes the given PID to it + * @param lock_file to use for locking + * @param pid of the process + * @return true if the file could be successfully locked. + */ + bool lock(const boost::filesystem::path& lock_file, int pid); + + void unlock(); + + virtual ~LockFileGuard(); + + private: + bool locked_; + boost::filesystem::path lock_file_; + int fid_; +}; + +/** + * Wait for file to exist. Panics if timeout exceeded (0 means no timeout). + * @param path to the file + * @param timeout in seconds + */ +void wait_for_file(const boost::filesystem::path& path, unsigned int timeout); + +/** + * Expands path using wordexp + * @param path to be expanded + * @return expanded path + */ +boost::filesystem::path expand_path(const boost::filesystem::path& path); + +/** + * Temporary scope file, created from a path with a random suffix and ".tmp". + * If the file exists at the destruction, it will be deleted. + * All errors will be ignored in the destructor. + */ +class NamedTempfile { + public: + explicit NamedTempfile(const boost::filesystem::path& path); + + /** + * Deletes the temporary file if it was not renamed before. + */ + ~NamedTempfile(); + + const boost::filesystem::path& path() const { return tmp_pth_; } + + /** + * Renames the temporary file to the path passed in at the constructor. + */ + void rename(); + + private: + const boost::filesystem::path pth_; + const boost::filesystem::path tmp_pth_; + bool renamed_; +}; + +} // namespace filesystem +} // namespace pqry diff --git a/cpp/include/lmdb++.h b/cpp/include/lmdb++.h new file mode 100644 index 0000000..ab75f8c --- /dev/null +++ b/cpp/include/lmdb++.h @@ -0,0 +1,1913 @@ +/* This is free and unencumbered software released into the public domain. */ + +#ifndef LMDBXX_H +#define LMDBXX_H + +/** + * - C++11 wrapper for LMDB. + * + * @author Arto Bendiken + * @see https://sourceforge.net/projects/lmdbxx/ + */ + +#ifndef __cplusplus +#error " requires a C++ compiler" +#endif + +#if __cplusplus < 201103L +#if !defined(_MSC_VER) || _MSC_VER < 1900 +#error " requires a C++11 compiler (CXXFLAGS='-std=c++11')" +#endif // _MSC_VER check +#endif + +//////////////////////////////////////////////////////////////////////////////// + +#include /* for MDB_*, mdb_*() */ + +#ifdef LMDBXX_DEBUG +#include /* for assert() */ +#endif +#include /* for std::size_t */ +#include /* for std::snprintf() */ +#include /* for std::strlen() */ +#include /* for std::runtime_error */ +#include /* for std::string */ +#include /* for std::is_pod<> */ + +namespace lmdb { + using mode = mdb_mode_t; +} + +//////////////////////////////////////////////////////////////////////////////// +/* Error Handling */ + +namespace lmdb { + class error; + class logic_error; + class fatal_error; + class runtime_error; + class key_exist_error; + class not_found_error; + class corrupted_error; + class panic_error; + class version_mismatch_error; + class map_full_error; + class bad_dbi_error; +} + +/** + * Base class for LMDB exception conditions. + * + * @see http://symas.com/mdb/doc/group__errors.html + */ +class lmdb::error : public std::runtime_error { +protected: + const int _code; + +public: + /** + * Throws an error based on the given LMDB return code. + */ + [[noreturn]] static inline void raise(const char* origin, int rc); + + /** + * Constructor. + */ + error(const char* const origin, + const int rc) noexcept + : runtime_error{origin}, + _code{rc} {} + + /** + * Returns the underlying LMDB error code. + */ + int code() const noexcept { + return _code; + } + + /** + * Returns the origin of the LMDB error. + */ + const char* origin() const noexcept { + return runtime_error::what(); + } + + /** + * Returns the underlying LMDB error code. + */ + virtual const char* what() const noexcept { + static thread_local char buffer[1024]; + std::snprintf(buffer, sizeof(buffer), + "%s: %s", origin(), ::mdb_strerror(code())); + return buffer; + } +}; + +/** + * Base class for logic error conditions. + */ +class lmdb::logic_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Base class for fatal error conditions. + */ +class lmdb::fatal_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Base class for runtime error conditions. + */ +class lmdb::runtime_error : public lmdb::error { +public: + using error::error; +}; + +/** + * Exception class for `MDB_KEYEXIST` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga05dc5bbcc7da81a7345bd8676e8e0e3b + */ +class lmdb::key_exist_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_NOTFOUND` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gabeb52e4c4be21b329e31c4add1b71926 + */ +class lmdb::not_found_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_CORRUPTED` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gaf8148bf1b85f58e264e57194bafb03ef + */ +class lmdb::corrupted_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_PANIC` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#gae37b9aedcb3767faba3de8c1cf6d3473 + */ +class lmdb::panic_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_VERSION_MISMATCH` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga909b2db047fa90fb0d37a78f86a6f99b + */ +class lmdb::version_mismatch_error final : public lmdb::fatal_error { +public: + using fatal_error::fatal_error; +}; + +/** + * Exception class for `MDB_MAP_FULL` errors. + * + * @see http://symas.com/mdb/doc/group__errors.html#ga0a83370402a060c9175100d4bbfb9f25 + */ +class lmdb::map_full_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +/** + * Exception class for `MDB_BAD_DBI` errors. + * + * @since 0.9.14 (2014/09/20) + * @see http://symas.com/mdb/doc/group__errors.html#gab4c82e050391b60a18a5df08d22a7083 + */ +class lmdb::bad_dbi_error final : public lmdb::runtime_error { +public: + using runtime_error::runtime_error; +}; + +inline void +lmdb::error::raise(const char* const origin, + const int rc) { + switch (rc) { + case MDB_KEYEXIST: throw key_exist_error{origin, rc}; + case MDB_NOTFOUND: throw not_found_error{origin, rc}; + case MDB_CORRUPTED: throw corrupted_error{origin, rc}; + case MDB_PANIC: throw panic_error{origin, rc}; + case MDB_VERSION_MISMATCH: throw version_mismatch_error{origin, rc}; + case MDB_MAP_FULL: throw map_full_error{origin, rc}; +#ifdef MDB_BAD_DBI + case MDB_BAD_DBI: throw bad_dbi_error{origin, rc}; +#endif + default: throw lmdb::runtime_error{origin, rc}; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Metadata */ + +namespace lmdb { + // TODO: mdb_version() + // TODO: mdb_strerror() +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Environment */ + +namespace lmdb { + static inline void env_create(MDB_env** env); + static inline void env_open(MDB_env* env, + const char* path, unsigned int flags, mode mode); +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + static inline void env_copy(MDB_env* env, const char* path, unsigned int flags); + static inline void env_copy_fd(MDB_env* env, mdb_filehandle_t fd, unsigned int flags); +#else + static inline void env_copy(MDB_env* env, const char* path); + static inline void env_copy_fd(MDB_env* env, mdb_filehandle_t fd); +#endif + static inline void env_stat(MDB_env* env, MDB_stat* stat); + static inline void env_info(MDB_env* env, MDB_envinfo* stat); + static inline void env_sync(MDB_env* env, bool force); + static inline void env_close(MDB_env* env) noexcept; + static inline void env_set_flags(MDB_env* env, unsigned int flags, bool onoff); + static inline void env_get_flags(MDB_env* env, unsigned int* flags); + static inline void env_get_path(MDB_env* env, const char** path); + static inline void env_get_fd(MDB_env* env, mdb_filehandle_t* fd); + static inline void env_set_mapsize(MDB_env* env, std::size_t size); + static inline void env_set_max_readers(MDB_env* env, unsigned int count); + static inline void env_get_max_readers(MDB_env* env, unsigned int* count); + static inline void env_set_max_dbs(MDB_env* env, MDB_dbi count); + static inline unsigned int env_get_max_keysize(MDB_env* env); +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) + static inline void env_set_userctx(MDB_env* env, void* ctx); + static inline void* env_get_userctx(MDB_env* env); +#endif + // TODO: mdb_env_set_assert() + // TODO: mdb_reader_list() + // TODO: mdb_reader_check() +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaad6be3d8dcd4ea01f8df436f41d158d4 + */ +static inline void +lmdb::env_create(MDB_env** env) { + const int rc = ::mdb_env_create(env); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_create", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga32a193c6bf4d7d5c5d579e71f22e9340 + */ +static inline void +lmdb::env_open(MDB_env* const env, + const char* const path, + const unsigned int flags, + const mode mode) { + const int rc = ::mdb_env_open(env, path, flags, mode); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_open", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga3bf50d7793b36aaddf6b481a44e24244 + * @see http://symas.com/mdb/doc/group__mdb.html#ga5d51d6130325f7353db0955dbedbc378 + */ +static inline void +lmdb::env_copy(MDB_env* const env, +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + const char* const path, + const unsigned int flags = 0) { + const int rc = ::mdb_env_copy2(env, path, flags); +#else + const char* const path) { + const int rc = ::mdb_env_copy(env, path); +#endif + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_copy2", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga5040d0de1f14000fa01fc0b522ff1f86 + * @see http://symas.com/mdb/doc/group__mdb.html#ga470b0bcc64ac417de5de5930f20b1a28 + */ +static inline void +lmdb::env_copy_fd(MDB_env* const env, +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 14) + const mdb_filehandle_t fd, + const unsigned int flags = 0) { + const int rc = ::mdb_env_copyfd2(env, fd, flags); +#else + const mdb_filehandle_t fd) { + const int rc = ::mdb_env_copyfd(env, fd); +#endif + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_copyfd2", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaf881dca452050efbd434cd16e4bae255 + */ +static inline void +lmdb::env_stat(MDB_env* const env, + MDB_stat* const stat) { + const int rc = ::mdb_env_stat(env, stat); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_stat", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga18769362c7e7d6cf91889a028a5c5947 + */ +static inline void +lmdb::env_info(MDB_env* const env, + MDB_envinfo* const stat) { + const int rc = ::mdb_env_info(env, stat); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_info", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga85e61f05aa68b520cc6c3b981dba5037 + */ +static inline void +lmdb::env_sync(MDB_env* const env, + const bool force = true) { + const int rc = ::mdb_env_sync(env, force); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_sync", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga4366c43ada8874588b6a62fbda2d1e95 + */ +static inline void +lmdb::env_close(MDB_env* const env) noexcept { + ::mdb_env_close(env); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga83f66cf02bfd42119451e9468dc58445 + */ +static inline void +lmdb::env_set_flags(MDB_env* const env, + const unsigned int flags, + const bool onoff = true) { + const int rc = ::mdb_env_set_flags(env, flags, onoff ? 1 : 0); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_flags", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga2733aefc6f50beb49dd0c6eb19b067d9 + */ +static inline void +lmdb::env_get_flags(MDB_env* const env, + unsigned int* const flags) { + const int rc = ::mdb_env_get_flags(env, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_flags", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac699fdd8c4f8013577cb933fb6a757fe + */ +static inline void +lmdb::env_get_path(MDB_env* const env, + const char** path) { + const int rc = ::mdb_env_get_path(env, path); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_path", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaf1570e7c0e5a5d860fef1032cec7d5f2 + */ +static inline void +lmdb::env_get_fd(MDB_env* const env, + mdb_filehandle_t* const fd) { + const int rc = ::mdb_env_get_fd(env, fd); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_fd", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5 + */ +static inline void +lmdb::env_set_mapsize(MDB_env* const env, + const std::size_t size) { + const int rc = ::mdb_env_set_mapsize(env, size); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_mapsize", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gae687966c24b790630be2a41573fe40e2 + */ +static inline void +lmdb::env_set_max_readers(MDB_env* const env, + const unsigned int count) { + const int rc = ::mdb_env_set_maxreaders(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_maxreaders", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga70e143cf11760d869f754c9c9956e6cc + */ +static inline void +lmdb::env_get_max_readers(MDB_env* const env, + unsigned int* const count) { + const int rc = ::mdb_env_get_maxreaders(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_get_maxreaders", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gaa2fc2f1f37cb1115e733b62cab2fcdbc + */ +static inline void +lmdb::env_set_max_dbs(MDB_env* const env, + const MDB_dbi count) { + const int rc = ::mdb_env_set_maxdbs(env, count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_maxdbs", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94 + */ +static inline unsigned int +lmdb::env_get_max_keysize(MDB_env* const env) { + const int rc = ::mdb_env_get_maxkeysize(env); +#ifdef LMDBXX_DEBUG + assert(rc >= 0); +#endif + return static_cast(rc); +} + +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) +/** + * @throws lmdb::error on failure + * @since 0.9.11 (2014/01/15) + * @see http://symas.com/mdb/doc/group__mdb.html#gaf2fe09eb9c96eeb915a76bf713eecc46 + */ +static inline void +lmdb::env_set_userctx(MDB_env* const env, + void* const ctx) { + const int rc = ::mdb_env_set_userctx(env, ctx); + if (rc != MDB_SUCCESS) { + error::raise("mdb_env_set_userctx", rc); + } +} +#endif + +#if MDB_VERSION_FULL >= MDB_VERINT(0, 9, 11) +/** + * @since 0.9.11 (2014/01/15) + * @see http://symas.com/mdb/doc/group__mdb.html#ga45df6a4fb150cda2316b5ae224ba52f1 + */ +static inline void* +lmdb::env_get_userctx(MDB_env* const env) { + return ::mdb_env_get_userctx(env); +} +#endif + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Transactions */ + +namespace lmdb { + static inline void txn_begin( + MDB_env* env, MDB_txn* parent, unsigned int flags, MDB_txn** txn); + static inline MDB_env* txn_env(MDB_txn* txn) noexcept; +#ifdef LMDBXX_TXN_ID + static inline std::size_t txn_id(MDB_txn* txn) noexcept; +#endif + static inline void txn_commit(MDB_txn* txn); + static inline void txn_abort(MDB_txn* txn) noexcept; + static inline void txn_reset(MDB_txn* txn) noexcept; + static inline void txn_renew(MDB_txn* txn); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gad7ea55da06b77513609efebd44b26920 + */ +static inline void +lmdb::txn_begin(MDB_env* const env, + MDB_txn* const parent, + const unsigned int flags, + MDB_txn** txn) { + const int rc = ::mdb_txn_begin(env, parent, flags, txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_begin", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gaeb17735b8aaa2938a78a45cab85c06a0 + */ +static inline MDB_env* +lmdb::txn_env(MDB_txn* const txn) noexcept { + return ::mdb_txn_env(txn); +} + +#ifdef LMDBXX_TXN_ID +/** + * @note Only available in HEAD, not yet in any 0.9.x release (as of 0.9.16). + */ +static inline std::size_t +lmdb::txn_id(MDB_txn* const txn) noexcept { + return ::mdb_txn_id(txn); +} +#endif + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga846fbd6f46105617ac9f4d76476f6597 + */ +static inline void +lmdb::txn_commit(MDB_txn* const txn) { + const int rc = ::mdb_txn_commit(txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_commit", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga73a5938ae4c3239ee11efa07eb22b882 + */ +static inline void +lmdb::txn_abort(MDB_txn* const txn) noexcept { + ::mdb_txn_abort(txn); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga02b06706f8a66249769503c4e88c56cd + */ +static inline void +lmdb::txn_reset(MDB_txn* const txn) noexcept { + ::mdb_txn_reset(txn); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga6c6f917959517ede1c504cf7c720ce6d + */ +static inline void +lmdb::txn_renew(MDB_txn* const txn) { + const int rc = ::mdb_txn_renew(txn); + if (rc != MDB_SUCCESS) { + error::raise("mdb_txn_renew", rc); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Databases */ + +namespace lmdb { + static inline void dbi_open( + MDB_txn* txn, const char* name, unsigned int flags, MDB_dbi* dbi); + static inline void dbi_stat(MDB_txn* txn, MDB_dbi dbi, MDB_stat* stat); + static inline void dbi_flags(MDB_txn* txn, MDB_dbi dbi, unsigned int* flags); + static inline void dbi_close(MDB_env* env, MDB_dbi dbi) noexcept; + static inline void dbi_drop(MDB_txn* txn, MDB_dbi dbi, bool del); + static inline void dbi_set_compare(MDB_txn* txn, MDB_dbi dbi, MDB_cmp_func* cmp); + static inline void dbi_set_dupsort(MDB_txn* txn, MDB_dbi dbi, MDB_cmp_func* cmp); + static inline void dbi_set_relfunc(MDB_txn* txn, MDB_dbi dbi, MDB_rel_func* rel); + static inline void dbi_set_relctx(MDB_txn* txn, MDB_dbi dbi, void* ctx); + static inline bool dbi_get(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, MDB_val* data); + static inline bool dbi_put(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, MDB_val* data, unsigned int flags); + static inline bool dbi_del(MDB_txn* txn, MDB_dbi dbi, const MDB_val* key, const MDB_val* data); + // TODO: mdb_cmp() + // TODO: mdb_dcmp() +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac08cad5b096925642ca359a6d6f0562a + */ +static inline void +lmdb::dbi_open(MDB_txn* const txn, + const char* const name, + const unsigned int flags, + MDB_dbi* const dbi) { + const int rc = ::mdb_dbi_open(txn, name, flags, dbi); + if (rc != MDB_SUCCESS) { + error::raise("mdb_dbi_open", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gae6c1069febe94299769dbdd032fadef6 + */ +static inline void +lmdb::dbi_stat(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_stat* const result) { + const int rc = ::mdb_stat(txn, dbi, result); + if (rc != MDB_SUCCESS) { + error::raise("mdb_stat", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga95ba4cb721035478a8705e57b91ae4d4 + */ +static inline void +lmdb::dbi_flags(MDB_txn* const txn, + const MDB_dbi dbi, + unsigned int* const flags) { + const int rc = ::mdb_dbi_flags(txn, dbi, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_dbi_flags", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga52dd98d0c542378370cd6b712ff961b5 + */ +static inline void +lmdb::dbi_close(MDB_env* const env, + const MDB_dbi dbi) noexcept { + ::mdb_dbi_close(env, dbi); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gab966fab3840fc54a6571dfb32b00f2db + */ +static inline void +lmdb::dbi_drop(MDB_txn* const txn, + const MDB_dbi dbi, + const bool del = false) { + const int rc = ::mdb_drop(txn, dbi, del ? 1 : 0); + if (rc != MDB_SUCCESS) { + error::raise("mdb_drop", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga68e47ffcf72eceec553c72b1784ee0fe + */ +static inline void +lmdb::dbi_set_compare(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cmp_func* const cmp = nullptr) { + const int rc = ::mdb_set_compare(txn, dbi, cmp); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_compare", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gacef4ec3dab0bbd9bc978b73c19c879ae + */ +static inline void +lmdb::dbi_set_dupsort(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cmp_func* const cmp = nullptr) { + const int rc = ::mdb_set_dupsort(txn, dbi, cmp); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_dupsort", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga697d82c7afe79f142207ad5adcdebfeb + */ +static inline void +lmdb::dbi_set_relfunc(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_rel_func* const rel) { + const int rc = ::mdb_set_relfunc(txn, dbi, rel); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_relfunc", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga7c34246308cee01724a1839a8f5cc594 + */ +static inline void +lmdb::dbi_set_relctx(MDB_txn* const txn, + const MDB_dbi dbi, + void* const ctx) { + const int rc = ::mdb_set_relctx(txn, dbi, ctx); + if (rc != MDB_SUCCESS) { + error::raise("mdb_set_relctx", rc); + } +} + +/** + * @retval true if the key/value pair was retrieved + * @retval false if the key wasn't found + * @see http://symas.com/mdb/doc/group__mdb.html#ga8bf10cd91d3f3a83a34d04ce6b07992d + */ +static inline bool +lmdb::dbi_get(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + MDB_val* const data) { + const int rc = ::mdb_get(txn, dbi, const_cast(key), data); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_get", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @retval true if the key/value pair was inserted + * @retval false if the key already existed + * @see http://symas.com/mdb/doc/group__mdb.html#ga4fa8573d9236d54687c61827ebf8cac0 + */ +static inline bool +lmdb::dbi_put(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + MDB_val* const data, + const unsigned int flags = 0) { + const int rc = ::mdb_put(txn, dbi, const_cast(key), data, flags); + if (rc != MDB_SUCCESS && rc != MDB_KEYEXIST) { + error::raise("mdb_put", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @retval true if the key/value pair was removed + * @retval false if the key wasn't found + * @see http://symas.com/mdb/doc/group__mdb.html#gab8182f9360ea69ac0afd4a4eaab1ddb0 + */ +static inline bool +lmdb::dbi_del(MDB_txn* const txn, + const MDB_dbi dbi, + const MDB_val* const key, + const MDB_val* const data = nullptr) { + const int rc = ::mdb_del(txn, dbi, const_cast(key), const_cast(data)); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_del", rc); + } + return (rc == MDB_SUCCESS); +} + +//////////////////////////////////////////////////////////////////////////////// +/* Procedural Interface: Cursors */ + +namespace lmdb { + static inline void cursor_open(MDB_txn* txn, MDB_dbi dbi, MDB_cursor** cursor); + static inline void cursor_close(MDB_cursor* cursor) noexcept; + static inline void cursor_renew(MDB_txn* txn, MDB_cursor* cursor); + static inline MDB_txn* cursor_txn(MDB_cursor* cursor) noexcept; + static inline MDB_dbi cursor_dbi(MDB_cursor* cursor) noexcept; + static inline bool cursor_get(MDB_cursor* cursor, MDB_val* key, MDB_val* data, MDB_cursor_op op); + static inline void cursor_put(MDB_cursor* cursor, MDB_val* key, MDB_val* data, unsigned int flags); + static inline void cursor_del(MDB_cursor* cursor, unsigned int flags); + static inline void cursor_count(MDB_cursor* cursor, std::size_t& count); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga9ff5d7bd42557fd5ee235dc1d62613aa + */ +static inline void +lmdb::cursor_open(MDB_txn* const txn, + const MDB_dbi dbi, + MDB_cursor** const cursor) { + const int rc = ::mdb_cursor_open(txn, dbi, cursor); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_open", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#gad685f5d73c052715c7bd859cc4c05188 + */ +static inline void +lmdb::cursor_close(MDB_cursor* const cursor) noexcept { + ::mdb_cursor_close(cursor); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#gac8b57befb68793070c85ea813df481af + */ +static inline void +lmdb::cursor_renew(MDB_txn* const txn, + MDB_cursor* const cursor) { + const int rc = ::mdb_cursor_renew(txn, cursor); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_renew", rc); + } +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga7bf0d458f7f36b5232fcb368ebda79e0 + */ +static inline MDB_txn* +lmdb::cursor_txn(MDB_cursor* const cursor) noexcept { + return ::mdb_cursor_txn(cursor); +} + +/** + * @see http://symas.com/mdb/doc/group__mdb.html#ga2f7092cf70ee816fb3d2c3267a732372 + */ +static inline MDB_dbi +lmdb::cursor_dbi(MDB_cursor* const cursor) noexcept { + return ::mdb_cursor_dbi(cursor); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga48df35fb102536b32dfbb801a47b4cb0 + */ +static inline bool +lmdb::cursor_get(MDB_cursor* const cursor, + MDB_val* const key, + MDB_val* const data, + const MDB_cursor_op op) { + const int rc = ::mdb_cursor_get(cursor, key, data, op); + if (rc != MDB_SUCCESS && rc != MDB_NOTFOUND) { + error::raise("mdb_cursor_get", rc); + } + return (rc == MDB_SUCCESS); +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga1f83ccb40011837ff37cc32be01ad91e + */ +static inline void +lmdb::cursor_put(MDB_cursor* const cursor, + MDB_val* const key, + MDB_val* const data, + const unsigned int flags = 0) { + const int rc = ::mdb_cursor_put(cursor, key, data, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_put", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga26a52d3efcfd72e5bf6bd6960bf75f95 + */ +static inline void +lmdb::cursor_del(MDB_cursor* const cursor, + const unsigned int flags = 0) { + const int rc = ::mdb_cursor_del(cursor, flags); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_del", rc); + } +} + +/** + * @throws lmdb::error on failure + * @see http://symas.com/mdb/doc/group__mdb.html#ga4041fd1e1862c6b7d5f10590b86ffbe2 + */ +static inline void +lmdb::cursor_count(MDB_cursor* const cursor, + std::size_t& count) { + const int rc = ::mdb_cursor_count(cursor, &count); + if (rc != MDB_SUCCESS) { + error::raise("mdb_cursor_count", rc); + } +} + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Values */ + +namespace lmdb { + class val; +} + +/** + * Wrapper class for `MDB_val` structures. + * + * @note Instances of this class are movable and copyable both. + * @see http://symas.com/mdb/doc/group__mdb.html#structMDB__val + */ +class lmdb::val { +protected: + MDB_val _val; + +public: + /** + * Default constructor. + */ + val() noexcept = default; + + /** + * Constructor. + */ + val(const std::string& data) noexcept + : val{data.data(), data.size()} {} + + /** + * Constructor. + */ + val(const char* const data) noexcept + : val{data, std::strlen(data)} {} + + /** + * Constructor. + */ + val(const void* const data, + const std::size_t size) noexcept + : _val{size, const_cast(data)} {} + + /** + * Move constructor. + */ + val(val&& other) noexcept = default; + + /** + * Move assignment operator. + */ + val& operator=(val&& other) noexcept = default; + + /** + * Destructor. + */ + ~val() noexcept = default; + + /** + * Returns an `MDB_val*` pointer. + */ + operator MDB_val*() noexcept { + return &_val; + } + + /** + * Returns an `MDB_val*` pointer. + */ + operator const MDB_val*() const noexcept { + return &_val; + } + + /** + * Determines whether this value is empty. + */ + bool empty() const noexcept { + return size() == 0; + } + + /** + * Returns the size of the data. + */ + std::size_t size() const noexcept { + return _val.mv_size; + } + + /** + * Returns a pointer to the data. + */ + template + T* data() noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + template + const T* data() const noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + char* data() noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Returns a pointer to the data. + */ + const char* data() const noexcept { + return reinterpret_cast(_val.mv_data); + } + + /** + * Assigns the value. + */ + template + val& assign(const T* const data, + const std::size_t size) noexcept { + _val.mv_size = size; + _val.mv_data = const_cast(reinterpret_cast(data)); + return *this; + } + + /** + * Assigns the value. + */ + val& assign(const char* const data) noexcept { + return assign(data, std::strlen(data)); + } + + /** + * Assigns the value. + */ + val& assign(const std::string& data) noexcept { + return assign(data.data(), data.size()); + } +}; + +#if !(defined(__COVERITY__) || defined(_MSC_VER)) +static_assert(std::is_pod::value, "lmdb::val must be a POD type"); +static_assert(sizeof(lmdb::val) == sizeof(MDB_val), "sizeof(lmdb::val) != sizeof(MDB_val)"); +#endif + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Environment */ + +namespace lmdb { + class env; +} + +/** + * Resource class for `MDB_env*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__env + */ +class lmdb::env { +protected: + MDB_env* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + static constexpr mode default_mode = 0644; /* -rw-r--r-- */ + + /** + * Creates a new LMDB environment. + * + * @param flags + * @throws lmdb::error on failure + */ + static env create(const unsigned int flags = default_flags) { + MDB_env* handle{nullptr}; + lmdb::env_create(&handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + if (flags) { + try { + lmdb::env_set_flags(handle, flags); + } + catch (const lmdb::error&) { + lmdb::env_close(handle); + throw; + } + } + return env{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_env*` handle + */ + env(MDB_env* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + env(env&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + env& operator=(env&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~env() noexcept { + try { close(); } catch (...) {} + } + + /** + * Returns the underlying `MDB_env*` handle. + */ + operator MDB_env*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_env*` handle. + */ + MDB_env* handle() const noexcept { + return _handle; + } + + /** + * Flushes data buffers to disk. + * + * @param force + * @throws lmdb::error on failure + */ + void sync(const bool force = true) { + lmdb::env_sync(handle(), force); + } + + /** + * Closes this environment, releasing the memory map. + * + * @note this method is idempotent + * @post `handle() == nullptr` + */ + void close() noexcept { + if (handle()) { + lmdb::env_close(handle()); + _handle = nullptr; + } + } + + /** + * Opens this environment. + * + * @param path + * @param flags + * @param mode + * @throws lmdb::error on failure + */ + env& open(const char* const path, + const unsigned int flags = default_flags, + const mode mode = default_mode) { + lmdb::env_open(handle(), path, flags, mode); + return *this; + } + + /** + * @param flags + * @param onoff + * @throws lmdb::error on failure + */ + env& set_flags(const unsigned int flags, + const bool onoff = true) { + lmdb::env_set_flags(handle(), flags, onoff); + return *this; + } + + /** + * @param size + * @throws lmdb::error on failure + */ + env& set_mapsize(const std::size_t size) { + lmdb::env_set_mapsize(handle(), size); + return *this; + } + + /** + * @param count + * @throws lmdb::error on failure + */ + env& set_max_readers(const unsigned int count) { + lmdb::env_set_max_readers(handle(), count); + return *this; + } + + /** + * @param count + * @throws lmdb::error on failure + */ + env& set_max_dbs(const MDB_dbi count) { + lmdb::env_set_max_dbs(handle(), count); + return *this; + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Transactions */ + +namespace lmdb { + class txn; +} + +/** + * Resource class for `MDB_txn*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__txn + */ +class lmdb::txn { +protected: + MDB_txn* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + + /** + * Creates a new LMDB transaction. + * + * @param env the environment handle + * @param parent + * @param flags + * @throws lmdb::error on failure + */ + static txn begin(MDB_env* const env, + MDB_txn* const parent = nullptr, + const unsigned int flags = default_flags) { + MDB_txn* handle{nullptr}; + lmdb::txn_begin(env, parent, flags, &handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + return txn{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_txn*` handle + */ + txn(MDB_txn* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + txn(txn&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + txn& operator=(txn&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~txn() noexcept { + if (_handle) { + try { abort(); } catch (...) {} + _handle = nullptr; + } + } + + /** + * Returns the underlying `MDB_txn*` handle. + */ + operator MDB_txn*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_txn*` handle. + */ + MDB_txn* handle() const noexcept { + return _handle; + } + + /** + * Returns the transaction's `MDB_env*` handle. + */ + MDB_env* env() const noexcept { + return lmdb::txn_env(handle()); + } + + /** + * Commits this transaction. + * + * @throws lmdb::error on failure + * @post `handle() == nullptr` + */ + void commit() { + lmdb::txn_commit(_handle); + _handle = nullptr; + } + + /** + * Aborts this transaction. + * + * @post `handle() == nullptr` + */ + void abort() noexcept { + lmdb::txn_abort(_handle); + _handle = nullptr; + } + + /** + * Resets this read-only transaction. + */ + void reset() noexcept { + lmdb::txn_reset(_handle); + } + + /** + * Renews this read-only transaction. + * + * @throws lmdb::error on failure + */ + void renew() { + lmdb::txn_renew(_handle); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Databases */ + +namespace lmdb { + class dbi; +} + +/** + * Resource class for `MDB_dbi` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__mdb.html#gadbe68a06c448dfb62da16443d251a78b + */ +class lmdb::dbi { +protected: + MDB_dbi _handle{0}; + +public: + static constexpr unsigned int default_flags = 0; + static constexpr unsigned int default_put_flags = 0; + + /** + * Opens a database handle. + * + * @param txn the transaction handle + * @param name + * @param flags + * @throws lmdb::error on failure + */ + static dbi + open(MDB_txn* const txn, + const char* const name = nullptr, + const unsigned int flags = default_flags) { + MDB_dbi handle{}; + lmdb::dbi_open(txn, name, flags, &handle); + return dbi{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_dbi` handle + */ + dbi(const MDB_dbi handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + dbi(dbi&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + dbi& operator=(dbi&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~dbi() noexcept { + if (_handle) { + /* No need to call close() here. */ + } + } + + /** + * Returns the underlying `MDB_dbi` handle. + */ + operator MDB_dbi() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_dbi` handle. + */ + MDB_dbi handle() const noexcept { + return _handle; + } + + /** + * Returns statistics for this database. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + MDB_stat stat(MDB_txn* const txn) const { + MDB_stat result; + lmdb::dbi_stat(txn, handle(), &result); + return result; + } + + /** + * Retrieves the flags for this database handle. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + unsigned int flags(MDB_txn* const txn) const { + unsigned int result{}; + lmdb::dbi_flags(txn, handle(), &result); + return result; + } + + /** + * Returns the number of records in this database. + * + * @param txn a transaction handle + * @throws lmdb::error on failure + */ + std::size_t size(MDB_txn* const txn) const { + return stat(txn).ms_entries; + } + + /** + * @param txn a transaction handle + * @param del + * @throws lmdb::error on failure + */ + void drop(MDB_txn* const txn, + const bool del = false) { + lmdb::dbi_drop(txn, handle(), del); + } + + /** + * Sets a custom key comparison function for this database. + * + * @param txn a transaction handle + * @param cmp the comparison function + * @throws lmdb::error on failure + */ + dbi& set_compare(MDB_txn* const txn, + MDB_cmp_func* const cmp = nullptr) { + lmdb::dbi_set_compare(txn, handle(), cmp); + return *this; + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @param data + * @throws lmdb::error on failure + */ + bool get(MDB_txn* const txn, + const val& key, + val& data) { + return lmdb::dbi_get(txn, handle(), key, data); + } + + /** + * Retrieves a key from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const K& key) const { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + return lmdb::dbi_get(txn, handle(), k, v); + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @param val + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const K& key, + V& val) const { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + const bool result = lmdb::dbi_get(txn, handle(), k, v); + if (result) { + val = *v.data(); + } + return result; + } + + /** + * Retrieves a key/value pair from this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val + * @throws lmdb::error on failure + */ + template + bool get(MDB_txn* const txn, + const char* const key, + V& val) const { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{}; + const bool result = lmdb::dbi_get(txn, handle(), k, v); + if (result) { + val = *v.data(); + } + return result; + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key + * @param data + * @param flags + * @throws lmdb::error on failure + */ + bool put(MDB_txn* const txn, + const val& key, + val& data, + const unsigned int flags = default_put_flags) { + return lmdb::dbi_put(txn, handle(), key, data, flags); + } + + /** + * Stores a key into this database. + * + * @param txn a transaction handle + * @param key + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const K& key, + const unsigned int flags = default_put_flags) { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key + * @param val + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const K& key, + const V& val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{&key, sizeof(K)}; + lmdb::val v{&val, sizeof(V)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val + * @param flags + * @throws lmdb::error on failure + */ + template + bool put(MDB_txn* const txn, + const char* const key, + const V& val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{&val, sizeof(V)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Stores a key/value pair into this database. + * + * @param txn a transaction handle + * @param key a NUL-terminated string key + * @param val a NUL-terminated string key + * @param flags + * @throws lmdb::error on failure + */ + bool put(MDB_txn* const txn, + const char* const key, + const char* const val, + const unsigned int flags = default_put_flags) { + const lmdb::val k{key, std::strlen(key)}; + lmdb::val v{val, std::strlen(val)}; + return lmdb::dbi_put(txn, handle(), k, v, flags); + } + + /** + * Removes a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + bool del(MDB_txn* const txn, + const val& key) { + return lmdb::dbi_del(txn, handle(), key); + } + + /** + * Removes a key/value pair from this database. + * + * @param txn a transaction handle + * @param key + * @throws lmdb::error on failure + */ + template + bool del(MDB_txn* const txn, + const K& key) { + const lmdb::val k{&key, sizeof(K)}; + return lmdb::dbi_del(txn, handle(), k); + } +}; + +//////////////////////////////////////////////////////////////////////////////// +/* Resource Interface: Cursors */ + +namespace lmdb { + class cursor; +} + +/** + * Resource class for `MDB_cursor*` handles. + * + * @note Instances of this class are movable, but not copyable. + * @see http://symas.com/mdb/doc/group__internal.html#structMDB__cursor + */ +class lmdb::cursor { +protected: + MDB_cursor* _handle{nullptr}; + +public: + static constexpr unsigned int default_flags = 0; + + /** + * Creates an LMDB cursor. + * + * @param txn the transaction handle + * @param dbi the database handle + * @throws lmdb::error on failure + */ + static cursor + open(MDB_txn* const txn, + const MDB_dbi dbi) { + MDB_cursor* handle{}; + lmdb::cursor_open(txn, dbi, &handle); +#ifdef LMDBXX_DEBUG + assert(handle != nullptr); +#endif + return cursor{handle}; + } + + /** + * Constructor. + * + * @param handle a valid `MDB_cursor*` handle + */ + cursor(MDB_cursor* const handle) noexcept + : _handle{handle} {} + + /** + * Move constructor. + */ + cursor(cursor&& other) noexcept { + std::swap(_handle, other._handle); + } + + /** + * Move assignment operator. + */ + cursor& operator=(cursor&& other) noexcept { + if (this != &other) { + std::swap(_handle, other._handle); + } + return *this; + } + + /** + * Destructor. + */ + ~cursor() noexcept { + try { close(); } catch (...) {} + } + + /** + * Returns the underlying `MDB_cursor*` handle. + */ + operator MDB_cursor*() const noexcept { + return _handle; + } + + /** + * Returns the underlying `MDB_cursor*` handle. + */ + MDB_cursor* handle() const noexcept { + return _handle; + } + + /** + * Closes this cursor. + * + * @note this method is idempotent + * @post `handle() == nullptr` + */ + void close() noexcept { + if (_handle) { + lmdb::cursor_close(_handle); + _handle = nullptr; + } + } + + /** + * Renews this cursor. + * + * @param txn the transaction scope + * @throws lmdb::error on failure + */ + void renew(MDB_txn* const txn) { + lmdb::cursor_renew(txn, handle()); + } + + /** + * Returns the cursor's transaction handle. + */ + MDB_txn* txn() const noexcept { + return lmdb::cursor_txn(handle()); + } + + /** + * Returns the cursor's database handle. + */ + MDB_dbi dbi() const noexcept { + return lmdb::cursor_dbi(handle()); + } + + /** + * Retrieves a key from the database. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + bool get(MDB_val* const key, + const MDB_cursor_op op) { + return get(key, nullptr, op); + } + + /** + * Retrieves a key from the database. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + bool get(lmdb::val& key, + const MDB_cursor_op op) { + return get(key, nullptr, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val (may be `nullptr`) + * @param op + * @throws lmdb::error on failure + */ + bool get(MDB_val* const key, + MDB_val* const val, + const MDB_cursor_op op) { + return lmdb::cursor_get(handle(), key, val, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val + * @param op + * @throws lmdb::error on failure + */ + bool get(lmdb::val& key, + lmdb::val& val, + const MDB_cursor_op op) { + return lmdb::cursor_get(handle(), key, val, op); + } + + /** + * Retrieves a key/value pair from the database. + * + * @param key + * @param val + * @param op + * @throws lmdb::error on failure + */ + bool get(std::string& key, + std::string& val, + const MDB_cursor_op op) { + lmdb::val k{}, v{}; + const bool found = get(k, v, op); + if (found) { + key.assign(k.data(), k.size()); + val.assign(v.data(), v.size()); + } + return found; + } + + /** + * Positions this cursor at the given key. + * + * @param key + * @param op + * @throws lmdb::error on failure + */ + template + bool find(const K& key, + const MDB_cursor_op op = MDB_SET) { + lmdb::val k{&key, sizeof(K)}; + return get(k, nullptr, op); + } +}; + +//////////////////////////////////////////////////////////////////////////////// + +#endif /* LMDBXX_H */ diff --git a/cpp/include/prettyprint.hpp b/cpp/include/prettyprint.hpp new file mode 100644 index 0000000..ebba086 --- /dev/null +++ b/cpp/include/prettyprint.hpp @@ -0,0 +1,446 @@ +// https://github.com/louisdx/cxx-prettyprint/blob/master/prettyprint.hpp 27cad81cabff867ed3f72fdf7a195d2c2a90552f +// Copyright Louis Delacroix 2010 - 2014. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) +// +// A pretty printing library for C++ +// +// Usage: +// Include this header, and operator<< will "just work". + +#ifndef H_PRETTY_PRINT +#define H_PRETTY_PRINT + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pretty_print +{ + namespace detail + { + // SFINAE type trait to detect whether T::const_iterator exists. + + struct sfinae_base + { + using yes = char; + using no = yes[2]; + }; + + template + struct has_const_iterator : private sfinae_base + { + private: + template static yes & test(typename C::const_iterator*); + template static no & test(...); + public: + static const bool value = sizeof(test(nullptr)) == sizeof(yes); + using type = T; + }; + + template + struct has_begin_end : private sfinae_base + { + private: + template + static yes & f(typename std::enable_if< + std::is_same(&C::begin)), + typename C::const_iterator(C::*)() const>::value>::type *); + + template static no & f(...); + + template + static yes & g(typename std::enable_if< + std::is_same(&C::end)), + typename C::const_iterator(C::*)() const>::value, void>::type*); + + template static no & g(...); + + public: + static bool const beg_value = sizeof(f(nullptr)) == sizeof(yes); + static bool const end_value = sizeof(g(nullptr)) == sizeof(yes); + }; + + } // namespace detail + + + // Holds the delimiter values for a specific character type + + template + struct delimiters_values + { + using char_type = TChar; + const char_type * prefix; + const char_type * delimiter; + const char_type * postfix; + }; + + + // Defines the delimiter values for a specific container and character type + + template + struct delimiters + { + using type = delimiters_values; + static const type values; + }; + + + // Functor to print containers. You can use this directly if you want + // to specificy a non-default delimiters type. The printing logic can + // be customized by specializing the nested template. + + template , + typename TDelimiters = delimiters> + struct print_container_helper + { + using delimiters_type = TDelimiters; + using ostream_type = std::basic_ostream; + + template + struct printer + { + static void print_body(const U & c, ostream_type & stream) + { + using std::begin; + using std::end; + + auto it = begin(c); + const auto the_end = end(c); + + if (it != the_end) + { + for ( ; ; ) + { + stream << *it; + + if (++it == the_end) break; + + if (delimiters_type::values.delimiter != NULL) + stream << delimiters_type::values.delimiter; + } + } + } + }; + + print_container_helper(const T & container) + : container_(container) + { } + + inline void operator()(ostream_type & stream) const + { + if (delimiters_type::values.prefix != NULL) + stream << delimiters_type::values.prefix; + + printer::print_body(container_, stream); + + if (delimiters_type::values.postfix != NULL) + stream << delimiters_type::values.postfix; + } + + private: + const T & container_; + }; + + // Specialization for pairs + + template + template + struct print_container_helper::printer> + { + using ostream_type = typename print_container_helper::ostream_type; + + static void print_body(const std::pair & c, ostream_type & stream) + { + stream << c.first; + if (print_container_helper::delimiters_type::values.delimiter != NULL) + stream << print_container_helper::delimiters_type::values.delimiter; + stream << c.second; + } + }; + + // Specialization for tuples + + template + template + struct print_container_helper::printer> + { + using ostream_type = typename print_container_helper::ostream_type; + using element_type = std::tuple; + + template struct Int { }; + + static void print_body(const element_type & c, ostream_type & stream) + { + tuple_print(c, stream, Int<0>()); + } + + static void tuple_print(const element_type &, ostream_type &, Int) + { + } + + static void tuple_print(const element_type & c, ostream_type & stream, + typename std::conditional, std::nullptr_t>::type) + { + stream << std::get<0>(c); + tuple_print(c, stream, Int<1>()); + } + + template + static void tuple_print(const element_type & c, ostream_type & stream, Int) + { + if (print_container_helper::delimiters_type::values.delimiter != NULL) + stream << print_container_helper::delimiters_type::values.delimiter; + + stream << std::get(c); + + tuple_print(c, stream, Int()); + } + }; + + // Prints a print_container_helper to the specified stream. + + template + inline std::basic_ostream & operator<<( + std::basic_ostream & stream, + const print_container_helper & helper) + { + helper(stream); + return stream; + } + + + // Basic is_container template; specialize to derive from std::true_type for all desired container types + + template + struct is_container : public std::integral_constant::value && + detail::has_begin_end::beg_value && + detail::has_begin_end::end_value> { }; + + template + struct is_container : std::true_type { }; + + template + struct is_container : std::false_type { }; + + template + struct is_container> : std::true_type { }; + + template + struct is_container> : std::true_type { }; + + template + struct is_container> : std::true_type { }; + + + // Default delimiters + + template struct delimiters { static const delimiters_values values; }; + template const delimiters_values delimiters::values = { "[", ", ", "]" }; + template struct delimiters { static const delimiters_values values; }; + template const delimiters_values delimiters::values = { L"[", L", ", L"]" }; + + + // Delimiters for (multi)set and unordered_(multi)set + + template + struct delimiters< ::std::set, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::set, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::set, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::set, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::multiset, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::multiset, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::multiset, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::multiset, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::unordered_set, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_set, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::unordered_set, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_set, wchar_t>::values = { L"{", L", ", L"}" }; + + template + struct delimiters< ::std::unordered_multiset, char> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_multiset, char>::values = { "{", ", ", "}" }; + + template + struct delimiters< ::std::unordered_multiset, wchar_t> { static const delimiters_values values; }; + + template + const delimiters_values delimiters< ::std::unordered_multiset, wchar_t>::values = { L"{", L", ", L"}" }; + + + // Delimiters for pair and tuple + + template struct delimiters, char> { static const delimiters_values values; }; + template const delimiters_values delimiters, char>::values = { "(", ", ", ")" }; + template struct delimiters< ::std::pair, wchar_t> { static const delimiters_values values; }; + template const delimiters_values delimiters< ::std::pair, wchar_t>::values = { L"(", L", ", L")" }; + + template struct delimiters, char> { static const delimiters_values values; }; + template const delimiters_values delimiters, char>::values = { "(", ", ", ")" }; + template struct delimiters< ::std::tuple, wchar_t> { static const delimiters_values values; }; + template const delimiters_values delimiters< ::std::tuple, wchar_t>::values = { L"(", L", ", L")" }; + + + // Type-erasing helper class for easy use of custom delimiters. + // Requires TCharTraits = std::char_traits and TChar = char or wchar_t, and MyDelims needs to be defined for TChar. + // Usage: "cout << pretty_print::custom_delims(x)". + + struct custom_delims_base + { + virtual ~custom_delims_base() { } + virtual std::ostream & stream(::std::ostream &) = 0; + virtual std::wostream & stream(::std::wostream &) = 0; + }; + + template + struct custom_delims_wrapper : custom_delims_base + { + custom_delims_wrapper(const T & t_) : t(t_) { } + + std::ostream & stream(std::ostream & s) + { + return s << print_container_helper, Delims>(t); + } + + std::wostream & stream(std::wostream & s) + { + return s << print_container_helper, Delims>(t); + } + + private: + const T & t; + }; + + template + struct custom_delims + { + template + custom_delims(const Container & c) : base(new custom_delims_wrapper(c)) { } + + std::unique_ptr base; + }; + + template + inline std::basic_ostream & operator<<(std::basic_ostream & s, const custom_delims & p) + { + return p.base->stream(s); + } + + + // A wrapper for a C-style array given as pointer-plus-size. + // Usage: std::cout << pretty_print_array(arr, n) << std::endl; + + template + struct array_wrapper_n + { + typedef const T * const_iterator; + typedef T value_type; + + array_wrapper_n(const T * const a, size_t n) : _array(a), _n(n) { } + inline const_iterator begin() const { return _array; } + inline const_iterator end() const { return _array + _n; } + + private: + const T * const _array; + size_t _n; + }; + + + // A wrapper for hash-table based containers that offer local iterators to each bucket. + // Usage: std::cout << bucket_print(m, 4) << std::endl; (Prints bucket 5 of container m.) + + template + struct bucket_print_wrapper + { + typedef typename T::const_local_iterator const_iterator; + typedef typename T::size_type size_type; + + const_iterator begin() const + { + return m_map.cbegin(n); + } + + const_iterator end() const + { + return m_map.cend(n); + } + + bucket_print_wrapper(const T & m, size_type bucket) : m_map(m), n(bucket) { } + + private: + const T & m_map; + const size_type n; + }; + +} // namespace pretty_print + + +// Global accessor functions for the convenience wrappers + +template +inline pretty_print::array_wrapper_n pretty_print_array(const T * const a, size_t n) +{ + return pretty_print::array_wrapper_n(a, n); +} + +template pretty_print::bucket_print_wrapper +bucket_print(const T & m, typename T::size_type n) +{ + return pretty_print::bucket_print_wrapper(m, n); +} + + +// Main magic entry point: An overload snuck into namespace std. +// Can we do better? + +namespace std +{ + // Prints a container to the stream using default delimiters + + template + inline typename enable_if< ::pretty_print::is_container::value, + basic_ostream &>::type + operator<<(basic_ostream & stream, const T & container) + { + return stream << ::pretty_print::print_container_helper(container); + } +} + + + +#endif // H_PRETTY_PRINT diff --git a/cpp/include/scope_exit.hpp b/cpp/include/scope_exit.hpp new file mode 100644 index 0000000..f7cab9c --- /dev/null +++ b/cpp/include/scope_exit.hpp @@ -0,0 +1,39 @@ +// https://gist.github.com/socantre/b45b3e23e6f1f4715f08 +#include + +#pragma once + +namespace socantre { + +template struct scope_exit { + // construction + explicit scope_exit(EF &&f) noexcept(true) + : exit_function(std::move(f)), execute_on_destruction{true} {} + // move + scope_exit(scope_exit &&rhs) noexcept(true) + : exit_function(std::move(rhs.exit_function)), + execute_on_destruction{rhs.execute_on_destruction} { + rhs.release(); + } + // release + ~scope_exit() noexcept(noexcept(this->exit_function())) { + if (execute_on_destruction) + this->exit_function(); + } + void release() noexcept(true) { this->execute_on_destruction = false; } + +private: + scope_exit(scope_exit const &) = delete; + void operator=(scope_exit const &) = delete; + scope_exit &operator=(scope_exit &&) = delete; + EF exit_function; + bool execute_on_destruction; // exposition only +}; + +template +auto make_scope_exit(EF &&exit_function) noexcept(true) -> decltype( + scope_exit>(std::forward(exit_function))) { + return scope_exit>( + std::forward(exit_function)); +} +} // namespace socantre diff --git a/cpp/include/tinyformat.h b/cpp/include/tinyformat.h new file mode 100644 index 0000000..1194d75 --- /dev/null +++ b/cpp/include/tinyformat.h @@ -0,0 +1,1003 @@ +// tinyformat.h +// Copyright (C) 2011, Chris Foster [chris42f (at) gmail (d0t) com] +// +// Boost Software License - Version 1.0 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//------------------------------------------------------------------------------ +// Tinyformat: A minimal type safe printf replacement +// +// tinyformat.h is a type safe printf replacement library in a single C++ +// header file. Design goals include: +// +// * Type safety and extensibility for user defined types. +// * C99 printf() compatibility, to the extent possible using std::ostream +// * Simplicity and minimalism. A single header file to include and distribute +// with your projects. +// * Augment rather than replace the standard stream formatting mechanism +// * C++98 support, with optional C++11 niceties +// +// +// Main interface example usage +// ---------------------------- +// +// To print a date to std::cout: +// +// std::string weekday = "Wednesday"; +// const char* month = "July"; +// size_t day = 27; +// long hour = 14; +// int min = 44; +// +// tfm::printf("%s, %s %d, %.2d:%.2d\n", weekday, month, day, hour, min); +// +// The strange types here emphasize the type safety of the interface; it is +// possible to print a std::string using the "%s" conversion, and a +// size_t using the "%d" conversion. A similar result could be achieved +// using either of the tfm::format() functions. One prints on a user provided +// stream: +// +// tfm::format(std::cerr, "%s, %s %d, %.2d:%.2d\n", +// weekday, month, day, hour, min); +// +// The other returns a std::string: +// +// std::string date = tfm::format("%s, %s %d, %.2d:%.2d\n", +// weekday, month, day, hour, min); +// std::cout << date; +// +// These are the three primary interface functions. +// +// +// User defined format functions +// ----------------------------- +// +// Simulating variadic templates in C++98 is pretty painful since it requires +// writing out the same function for each desired number of arguments. To make +// this bearable tinyformat comes with a set of macros which are used +// internally to generate the API, but which may also be used in user code. +// +// The three macros TINYFORMAT_ARGTYPES(n), TINYFORMAT_VARARGS(n) and +// TINYFORMAT_PASSARGS(n) will generate a list of n argument types, +// type/name pairs and argument names respectively when called with an integer +// n between 1 and 16. We can use these to define a macro which generates the +// desired user defined function with n arguments. To generate all 16 user +// defined function bodies, use the macro TINYFORMAT_FOREACH_ARGNUM. For an +// example, see the implementation of printf() at the end of the source file. +// +// Sometimes it's useful to be able to pass a list of format arguments through +// to a non-template function. The FormatList class is provided as a way to do +// this by storing the argument list in a type-opaque way. Continuing the +// example from above, we construct a FormatList using makeFormatList(): +// +// FormatListRef formatList = tfm::makeFormatList(weekday, month, day, hour, min); +// +// The format list can now be passed into any non-template function and used +// via a call to the vformat() function: +// +// tfm::vformat(std::cout, "%s, %s %d, %.2d:%.2d\n", formatList); +// +// +// Additional API information +// -------------------------- +// +// Error handling: Define TINYFORMAT_ERROR to customize the error handling for +// format strings which are unsupported or have the wrong number of format +// specifiers (calls assert() by default). +// +// User defined types: Uses operator<< for user defined types by default. +// Overload formatValue() for more control. + + +#ifndef TINYFORMAT_H_INCLUDED +#define TINYFORMAT_H_INCLUDED + +namespace tinyformat {} +//------------------------------------------------------------------------------ +// Config section. Customize to your liking! + +// Namespace alias to encourage brevity +namespace tfm = tinyformat; + +// Error handling; calls assert() by default. +// #define TINYFORMAT_ERROR(reasonString) your_error_handler(reasonString) + +// Define for C++11 variadic templates which make the code shorter & more +// general. If you don't define this, C++11 support is autodetected below. +// #define TINYFORMAT_USE_VARIADIC_TEMPLATES + + +//------------------------------------------------------------------------------ +// Implementation details. +#include +#include +#include +#include + +#ifndef TINYFORMAT_ERROR +# define TINYFORMAT_ERROR(reason) assert(0 && reason) +#endif + +#if !defined(TINYFORMAT_USE_VARIADIC_TEMPLATES) && !defined(TINYFORMAT_NO_VARIADIC_TEMPLATES) +# ifdef __GXX_EXPERIMENTAL_CXX0X__ +# define TINYFORMAT_USE_VARIADIC_TEMPLATES +# endif +#endif + +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20080201 +// std::showpos is broken on old libstdc++ as provided with OSX. See +// http://gcc.gnu.org/ml/libstdc++/2007-11/msg00075.html +# define TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND +#endif + +namespace tinyformat { + +//------------------------------------------------------------------------------ +namespace detail { + +// Test whether type T1 is convertible to type T2 +template +struct is_convertible +{ + private: + // two types of different size + struct fail { char dummy[2]; }; + struct succeed { char dummy; }; + // Try to convert a T1 to a T2 by plugging into tryConvert + static fail tryConvert(...); + static succeed tryConvert(const T2&); + static const T1& makeT1(); + public: +# ifdef _MSC_VER + // Disable spurious loss of precision warnings in tryConvert(makeT1()) +# pragma warning(push) +# pragma warning(disable:4244) +# pragma warning(disable:4267) +# endif + // Standard trick: the (...) version of tryConvert will be chosen from + // the overload set only if the version taking a T2 doesn't match. + // Then we compare the sizes of the return types to check which + // function matched. Very neat, in a disgusting kind of way :) + static const bool value = + sizeof(tryConvert(makeT1())) == sizeof(succeed); +# ifdef _MSC_VER +# pragma warning(pop) +# endif +}; + + +// Detect when a type is not a wchar_t string +template struct is_wchar { typedef int tinyformat_wchar_is_not_supported; }; +template<> struct is_wchar {}; +template<> struct is_wchar {}; +template struct is_wchar {}; +template struct is_wchar {}; + + +// Format the value by casting to type fmtT. This default implementation +// should never be called. +template::value> +struct formatValueAsType +{ + static void invoke(std::ostream& /*out*/, const T& /*value*/) { assert(0); } +}; +// Specialized version for types that can actually be converted to fmtT, as +// indicated by the "convertible" template parameter. +template +struct formatValueAsType +{ + static void invoke(std::ostream& out, const T& value) + { out << static_cast(value); } +}; + +#ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND +template::value> +struct formatZeroIntegerWorkaround +{ + static bool invoke(std::ostream& /**/, const T& /**/) { return false; } +}; +template +struct formatZeroIntegerWorkaround +{ + static bool invoke(std::ostream& out, const T& value) + { + if (static_cast(value) == 0 && out.flags() & std::ios::showpos) + { + out << "+0"; + return true; + } + return false; + } +}; +#endif // TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND + +// Convert an arbitrary type to integer. The version with convertible=false +// throws an error. +template::value> +struct convertToInt +{ + static int invoke(const T& /*value*/) + { + TINYFORMAT_ERROR("tinyformat: Cannot convert from argument type to " + "integer for use as variable width or precision"); + return 0; + } +}; +// Specialization for convertToInt when conversion is possible +template +struct convertToInt +{ + static int invoke(const T& value) { return static_cast(value); } +}; + +// Format at most ntrunc characters to the given stream. +template +inline void formatTruncated(std::ostream& out, const T& value, int ntrunc) +{ + std::ostringstream tmp; + tmp << value; + std::string result = tmp.str(); + out.write(result.c_str(), std::min(ntrunc, static_cast(result.size()))); +} +#define TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(type) \ +inline void formatTruncated(std::ostream& out, type* value, int ntrunc) \ +{ \ + std::streamsize len = 0; \ + while(len < ntrunc && value[len] != 0) \ + ++len; \ + out.write(value, len); \ +} +// Overload for const char* and char*. Could overload for signed & unsigned +// char too, but these are technically unneeded for printf compatibility. +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(const char) +TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char) +#undef TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR + +} // namespace detail + + +//------------------------------------------------------------------------------ +// Variable formatting functions. May be overridden for user-defined types if +// desired. + + +/// Format a value into a stream, delegating to operator<< by default. +/// +/// Users may override this for their own types. When this function is called, +/// the stream flags will have been modified according to the format string. +/// The format specification is provided in the range [fmtBegin, fmtEnd). For +/// truncating conversions, ntrunc is set to the desired maximum number of +/// characters, for example "%.7s" calls formatValue with ntrunc = 7. +/// +/// By default, formatValue() uses the usual stream insertion operator +/// operator<< to format the type T, with special cases for the %c and %p +/// conversions. +template +inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, + const char* fmtEnd, int ntrunc, const T& value) +{ +#ifndef TINYFORMAT_ALLOW_WCHAR_STRINGS + // Since we don't support printing of wchar_t using "%ls", make it fail at + // compile time in preference to printing as a void* at runtime. + typedef typename detail::is_wchar::tinyformat_wchar_is_not_supported DummyType; + (void) DummyType(); // avoid unused type warning with gcc-4.8 +#endif + // The mess here is to support the %c and %p conversions: if these + // conversions are active we try to convert the type to a char or const + // void* respectively and format that instead of the value itself. For the + // %p conversion it's important to avoid dereferencing the pointer, which + // could otherwise lead to a crash when printing a dangling (const char*). + const bool canConvertToChar = detail::is_convertible::value; + const bool canConvertToVoidPtr = detail::is_convertible::value; + if(canConvertToChar && *(fmtEnd-1) == 'c') + detail::formatValueAsType::invoke(out, value); + else if(canConvertToVoidPtr && *(fmtEnd-1) == 'p') + detail::formatValueAsType::invoke(out, value); +#ifdef TINYFORMAT_OLD_LIBSTDCPLUSPLUS_WORKAROUND + else if(detail::formatZeroIntegerWorkaround::invoke(out, value)) /**/; +#endif + else if(ntrunc >= 0) + { + // Take care not to overread C strings in truncating conversions like + // "%.4s" where at most 4 characters may be read. + detail::formatTruncated(out, value, ntrunc); + } + else + out << value; +} + + +// Overloaded version for char types to support printing as an integer +#define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \ +inline void formatValue(std::ostream& out, const char* /*fmtBegin*/, \ + const char* fmtEnd, int /**/, charType value) \ +{ \ + switch(*(fmtEnd-1)) \ + { \ + case 'u': case 'd': case 'i': case 'o': case 'X': case 'x': \ + out << static_cast(value); break; \ + default: \ + out << value; break; \ + } \ +} +// per 3.9.1: char, signed char and unsigned char are all distinct types +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(char) +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(signed char) +TINYFORMAT_DEFINE_FORMATVALUE_CHAR(unsigned char) +#undef TINYFORMAT_DEFINE_FORMATVALUE_CHAR + + +//------------------------------------------------------------------------------ +// Tools for emulating variadic templates in C++98. The basic idea here is +// stolen from the boost preprocessor metaprogramming library and cut down to +// be just general enough for what we need. + +#define TINYFORMAT_ARGTYPES(n) TINYFORMAT_ARGTYPES_ ## n +#define TINYFORMAT_VARARGS(n) TINYFORMAT_VARARGS_ ## n +#define TINYFORMAT_PASSARGS(n) TINYFORMAT_PASSARGS_ ## n +#define TINYFORMAT_PASSARGS_TAIL(n) TINYFORMAT_PASSARGS_TAIL_ ## n + +// To keep it as transparent as possible, the macros below have been generated +// using python via the excellent cog.py code generation script. This avoids +// the need for a bunch of complex (but more general) preprocessor tricks as +// used in boost.preprocessor. +// +// To rerun the code generation in place, use `cog.py -r tinyformat.h` +// (see http://nedbatchelder.com/code/cog). Alternatively you can just create +// extra versions by hand. + +/*[[[cog +maxParams = 16 + +def makeCommaSepLists(lineTemplate, elemTemplate, startInd=1): + for j in range(startInd,maxParams+1): + list = ', '.join([elemTemplate % {'i':i} for i in range(startInd,j+1)]) + cog.outl(lineTemplate % {'j':j, 'list':list}) + +makeCommaSepLists('#define TINYFORMAT_ARGTYPES_%(j)d %(list)s', + 'class T%(i)d') + +cog.outl() +makeCommaSepLists('#define TINYFORMAT_VARARGS_%(j)d %(list)s', + 'const T%(i)d& v%(i)d') + +cog.outl() +makeCommaSepLists('#define TINYFORMAT_PASSARGS_%(j)d %(list)s', 'v%(i)d') + +cog.outl() +cog.outl('#define TINYFORMAT_PASSARGS_TAIL_1') +makeCommaSepLists('#define TINYFORMAT_PASSARGS_TAIL_%(j)d , %(list)s', + 'v%(i)d', startInd = 2) + +cog.outl() +cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' + + ' '.join(['m(%d)' % (j,) for j in range(1,maxParams+1)])) +]]]*/ +#define TINYFORMAT_ARGTYPES_1 class T1 +#define TINYFORMAT_ARGTYPES_2 class T1, class T2 +#define TINYFORMAT_ARGTYPES_3 class T1, class T2, class T3 +#define TINYFORMAT_ARGTYPES_4 class T1, class T2, class T3, class T4 +#define TINYFORMAT_ARGTYPES_5 class T1, class T2, class T3, class T4, class T5 +#define TINYFORMAT_ARGTYPES_6 class T1, class T2, class T3, class T4, class T5, class T6 +#define TINYFORMAT_ARGTYPES_7 class T1, class T2, class T3, class T4, class T5, class T6, class T7 +#define TINYFORMAT_ARGTYPES_8 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8 +#define TINYFORMAT_ARGTYPES_9 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9 +#define TINYFORMAT_ARGTYPES_10 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10 +#define TINYFORMAT_ARGTYPES_11 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11 +#define TINYFORMAT_ARGTYPES_12 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12 +#define TINYFORMAT_ARGTYPES_13 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13 +#define TINYFORMAT_ARGTYPES_14 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14 +#define TINYFORMAT_ARGTYPES_15 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14, class T15 +#define TINYFORMAT_ARGTYPES_16 class T1, class T2, class T3, class T4, class T5, class T6, class T7, class T8, class T9, class T10, class T11, class T12, class T13, class T14, class T15, class T16 + +#define TINYFORMAT_VARARGS_1 const T1& v1 +#define TINYFORMAT_VARARGS_2 const T1& v1, const T2& v2 +#define TINYFORMAT_VARARGS_3 const T1& v1, const T2& v2, const T3& v3 +#define TINYFORMAT_VARARGS_4 const T1& v1, const T2& v2, const T3& v3, const T4& v4 +#define TINYFORMAT_VARARGS_5 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5 +#define TINYFORMAT_VARARGS_6 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6 +#define TINYFORMAT_VARARGS_7 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7 +#define TINYFORMAT_VARARGS_8 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8 +#define TINYFORMAT_VARARGS_9 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9 +#define TINYFORMAT_VARARGS_10 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10 +#define TINYFORMAT_VARARGS_11 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11 +#define TINYFORMAT_VARARGS_12 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12 +#define TINYFORMAT_VARARGS_13 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13 +#define TINYFORMAT_VARARGS_14 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14 +#define TINYFORMAT_VARARGS_15 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14, const T15& v15 +#define TINYFORMAT_VARARGS_16 const T1& v1, const T2& v2, const T3& v3, const T4& v4, const T5& v5, const T6& v6, const T7& v7, const T8& v8, const T9& v9, const T10& v10, const T11& v11, const T12& v12, const T13& v13, const T14& v14, const T15& v15, const T16& v16 + +#define TINYFORMAT_PASSARGS_1 v1 +#define TINYFORMAT_PASSARGS_2 v1, v2 +#define TINYFORMAT_PASSARGS_3 v1, v2, v3 +#define TINYFORMAT_PASSARGS_4 v1, v2, v3, v4 +#define TINYFORMAT_PASSARGS_5 v1, v2, v3, v4, v5 +#define TINYFORMAT_PASSARGS_6 v1, v2, v3, v4, v5, v6 +#define TINYFORMAT_PASSARGS_7 v1, v2, v3, v4, v5, v6, v7 +#define TINYFORMAT_PASSARGS_8 v1, v2, v3, v4, v5, v6, v7, v8 +#define TINYFORMAT_PASSARGS_9 v1, v2, v3, v4, v5, v6, v7, v8, v9 +#define TINYFORMAT_PASSARGS_10 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 +#define TINYFORMAT_PASSARGS_11 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11 +#define TINYFORMAT_PASSARGS_12 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12 +#define TINYFORMAT_PASSARGS_13 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13 +#define TINYFORMAT_PASSARGS_14 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14 +#define TINYFORMAT_PASSARGS_15 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 +#define TINYFORMAT_PASSARGS_16 v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16 + +#define TINYFORMAT_PASSARGS_TAIL_1 +#define TINYFORMAT_PASSARGS_TAIL_2 , v2 +#define TINYFORMAT_PASSARGS_TAIL_3 , v2, v3 +#define TINYFORMAT_PASSARGS_TAIL_4 , v2, v3, v4 +#define TINYFORMAT_PASSARGS_TAIL_5 , v2, v3, v4, v5 +#define TINYFORMAT_PASSARGS_TAIL_6 , v2, v3, v4, v5, v6 +#define TINYFORMAT_PASSARGS_TAIL_7 , v2, v3, v4, v5, v6, v7 +#define TINYFORMAT_PASSARGS_TAIL_8 , v2, v3, v4, v5, v6, v7, v8 +#define TINYFORMAT_PASSARGS_TAIL_9 , v2, v3, v4, v5, v6, v7, v8, v9 +#define TINYFORMAT_PASSARGS_TAIL_10 , v2, v3, v4, v5, v6, v7, v8, v9, v10 +#define TINYFORMAT_PASSARGS_TAIL_11 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11 +#define TINYFORMAT_PASSARGS_TAIL_12 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12 +#define TINYFORMAT_PASSARGS_TAIL_13 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13 +#define TINYFORMAT_PASSARGS_TAIL_14 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14 +#define TINYFORMAT_PASSARGS_TAIL_15 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15 +#define TINYFORMAT_PASSARGS_TAIL_16 , v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16 + +#define TINYFORMAT_FOREACH_ARGNUM(m) \ + m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) m(10) m(11) m(12) m(13) m(14) m(15) m(16) +//[[[end]]] + + + +namespace detail { + +// Type-opaque holder for an argument to format(), with associated actions on +// the type held as explicit function pointers. This allows FormatArg's for +// each argument to be allocated as a homogenous array inside FormatList +// whereas a naive implementation based on inheritance does not. +class FormatArg +{ + public: + FormatArg() {} + + template + FormatArg(const T& value) + : m_value(static_cast(&value)), + m_formatImpl(&formatImpl), + m_toIntImpl(&toIntImpl) + { } + + void format(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc) const + { + m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); + } + + int toInt() const + { + return m_toIntImpl(m_value); + } + + private: + template + static void formatImpl(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value) + { + formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast(value)); + } + + template + static int toIntImpl(const void* value) + { + return convertToInt::invoke(*static_cast(value)); + } + + const void* m_value; + void (*m_formatImpl)(std::ostream& out, const char* fmtBegin, + const char* fmtEnd, int ntrunc, const void* value); + int (*m_toIntImpl)(const void* value); +}; + + +// Parse and return an integer from the string c, as atoi() +// On return, c is set to one past the end of the integer. +inline int parseIntAndAdvance(const char*& c) +{ + int i = 0; + for(;*c >= '0' && *c <= '9'; ++c) + i = 10*i + (*c - '0'); + return i; +} + +// Print literal part of format string and return next format spec +// position. +// +// Skips over any occurrences of '%%', printing a literal '%' to the +// output. The position of the first % character of the next +// nontrivial format spec is returned, or the end of string. +inline const char* printFormatStringLiteral(std::ostream& out, const char* fmt) +{ + const char* c = fmt; + for(; true; ++c) + { + switch(*c) + { + case '\0': + out.write(fmt, static_cast(c - fmt)); + return c; + case '%': + out.write(fmt, static_cast(c - fmt)); + if(*(c+1) != '%') + return c; + // for "%%", tack trailing % onto next literal section. + fmt = ++c; + break; + } + } +} + + +// Parse a format string and set the stream state accordingly. +// +// The format mini-language recognized here is meant to be the one from C99, +// with the form "%[flags][width][.precision][length]type". +// +// Formatting options which can't be natively represented using the ostream +// state are returned in spacePadPositive (for space padded positive numbers) +// and ntrunc (for truncating conversions). argIndex is incremented if +// necessary to pull out variable width and precision . The function returns a +// pointer to the character after the end of the current format spec. +inline const char* streamStateFromFormat(std::ostream& out, bool& spacePadPositive, + int& ntrunc, const char* fmtStart, + const detail::FormatArg* formatters, + int& argIndex, int numFormatters) +{ + if(*fmtStart != '%') + { + TINYFORMAT_ERROR("tinyformat: Not enough conversion specifiers in format string"); + return fmtStart; + } + // Reset stream state to defaults. + out.width(0); + out.precision(6); + out.fill(' '); + // Reset most flags; ignore irrelevant unitbuf & skipws. + out.unsetf(std::ios::adjustfield | std::ios::basefield | + std::ios::floatfield | std::ios::showbase | std::ios::boolalpha | + std::ios::showpoint | std::ios::showpos | std::ios::uppercase); + bool precisionSet = false; + bool widthSet = false; + int widthExtra = 0; + const char* c = fmtStart + 1; + // 1) Parse flags + for(;; ++c) + { + switch(*c) + { + case '#': + out.setf(std::ios::showpoint | std::ios::showbase); + continue; + case '0': + // overridden by left alignment ('-' flag) + if(!(out.flags() & std::ios::left)) + { + // Use internal padding so that numeric values are + // formatted correctly, eg -00010 rather than 000-10 + out.fill('0'); + out.setf(std::ios::internal, std::ios::adjustfield); + } + continue; + case '-': + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + continue; + case ' ': + // overridden by show positive sign, '+' flag. + if(!(out.flags() & std::ios::showpos)) + spacePadPositive = true; + continue; + case '+': + out.setf(std::ios::showpos); + spacePadPositive = false; + widthExtra = 1; + continue; + } + break; + } + // 2) Parse width + if(*c >= '0' && *c <= '9') + { + widthSet = true; + out.width(parseIntAndAdvance(c)); + } + if(*c == '*') + { + widthSet = true; + int width = 0; + if(argIndex < numFormatters) + width = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable width"); + if(width < 0) + { + // negative widths correspond to '-' flag set + out.fill(' '); + out.setf(std::ios::left, std::ios::adjustfield); + width = -width; + } + out.width(width); + ++c; + } + // 3) Parse precision + if(*c == '.') + { + ++c; + int precision = 0; + if(*c == '*') + { + ++c; + if(argIndex < numFormatters) + precision = formatters[argIndex++].toInt(); + else + TINYFORMAT_ERROR("tinyformat: Not enough arguments to read variable precision"); + } + else + { + if(*c >= '0' && *c <= '9') + precision = parseIntAndAdvance(c); + else if(*c == '-') // negative precisions ignored, treated as zero. + parseIntAndAdvance(++c); + } + out.precision(precision); + precisionSet = true; + } + // 4) Ignore any C99 length modifier + while(*c == 'l' || *c == 'h' || *c == 'L' || + *c == 'j' || *c == 'z' || *c == 't') + ++c; + // 5) We're up to the conversion specifier character. + // Set stream flags based on conversion specifier (thanks to the + // boost::format class for forging the way here). + bool intConversion = false; + switch(*c) + { + case 'u': case 'd': case 'i': + out.setf(std::ios::dec, std::ios::basefield); + intConversion = true; + break; + case 'o': + out.setf(std::ios::oct, std::ios::basefield); + intConversion = true; + break; + case 'X': + out.setf(std::ios::uppercase); + case 'x': case 'p': + out.setf(std::ios::hex, std::ios::basefield); + intConversion = true; + break; + case 'E': + out.setf(std::ios::uppercase); + case 'e': + out.setf(std::ios::scientific, std::ios::floatfield); + out.setf(std::ios::dec, std::ios::basefield); + break; + case 'F': + out.setf(std::ios::uppercase); + case 'f': + out.setf(std::ios::fixed, std::ios::floatfield); + break; + case 'G': + out.setf(std::ios::uppercase); + case 'g': + out.setf(std::ios::dec, std::ios::basefield); + // As in boost::format, let stream decide float format. + out.flags(out.flags() & ~std::ios::floatfield); + break; + case 'a': case 'A': + TINYFORMAT_ERROR("tinyformat: the %a and %A conversion specs " + "are not supported"); + break; + case 'c': + // Handled as special case inside formatValue() + break; + case 's': + if(precisionSet) + ntrunc = static_cast(out.precision()); + // Make %s print booleans as "true" and "false" + out.setf(std::ios::boolalpha); + break; + case 'n': + // Not supported - will cause problems! + TINYFORMAT_ERROR("tinyformat: %n conversion spec not supported"); + break; + case '\0': + TINYFORMAT_ERROR("tinyformat: Conversion spec incorrectly " + "terminated by end of string"); + return c; + } + if(intConversion && precisionSet && !widthSet) + { + // "precision" for integers gives the minimum number of digits (to be + // padded with zeros on the left). This isn't really supported by the + // iostreams, but we can approximately simulate it with the width if + // the width isn't otherwise used. + out.width(out.precision() + widthExtra); + out.setf(std::ios::internal, std::ios::adjustfield); + out.fill('0'); + } + return c+1; +} + + +//------------------------------------------------------------------------------ +inline void formatImpl(std::ostream& out, const char* fmt, + const detail::FormatArg* formatters, + int numFormatters) +{ + // Saved stream state + std::streamsize origWidth = out.width(); + std::streamsize origPrecision = out.precision(); + std::ios::fmtflags origFlags = out.flags(); + char origFill = out.fill(); + + for (int argIndex = 0; argIndex < numFormatters; ++argIndex) + { + // Parse the format string + fmt = printFormatStringLiteral(out, fmt); + bool spacePadPositive = false; + int ntrunc = -1; + const char* fmtEnd = streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, + formatters, argIndex, numFormatters); + if (argIndex >= numFormatters) + { + // Check args remain after reading any variable width/precision + TINYFORMAT_ERROR("tinyformat: Not enough format arguments"); + return; + } + const FormatArg& arg = formatters[argIndex]; + // Format the arg into the stream. + if(!spacePadPositive) + arg.format(out, fmt, fmtEnd, ntrunc); + else + { + // The following is a special case with no direct correspondence + // between stream formatting and the printf() behaviour. Simulate + // it crudely by formatting into a temporary string stream and + // munging the resulting string. + std::ostringstream tmpStream; + tmpStream.copyfmt(out); + tmpStream.setf(std::ios::showpos); + arg.format(tmpStream, fmt, fmtEnd, ntrunc); + std::string result = tmpStream.str(); // allocates... yuck. + for(size_t i = 0, iend = result.size(); i < iend; ++i) + if(result[i] == '+') result[i] = ' '; + out << result; + } + fmt = fmtEnd; + } + + // Print remaining part of format string. + fmt = printFormatStringLiteral(out, fmt); + if(*fmt != '\0') + TINYFORMAT_ERROR("tinyformat: Too many conversion specifiers in format string"); + + // Restore stream state + out.width(origWidth); + out.precision(origPrecision); + out.flags(origFlags); + out.fill(origFill); +} + +} // namespace detail + + +/// List of template arguments format(), held in a type-opaque way. +/// +/// A const reference to FormatList (typedef'd as FormatListRef) may be +/// conveniently used to pass arguments to non-template functions: All type +/// information has been stripped from the arguments, leaving just enough of a +/// common interface to perform formatting as required. +class FormatList +{ + public: + FormatList(detail::FormatArg* formatters, int N) + : m_formatters(formatters), m_N(N) { } + + friend void vformat(std::ostream& out, const char* fmt, + const FormatList& list); + + private: + const detail::FormatArg* m_formatters; + int m_N; +}; + +/// Reference to type-opaque format list for passing to vformat() +typedef const FormatList& FormatListRef; + + +namespace detail { + +// Format list subclass with fixed storage to avoid dynamic allocation +template +class FormatListN : public FormatList +{ + public: +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + template + FormatListN(const Args&... args) + : FormatList(&m_formatterStore[0], N), + m_formatterStore{FormatArg(args)...} + { static_assert(sizeof...(args) == N, "Number of args must be N"); } +#else // C++98 version + void init(int) {} +# define TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR(n) \ + \ + template \ + FormatListN(TINYFORMAT_VARARGS(n)) \ + : FormatList(&m_formatterStore[0], n) \ + { assert(n == N); init(0, TINYFORMAT_PASSARGS(n)); } \ + \ + template \ + void init(int i, TINYFORMAT_VARARGS(n)) \ + { \ + m_formatterStore[i] = FormatArg(v1); \ + init(i+1 TINYFORMAT_PASSARGS_TAIL(n)); \ + } + + TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR) +# undef TINYFORMAT_MAKE_FORMATLIST_CONSTRUCTOR +#endif + + private: + FormatArg m_formatterStore[N]; +}; + +// Special 0-arg version - MSVC says zero-sized C array in struct is nonstandard +template<> class FormatListN<0> : public FormatList +{ + public: FormatListN() : FormatList(0, 0) {} +}; + +} // namespace detail + + +//------------------------------------------------------------------------------ +// Primary API functions + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Make type-agnostic format list from list of template arguments. +/// +/// The exact return type of this function is an implementation detail and +/// shouldn't be relied upon. Instead it should be stored as a FormatListRef: +/// +/// FormatListRef formatList = makeFormatList( /*...*/ ); +template +detail::FormatListN makeFormatList(const Args&... args) +{ + return detail::FormatListN(args...); +} + +#else // C++98 version + +inline detail::FormatListN<0> makeFormatList() +{ + return detail::FormatListN<0>(); +} +#define TINYFORMAT_MAKE_MAKEFORMATLIST(n) \ +template \ +detail::FormatListN makeFormatList(TINYFORMAT_VARARGS(n)) \ +{ \ + return detail::FormatListN(TINYFORMAT_PASSARGS(n)); \ +} +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_MAKEFORMATLIST) +#undef TINYFORMAT_MAKE_MAKEFORMATLIST + +#endif + +/// Format list of arguments to the stream according to the given format string. +/// +/// The name vformat() is chosen for the semantic similarity to vprintf(): the +/// list of format arguments is held in a single function argument. +inline void vformat(std::ostream& out, const char* fmt, FormatListRef list) +{ + detail::formatImpl(out, fmt, list.m_formatters, list.m_N); +} + + +#ifdef TINYFORMAT_USE_VARIADIC_TEMPLATES + +/// Format list of arguments to the stream according to given format string. +template +void format(std::ostream& out, const char* fmt, const Args&... args) +{ + vformat(out, fmt, makeFormatList(args...)); +} + +/// Format list of arguments according to the given format string and return +/// the result as a string. +template +std::string format(const char* fmt, const Args&... args) +{ + std::ostringstream oss; + format(oss, fmt, args...); + return oss.str(); +} + +/// Format list of arguments to std::cout, according to the given format string +template +void printf(const char* fmt, const Args&... args) +{ + format(std::cout, fmt, args...); +} + + +#else // C++98 version + +inline void format(std::ostream& out, const char* fmt) +{ + vformat(out, fmt, makeFormatList()); +} + +inline std::string format(const char* fmt) +{ + std::ostringstream oss; + format(oss, fmt); + return oss.str(); +} + +inline void printf(const char* fmt) +{ + format(std::cout, fmt); +} + +#define TINYFORMAT_MAKE_FORMAT_FUNCS(n) \ + \ +template \ +void format(std::ostream& out, const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + vformat(out, fmt, makeFormatList(TINYFORMAT_PASSARGS(n))); \ +} \ + \ +template \ +std::string format(const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + std::ostringstream oss; \ + format(oss, fmt, TINYFORMAT_PASSARGS(n)); \ + return oss.str(); \ +} \ + \ +template \ +void printf(const char* fmt, TINYFORMAT_VARARGS(n)) \ +{ \ + format(std::cout, fmt, TINYFORMAT_PASSARGS(n)); \ +} + +TINYFORMAT_FOREACH_ARGNUM(TINYFORMAT_MAKE_FORMAT_FUNCS) +#undef TINYFORMAT_MAKE_FORMAT_FUNCS + +#endif + + +} // namespace tinyformat + +#endif // TINYFORMAT_H_INCLUDED diff --git a/cpp/include/unique_resource.hpp b/cpp/include/unique_resource.hpp new file mode 100644 index 0000000..56b3fe4 --- /dev/null +++ b/cpp/include/unique_resource.hpp @@ -0,0 +1,78 @@ +// https://gist.github.com/socantre/b45b3e23e6f1f4715f08 + +#pragma once + +#include + +namespace socantre { + +template class unique_resource { + R resource; + D deleter; + bool execute_on_destruction; // exposition only + unique_resource &operator=(unique_resource const &) = delete; + unique_resource(unique_resource const &) = delete; // no copies! + public: + // construction + explicit unique_resource(R &&resource, D &&deleter, + bool shouldrun = true) noexcept(true) + : resource(std::move(resource)), deleter(std::move(deleter)), + execute_on_destruction{shouldrun} {} + // move + unique_resource(unique_resource &&other) noexcept(true) + : resource(std::move(other.resource)), deleter(std::move(other.deleter)), + execute_on_destruction{other.execute_on_destruction} { + other.release(); + } + unique_resource & + operator=(unique_resource &&other) noexcept(noexcept(this->reset())) { + this->reset(); + this->deleter = std::move(other.deleter); + this->resource = std::move(other.resource); + this->execute_on_destruction = other.execute_on_destruction; + other.release(); + return *this; + } + // resource release + ~unique_resource() noexcept(noexcept(this->reset())) { this->reset(); } + void reset() noexcept(noexcept(this->get_deleter()(resource))) { + if (execute_on_destruction) { + this->execute_on_destruction = false; + this->get_deleter()(resource); + } + } + void reset(R &&newresource) noexcept(noexcept(this->reset())) { + this->reset(); + this->resource = std::move(newresource); + this->execute_on_destruction = true; + } + R const &release() noexcept(true) { + this->execute_on_destruction = false; + return this->get(); + } + // resource access + R const &get() const noexcept(true) { return this->resource; } + operator R const &() const noexcept(true) { return this->resource; } + R operator->() const noexcept(true) { return this->resource; } + std::add_lvalue_reference_t> operator*() const { + return *this->resource; + } + // deleter access + const D &get_deleter() const noexcept(true) { return this->deleter; } +}; +// factories +template +auto make_unique_resource(R &&r, D &&d) noexcept(true) +-> decltype(unique_resource>( + std::move(r), std::forward>(d), true)) { + return unique_resource>( + std::move(r), std::forward>(d), true); +} +template +auto make_unique_resource_checked(R r, R invalid, D d) noexcept(true) +-> decltype(unique_resource(std::move(r), std::move(d), true)) { + bool shouldrun = not bool(r == invalid); + return unique_resource(std::move(r), std::move(d), shouldrun); +} + +} // namespace socantre diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt new file mode 100644 index 0000000..cf8e5fa --- /dev/null +++ b/cpp/src/CMakeLists.txt @@ -0,0 +1,6 @@ +#For the shared library: +set ( PROJECT_LINK_LIBS liblmdb.so) +link_directories( /usr/local/lib) + +add_library(persipubsub SHARED queue.cpp queue.h library.h library.cpp filesystem.cpp) +target_link_libraries( persipubsub ${PROJECT_LINK_LIBS}) diff --git a/cpp/src/control.h b/cpp/src/control.h new file mode 100644 index 0000000..540c771 --- /dev/null +++ b/cpp/src/control.h @@ -0,0 +1,135 @@ +// +// Created by selim on 05.02.19. +// + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub { + namespace control { + /** + * Store queue parameter in lmdb. + * + * @param max_reader_num max number of reader of the lmdb + * @param max_db_num max number of named databases + * @param max_db_size_bytes max size of the lmdb in bytes + * @param env where parameters are stored + */ + void set_queue_parameters(int max_reader_num, int max_db_num, + int max_db_size_bytes, lmdb::env env); + + /** + * Set high water mark values for queue. + * + * @param hwm high water mark values + * @param env where values are stored + */ + void set_hwm(persipubsub::queue::HighWaterMark hwm, lmdb::env env); + + /** + * Set pruning strategy for queue. + * + * @param strategy pruning strategy + * @param env where strategy are stored + */ + void set_strategy(persipubsub::queue::Strategy strategy, lmdb::env env); + + /** + * Add a subscriber and create its lmdb. + * + * @param identifier ID of the subscriber which should be added + * @param env where subscriber is added + */ + void _add_sub(char identifier, lmdb::env env); + + /** + * Control and maintain one queue. + */ + class Control { + public: + /** + * Initialize control class. + * + * @param path to the queue. + */ + Control(boost::filesystem::path path) path_(path) {}; + + /** + * Initialize control with a (re)initialized queue. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_water_mark limit of the queue + * @param strategy used to prune queue + */ + void init(std::vector subscriber_ids, int max_readers, + int max_size, + persipubsub::queue::HighWaterMark high_water_mark, + persipubsub::queue::Strategy strategy); + + /** + * Check if queue is initialized. + * + * @return is initialized when all values for the given keys are set + */ + bool check_queue_is_initialized(); + + /** + * Clear all subscriber and delete all messages for queue. + */ + void clear_all_subscribers(); + + /** + * Prune all dangling messages from the lmdb. + */ + void prune_dangling_messages(); + + ~Control(); + private: + boost::filesystem::path path_; + persipubsub::queue::Queue queue_; + std::vector subscriber_ids_; + + /** + * Reinitialize the queue which is maintained by the control. + */ + void reinitialize_queue(); + + /** + * Initialize queue. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_water_mark limit of the queue + * @param strategy used to prune queue + */ + void initialize_queue(std::vector subscriber_ids, + int max_readers, int max_size, + persipubsub::queue::HighWaterMark high_water_mark, + persipubsub::queue::Strategy strategy); + + /** + * Prune all messages of a subscriber. + * + * @param identifier ID of the subscriber of which all messages should be pruned + */ + void prune_all_messages_for(std::vector identifier); + + /** + * Remove a subscriber and delete all its messages. + * + * @param identifier ID of the subscriber which should be removed + * @param env from where subscriber is removed + */ + void remove_sub(std::vector identifier, lmdb::env env); + + }; // class Control + } // namespace control +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/environment.h b/cpp/src/environment.h new file mode 100644 index 0000000..435ec1f --- /dev/null +++ b/cpp/src/environment.h @@ -0,0 +1,75 @@ +// +// Created by selim on 05.02.19. +// +#pragma once + +#include +#include +#include "queue.h" +#include "control.h" +#include "publisher.h" +#include "subscriber.h" + +namespace persipubsub{ + namespace environment{ + + /** + * Fabricate persipubsub components. + */ + class Environment{ + + public: + /** + * Initialize. + * + * @param path to the queue + */ + Environment(const boost::filesystem::path& path) : path_(path) {}; + + /** + * Fabricate a new control. + * + * @param subscriber_ids subscribers of the queue + * @param max_readers max number of reader of the lmdb + * @param max_size of the lmdb in bytes + * @param high_watermark limit of the queue + * @param strategy used to prune queue + * @return Control to create and maintain queue + */ + persipubsub::control::Control new_control( + std::vector subscriber_ids, int max_readers, + int max_size, + persipubsub::queue::HighWaterMark high_watermark, + persipubsub::queue::Strategy strategy); + + /** + * Fabricate a new publisher. + * + * @param autosync if True, store data automatically in lmdb + * @return Publisher to send messages + */ + persipubsub::publisher::Publisher new_publisher(bool autosync); + + /** + * Fabricate a new subscriber. + * + * @param identifier of the subscriber + * @return Subscriber to receive messages + */ + persipubsub::subscriber::Subscriber new_subscriber(char identifier); + + private: + boost::filesystem::path path_; + + }; // class Environment + + /** + * Fabricate a new environment. + * + * @param path to the queue + * @return Environment to create control, publisher and subscriber + */ + persipubsub::environment::Environment new_environment(boost::filesystem::path path); + + } // namespace environment +} // namespace persipubsub diff --git a/cpp/src/filesystem.cpp b/cpp/src/filesystem.cpp new file mode 100644 index 0000000..05ee137 --- /dev/null +++ b/cpp/src/filesystem.cpp @@ -0,0 +1,317 @@ +// Copyright (c) 2016 Parquery AG. All rights reserved. +// Created by mristin on 10/12/16. + +#include "filesystem.h" + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +std::vector pqry::filesystem::ls_lt(const fs::path& directory) { + if (!fs::exists(directory)) { + throw std::invalid_argument(tfm::format("Directory must exist: %s", directory)); + } + + if (!fs::is_directory(directory)) { + throw std::invalid_argument( + tfm::format("Not a directory: %s %s", directory, fs::status(directory).type())); + } + + std::list > ts_pths; + for (const auto& entry : boost::make_iterator_range(fs::directory_iterator(directory), {})) { + struct stat st = {}; + stat(entry.path().c_str(), &st); + + // get the time in nanoseconds + const int64_t ns = pqry::filesystem::modified_time(entry.path()); + + ts_pths.emplace_back(std::make_tuple(ns, entry.path())); + } + + ts_pths.sort(); + + std::vector res; + res.reserve(ts_pths.size()); + + for (const auto& ts_pth : ts_pths) { + res.push_back(std::get<1>(ts_pth)); + } + + return res; +} + +std::vector pqry::filesystem::listdir(const fs::path& directory) { + if (not fs::exists(directory)) { + std::printf( "Directory does not exist"); + } + + if (not fs::is_directory(directory)) { + std::printf( "Not a directory"); + } + + std::vector res; + for (const auto& entry : boost::make_iterator_range(fs::directory_iterator(directory), {})) { + res.push_back(entry.path()); + } + return res; +} + +int64_t pqry::filesystem::modified_time(const fs::path& path) { + struct stat st = {}; + stat(path.c_str(), &st); + + // get the time in nanoseconds + const int64_t ns( + static_cast(st.st_mtim.tv_sec * 1000L * 1000L * 1000L) + static_cast(st.st_mtim.tv_nsec)); + + return ns; +} + +fs::path pqry::filesystem::mkdtemp() { + const fs::path tmpdir = fs::temp_directory_path() / fs::unique_path(); + fs::create_directories(tmpdir); + + return tmpdir; +} + +std::string pqry::filesystem::read(const fs::path& path) { + std::string content; + + std::ifstream ifs(path.c_str(), std::ios::in | std::ios::binary | std::ios::ate); + + if (ifs.good()) { + auto fsize = ifs.tellg(); + if (fsize < 0) { + std::printf( "File size retrieval failed"); + } + auto fsize_nonneg = static_cast(fsize); + + ifs.seekg(0, std::ios::beg); + + std::vector bytes(fsize_nonneg); + ifs.read(&bytes[0], fsize); + + if (ifs.good()) { + content = std::string(&bytes[0], fsize_nonneg); + } else { + std::printf( "error reading"); + } + } else { + std::printf( "error reading"); + } + + return content; +} + +void pqry::filesystem::write(const fs::path& path, const std::string& text) { + std::ofstream ofs(path.c_str(), std::ios::out | std::ios::binary); + ofs << text; + + if (!ofs.good()) { + std::printf( "error writing"); + } +} + +void pqry::filesystem::write(const fs::path& path, const char* bytes, int size) { + std::ofstream ofs(path.c_str(), std::ios::out | std::ios::binary); + + if (!ofs.good()) { + std::printf( "error writing"); + } + + ofs.write(bytes, size); +} + +void pqry::filesystem::copy_directory(const fs::path& source, const fs::path& dest) { + if (not fs::exists(source)) { + std::printf( "Error attempting to copy: Source does not exist"); + } + if (not fs::is_directory(source)) { + std::printf( "Error attempting to copy: Source is not a directory"); + } + if (not fs::exists(dest)) { + if (not fs::create_directories(dest)) { + std::printf( "Error attempting to create destination folder"); + } + } + + using rec_iterator = fs::recursive_directory_iterator; + for (auto it = rec_iterator(source, fs::symlink_option::no_recurse), end = rec_iterator(); it != end; ++it) { + // Remove common root + std::string difference = it->path().string(); + boost::replace_first(difference, source.string(), ""); + + if (not fs::exists(dest / difference)) { + boost::system::error_code error; + fs::copy(it->path(), dest / difference, error); + if (error.value() != 0) { + std::printf("Failed to copy"); + } + } else { + if (not fs::is_directory(it->path())) { + fs::copy_file(it->path(), dest / difference, fs::copy_option::overwrite_if_exists); + } + } + } +} + +void pqry::filesystem::unzip_archive(const fs::path& path, const fs::path& dest_dir) { + if (not fs::exists(path)) { + std::printf( "File does not exist"); + } + if (not fs::exists(dest_dir)) { + std::printf( "Expected the destination directory to exist"); + } + if (not fs::is_directory(dest_dir)) { + std::printf( "Not a directory"); + } + + // Read from the archive + // libzippp::ZipArchive zf(path.c_str()); + // zf.open(libzippp::ZipArchive::READ_ONLY); + // + // std::vector entries = zf.getEntries(); + // std::vector::iterator it; + // for (it = entries.begin(); it != entries.end(); ++it) { + // libzippp::ZipEntry entry = *it; + // + // // Create directory if entry is a dir + // if (entry.isDirectory()) { + // fs::path dir(dest_dir / entry.getName()); + // if (not fs::is_directory(dir.parent_path())) { + // fs::create_directory(dir.parent_path()); + // } + // continue; + // } + // + // // Only process if entry is a file + // if (not entry.isFile()) { + // continue; + // } + // + // fs::path entry_path(dest_dir / entry.getName()); + // std::ofstream ofs(entry_path.string()); + // entry.readContent(ofs); + // ofs.close(); + // } +} + +bool pqry::filesystem::LockFileGuard::lock(const fs::path& lock_file, int pid) { + if (locked_) { + unlock(); + } + + if (lock_file.empty()) { + std::printf(R"(lock_file path is "")"); + } + + fid_ = ::open(lock_file.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0666); + if (fid_ == -1) { + std::printf( "error opening the lock file for writing"); + } + + const int rc = ::flock(fid_, LOCK_EX | LOCK_NB); + + if (fid_ >= 0 and rc < 0) { + ::close(fid_); + locked_ = false; + return locked_; + } + + const std::string pid_str = tfm::format("%d", pid); + const ssize_t n = ::write(fid_, pid_str.c_str(), pid_str.size()); + if (n != pid_str.size()) { + std::printf( "error writing to lock file"); + } + + lock_file_ = lock_file; + locked_ = true; + return locked_; +} + +void pqry::filesystem::LockFileGuard::unlock() { + if (locked_) { + std::printf("Deleting lock file"); + ::close(fid_); + fs::remove(lock_file_); + } +} + +pqry::filesystem::LockFileGuard::~LockFileGuard() { + unlock(); +} + +void pqry::filesystem::wait_for_file(const fs::path& path, unsigned int timeout) { + int i = 0; + while (not fs::exists(path)) { + ::sleep(1); + ++i; + if (timeout > 0 and i > timeout) { + std::printf( "File does not exist (waited for some seconds)"); + } + } +} + +fs::path pqry::filesystem::expand_path(const fs::path& path) { + wordexp_t p = {}; + const int ret = wordexp(path.c_str(), &p, 0); + if (ret != 0) { + wordfree(&p); + std::printf( "Wordexp failed: %d", ret); + } + + fs::path expanded(p.we_wordv[0]); // NOLINT + wordfree(&p); + + return expanded; +} + +/** + * @param path to be added a suffix for a named temporary file + * @return path with a unique suffix + */ +fs::path with_temp_suffix(const fs::path& path) { + return path.parent_path() / + (path.stem().string() + "." + boost::filesystem::unique_path().string() + ".tmp" + path.extension().string()); +} + +pqry::filesystem::NamedTempfile::NamedTempfile( + const boost::filesystem::path& path) : pth_(path), tmp_pth_(with_temp_suffix(path)), renamed_(false) {} + +pqry::filesystem::NamedTempfile::~NamedTempfile() { + if (not renamed_) { + boost::system::error_code ec; // ignore all errors + fs::remove(tmp_pth_, ec); + } +} + +void pqry::filesystem::NamedTempfile::rename() { + if (renamed_) { + std::printf( "The temporary file has been already renamed"); + } + fs::rename(tmp_pth_, pth_); + renamed_ = true; +} diff --git a/cpp/src/library.cpp b/cpp/src/library.cpp new file mode 100644 index 0000000..907f978 --- /dev/null +++ b/cpp/src/library.cpp @@ -0,0 +1,19 @@ +#include "library.h" + +#include + +void hello() { + std::cout << "Hello, World!" << std::endl; +} + +namespace persipubsub { + unsigned int MAX_READER_NUM = 1024; + unsigned int MAX_DB_NUM = 1024; + unsigned long MAX_DB_SIZE_BYTES = 32UL * 1024UL * 1024UL * 1024UL; + + // define all database names here + char DATA_DB[] = "data_db"; // msg_id | data + char PENDING_DB[] = "pending_db"; // msg_id | pending subscriber + char META_DB[] = "meta_db"; // msg_id | metadata + char QUEUE_DB[] = "queue_db"; // queue_pth | all queue data +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/library.h b/cpp/src/library.h new file mode 100644 index 0000000..946bbb8 --- /dev/null +++ b/cpp/src/library.h @@ -0,0 +1,21 @@ +// +// Created by selim on 01.02.19. +// + +#pragma once + +#include +#include +#include + +namespace persipubsub { + extern unsigned int MAX_READER_NUM; + extern unsigned int MAX_DB_NUM; + extern unsigned long MAX_DB_SIZE_BYTES; + + // define all database names here + extern char DATA_DB[]; // msg_id | data + extern char PENDING_DB[]; // msg_id | pending subscriber + extern char META_DB[]; // msg_id | metadata + extern char QUEUE_DB[]; // queue_pth | all queue data +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/publisher.h b/cpp/src/publisher.h new file mode 100644 index 0000000..bbeb108 --- /dev/null +++ b/cpp/src/publisher.h @@ -0,0 +1,52 @@ +// +// Created by selim on 05.02.19. +// + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub{ + namespace publisher{ + /** + * Create Publisher ready to send messages. + */ + class Publisher{ + + public: + Publisher() {}; + + /** + * Initialize. + * + * @param autosync sync after each message or after multiple messages + * @param path to the queue + */ + void init(bool autosync, boost::filesystem::path path); + + /** + * Send message to subscribers. + * + * @param msg to send to all subscribers + */ + void send(char msg); + + /** + * Send messages to subscribers. + * + * @param msgs to send to all subscribers + */ + void send_many(std::vector msgs); + + ~Publisher(); + private: + bool autosync_; + persipubsub::queue::Queue queue_; + + }; // class Publisher + } // namespace publisher +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/queue.cpp b/cpp/src/queue.cpp new file mode 100644 index 0000000..941e9d7 --- /dev/null +++ b/cpp/src/queue.cpp @@ -0,0 +1,404 @@ +// +// Created by selim on 13.02.19. +// + +#include "queue.h" + +// +// Created by selim on 01.02.19. +// + +#include "library.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = boost::filesystem; + +std::map strategyMap; + +persipubsub::queue::Strategy persipubsub::queue::parse_strategy(const std::string &strategy) { + strategyMap["prune_first"] = persipubsub::queue::Strategy::prune_first; + strategyMap["prune_last"] = persipubsub::queue::Strategy::prune_last; + + auto result = strategyMap.find(strategy); + if (result == strategyMap.end()) + throw std::runtime_error("Unknown strategy: " + strategy); + return result->second; + + +} + + +lmdb::env persipubsub::queue::initialize_environment(const fs::path& queue_dir, + const unsigned int& max_reader_num, + const unsigned int& max_db_num, + const unsigned long& max_db_size_bytes) { + + if (not fs::exists(queue_dir)) + throw std::runtime_error("The queue directory does not exist: " + + queue_dir.filename().string()); + + auto env = lmdb::env::create(); + env.set_mapsize(max_db_size_bytes); + env.set_max_dbs(max_db_num); + env.set_max_readers(max_reader_num); + env.open(queue_dir.filename().string().c_str(), 0, 0664); + return env; + +} + + +void persipubsub::queue::prune_dangling_messages_for(persipubsub::queue::Queue * queue, + std::vector &subscriber_ids){ + + auto wtxn = lmdb::txn::begin(queue -> env_); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + // Definition of dangling messages: + // - having no pending subscribers + // - exists longer than timeout allows + std::set msgs_to_delete; + + auto pending_cursor = lmdb::cursor::open(wtxn, pending_dbi); + + lmdb::val pending_key, pending_subscribers_num; + while (pending_cursor.get(pending_key, pending_subscribers_num, MDB_NEXT)) { + if (std::stoi(pending_subscribers_num.data()) == 0) + msgs_to_delete.insert(std::move(pending_key)); + } + + time_t timer; + time(&timer); + const auto timestamp_now = boost::numeric_cast(timer); + + std::set msgs_to_delete_timeout; + + auto meta_cursor = lmdb::cursor::open(wtxn, pending_dbi); + + lmdb::val meta_key, timestamp; + while (meta_cursor.get(meta_key, timestamp, MDB_NEXT)) { + if (timestamp_now - std::stoi(timestamp.data()) > queue -> hwm_.msg_timeout_secs_) + msgs_to_delete_timeout.insert(std::move(meta_key)); + } + + for (auto timeout_msg = msgs_to_delete_timeout.begin(); timeout_msg != msgs_to_delete_timeout.end(); ++timeout_msg){ + lmdb::val new_timeout_msg((*timeout_msg).data()); + msgs_to_delete.insert(std::move(new_timeout_msg)); + } + + for (auto delete_it = msgs_to_delete.begin(); delete_it != msgs_to_delete.end(); ++delete_it){ + pending_dbi.del(wtxn, *delete_it); + meta_dbi.del(wtxn, *delete_it); + data_dbi.del(wtxn, *delete_it); + } + + for (auto sub_it = queue -> subscriber_ids_.begin(); sub_it != queue -> subscriber_ids_.end(); ++sub_it){ + auto sub_dbi = lmdb::dbi::open(wtxn, (*sub_it).c_str()); + for (auto key_it = msgs_to_delete_timeout.begin(); key_it != msgs_to_delete_timeout.end(); ++key_it){ + sub_dbi.del(wtxn, *key_it); + } + } + + // todo replace with this style? + // for (auto sub_id : queue -> subscriber_ids_){ + // auto sub_dbi = lmdb::dbi::open(wtxn, sub_id); + // for (auto key_id : msgs_to_delete_timeout){ + // sub_dbi.del(wtxn, key_id); + // } + // } + + wtxn.commit(); +} + + +void persipubsub::queue::Queue::init(boost::filesystem::path path, lmdb::env env) { + path_ = path; + + if (env) + // todo check if swap works + std::swap(env_, env); + else + env_ = persipubsub::queue::initialize_environment(path, persipubsub::MAX_READER_NUM, persipubsub::MAX_DB_NUM, persipubsub::MAX_DB_SIZE_BYTES); + // todo continue init + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB, MDB_CREATE); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB, MDB_CREATE); + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + wtxn.commit(); + + // todo lookup hwm data + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + HighWaterMark hwm = HighWaterMark(msg_timeout_secs, max_msgs_num, hwm_lmdb_size); + std::swap(hwm_, hwm); + + // todo lookup strategy data + std::string strategy_str = "prune_first"; + + strategy_ = persipubsub::queue::parse_strategy(strategy_str); + + // todo lookup subscriber list + char sub[] = "sub"; + subscriber_ids_.push_back(sub); +} + + +void persipubsub::queue::Queue::put(std::string msg, std::vector &subscriber_ids){ + + // todo uncomment + // vacuum(); + // todo check UTC + time_t timer; + time(&timer); + const auto time = boost::lexical_cast(timer); + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + const std::string tmp = boost::uuids::to_string(uuid); + + // todo change all + std::string key = (time + tmp); + lmdb::val msg_id(key.c_str()); + + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + lmdb::val pending_subs((boost::lexical_cast(subscriber_ids.size())).c_str()); + pending_dbi.put(wtxn, msg_id, pending_subs); + + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + lmdb::val time_val(time); + meta_dbi.put(wtxn, msg_id, time_val); + + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + lmdb::val data_val(msg.c_str()); + data_dbi.put(wtxn, msg_id, data_val); + // todo change? + for (auto it = subscriber_ids.begin(); it != subscriber_ids.end(); ++it) { + auto sub_dbi = lmdb::dbi::open(wtxn, (*it).c_str()); + lmdb::val sub_data(""); + sub_dbi.put(wtxn, msg_id, sub_data); + } + + wtxn.commit(); + +} + +void persipubsub::queue::Queue::put_many_flush_once(std::vector &msgs, + std::vector &subscriber_ids){ + vacuum(); + + // todo check UTC + time_t timer; + time(&timer); + const auto time = boost::lexical_cast(timer); + + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + // todo change? + std::vector sub_dbis; + for (auto it = subscriber_ids.begin(); it != subscriber_ids.end(); ++it) { + auto sub_dbi = lmdb::dbi::open(wtxn, (*it).c_str()); + sub_dbis.push_back(std::move(sub_dbi)); + } + + for (auto msg = msgs.begin(); msg != msgs.end(); ++msg) { + + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + const std::string tmp = boost::uuids::to_string(uuid); + std::string key = (time + tmp); + lmdb::val msg_id(key.c_str()); + + lmdb::val pending_subs((boost::lexical_cast(subscriber_ids.size())).c_str()); + pending_dbi.put(wtxn, msg_id, pending_subs); + lmdb::val time_val(time); + meta_dbi.put(wtxn, msg_id, time_val); + lmdb::val data_val(msg -> c_str()); + data_dbi.put(wtxn, msg_id, data_val); + + // todo change? + for (auto sub_dbi_it = sub_dbis.begin(); sub_dbi_it != sub_dbis.end(); ++sub_dbi_it) { + lmdb::val sub_data(""); + sub_dbi_it -> put(wtxn, msg_id, sub_data); + } + } + + wtxn.commit(); + +} + +// todo msg as pointer or address? +void persipubsub::queue::Queue::front(std::string identifier, std::string& msg){ + + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto sub_dbi = lmdb::dbi::open(rtxn, identifier.c_str()); + auto cursor = lmdb::cursor::open(rtxn, sub_dbi); + lmdb::val key, value; + if (cursor.get(key, value, MDB_FIRST)) { + auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); + auto data_cursor = lmdb::cursor::open(rtxn, data_dbi); + lmdb::val data_value; + + bool found = data_cursor.get(key, data_value, MDB_FIRST); + if (found){ + std::string tmp_str = data_value.data(); + std::swap(msg, tmp_str); + } + else{ + cursor.close(); + rtxn.abort(); + throw std::runtime_error("Data not found"); + } + } + cursor.close(); + rtxn.abort(); +} + +void persipubsub::queue::Queue::pop(std::string identifier){ + auto wtxn = lmdb::txn::begin(env_); + auto sub_dbi = lmdb::dbi::open(wtxn, identifier.c_str()); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + + auto cursor = lmdb::cursor::open(wtxn, sub_dbi); + lmdb::val key, value; + + if (cursor.get(key, value, MDB_FIRST)) { + sub_dbi.del(wtxn, key); + + lmdb::val pending_value; + pending_dbi.get(wtxn, key, pending_value); + auto pending_num = boost::numeric_cast(std::stoi(pending_value.data())); + pending_num--; + lmdb::val pending_new_value((boost::lexical_cast(pending_num)).c_str()); + pending_dbi.put(wtxn, key, pending_new_value); + } + else + throw std::runtime_error("No message to pop"); +} + + +void persipubsub::queue::Queue::prune_dangling_messages(){ + persipubsub::queue::prune_dangling_messages_for(this, subscriber_ids_); +} + + +// todo size_t instead of long? +unsigned long persipubsub::queue::Queue::check_current_lmdb_size(){ + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); + + MDB_stat data_stat = data_dbi.stat(rtxn); + + // todo check overflow, boost::numeric_cast to unsigned long? + unsigned long lmdb_size_bytes = data_stat.ms_psize * (data_stat.ms_branch_pages + data_stat.ms_leaf_pages + data_stat.ms_overflow_pages); + + rtxn.abort(); + + return lmdb_size_bytes; + +} + + +unsigned int persipubsub::queue::Queue::count_msgs(){ + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto meta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); + + auto msgs_num = boost::numeric_cast(meta_dbi.stat(rtxn).ms_entries); + rtxn.abort(); + + return msgs_num; +} + + +void persipubsub::queue::Queue::vacuum(){ + prune_dangling_messages(); + + unsigned int msgs_num = count_msgs(); + + if (msgs_num >= hwm_.max_msgs_num_) + prune_messages(); + unsigned long lmdb_size_bytes = check_current_lmdb_size(); + if (lmdb_size_bytes >= hwm_.hwm_lmdb_size_bytes_) + prune_messages(); +} + + +void persipubsub::queue::Queue::prune_messages(){ + + std::set messages_to_delete; + + auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); + auto rmeta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); + auto entries = boost::numeric_cast(rmeta_dbi.stat(rtxn).ms_entries); + + auto cursor = lmdb::cursor::open(rtxn, rmeta_dbi); + lmdb::val key, value; + + if (strategy_ == persipubsub::queue::Strategy::prune_first){ + cursor.get(key, value, MDB_FIRST); + for (unsigned int counter = 0; counter<=(entries/2); counter++){ + messages_to_delete.insert(std::move(key)); + cursor.get(key, value, MDB_NEXT); + } + } + else if (strategy_ == persipubsub::queue::Strategy::prune_last){ + cursor.get(key, value, MDB_LAST); + for (unsigned int counter = 0; counter<=(entries/2); counter++){ + messages_to_delete.insert(std::move(key)); + cursor.get(key, value, MDB_PREV); + } + } + else { + cursor.close(); + rtxn.abort(); + throw std::runtime_error("Pruning strategy not set."); + } + + cursor.close(); + rtxn.abort(); + + auto wtxn = lmdb::txn::begin(env_); + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); + + std::vector dbis; + dbis.push_back(std::move(pending_dbi)); + dbis.push_back(std::move(meta_dbi)); + dbis.push_back(std::move(data_dbi)); + + for (auto sub_it = subscriber_ids_.begin(); sub_it != subscriber_ids_.end(); ++sub_it){ + auto sub_dbi = lmdb::dbi::open(wtxn, (*sub_it).c_str()); + dbis.push_back(std::move(sub_dbi)); + } + + for (auto msg_it = messages_to_delete.begin(); msg_it != messages_to_delete.end(); ++msg_it) { + for (auto db_it = dbis.begin(); db_it != dbis.end(); ++db_it){ + db_it -> del(wtxn, *msg_it); + } + } + + wtxn.commit(); +} + +persipubsub::queue::Queue::~Queue() {} + diff --git a/cpp/src/queue.h b/cpp/src/queue.h new file mode 100644 index 0000000..1b76f13 --- /dev/null +++ b/cpp/src/queue.h @@ -0,0 +1,172 @@ +// +// Created by selim on 01.02.19. +// + +#pragma once + +#include +#include +#include + +namespace persipubsub { + namespace queue { + + class Queue; + + /** + * Store possible strategies. + */ + enum Strategy {prune_first, prune_last}; + + /** + * Parse overflow strategy. + * + * @param strategy Strategy stored in config + * @return set overflow strategy + */ + Strategy parse_strategy(const std::string& strategy); + + /** + * Store high water mark limits. + */ + struct HighWaterMark{ + /** + * Initialize. + * + * @param msg_timeout_secs time after which msg is classified as dangling msg (secs) + * @param max_masgs_num maximal amount of msg + * @param hwm_lmdb_size_bytes high water mark for total size of lmdb (bytes) + */ + unsigned int msg_timeout_secs_; + unsigned int max_msgs_num_; + unsigned long hwm_lmdb_size_bytes_; + + HighWaterMark() : msg_timeout_secs_(), max_msgs_num_(), hwm_lmdb_size_bytes_() {} + + HighWaterMark(unsigned int msg_timeout_secs, unsigned int max_msgs_num, + unsigned long hwm_lmdb_size) : msg_timeout_secs_(msg_timeout_secs), max_msgs_num_(max_msgs_num), hwm_lmdb_size_bytes_(hwm_lmdb_size) {} + }; + + /** + * Initialize the queue; the queue directory is assumed to exist. + * + * @param queue_dir where the queue is stored + * @param max_reader_num maximal number of reader + * @param max_db_num maximal number of databases + * @param max_db_size_bytes maximal size of database (bytes) + * @return Load or if needed create LMDB queue from directory + */ + lmdb::env initialize_environment(const boost::filesystem::path& queue_dir, const unsigned int& max_reader_num, const unsigned int& max_db_num, const unsigned long& max_db_size_bytes); + + /** + * Prune all dangling messages for subscribers of a queue from lmdb. + * + * @param queue of which dangling messages should be pruned + * @param subscriber_ids subscribers of which dangling messages should be pruned + */ + void prune_dangling_messages_for(persipubsub::queue::Queue * queue, std::vector& subscriber_ids); + + /** + * Queue messages persistently from many publishers for many subscribers. + */ + class Queue{ + + public: + /** + * Initialize class object. + */ + Queue() : path_(), + env_(nullptr), hwm_(), strategy_(), + subscriber_ids_() {} + + /** + * Initialize the queue. + * + * @param config_pth path to the JSON config file + * @param queue_dir where the queue is stored + * @param max_reader_num maximal number of reader + * @param max_db_num maximal number of databases + * @param max_db_size_bytes maximal size of database (bytes) + */ + void init(boost::filesystem::path path, lmdb::env env = nullptr); + + /** + * Put message to lmdb queue. + * + * @param msg message send from publisher to subscribers + * @param subscriber_ids List of subscribers + */ + void put(std::string msg, std::vector& subscriber_ids); + + /** + * Put many message to lmdb queue. + * + * @param msgs messages send from publisher to subscribers + * @param subscriber_ids List of subscribers + */ + void put_many_flush_once(std::vector& msgs, std::vector& subscriber_ids); + + /** + * Peek at next message in lmdb queue. + * + * Load from LMDB queue into memory and process msg afterwards. + * @param identifier Subscriber ID + * @param message peek on the message + */ + void front(std::string identifier, std::string& msg); + + /** + * Remove msg from the subscriber's queue and reduce pending subscribers. + * + * @param identifier Subscriber ID + */ + void pop(std::string identifier); + + /** + * Prune dangling messages in the queue. + */ + void prune_dangling_messages(); + + /** + * Check current lmdb size in bytes. + * + * Check size of data database by approximating size with multiplying page size with number of pages. + * @return data database size in bytes + */ + unsigned long check_current_lmdb_size(); + + /** + * Count number of messages in database. + * + * Count number of messages stored in meta database. + * + * @return number of messages in database + */ + unsigned int count_msgs(); + + /** + * Clean database when needed. + */ + void vacuum(); + + /** + * Prune one half of the messages stored. + * + * Depending on the strategy the first or the last will be deleted. + */ + void prune_messages(); + + boost::filesystem::path path_; + lmdb::env env_; + HighWaterMark hwm_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + ~Queue(); + + private: + + }; // class Queue + + } // namespace queue +} // namespace persipubsub diff --git a/cpp/src/subscriber.h b/cpp/src/subscriber.h new file mode 100644 index 0000000..636ea9f --- /dev/null +++ b/cpp/src/subscriber.h @@ -0,0 +1,70 @@ +// +// Created by selim on 05.02.19. +// + +#pragma once + +#include +#include +#include + +#include "queue.h" + +namespace persipubsub{ + namespace subscriber{ + /** + * Create Subscriber ready to receive messages. + */ + class Subscriber{ + + public: + /** + * Initialize class object. + */ + Subscriber() {}; + + /** + * Initialize. + * + * @param identifier unique subscriber id + * @param path to the queue + */ + void init(std::vector identifier, boost::filesystem::path path); + + /** + * Receive messages from the publisher. + * + * @param timeout time waiting for a message. If none arrived until the timeout then None will be returned. (secs) + * @param retries number of tries to check if a msg arrived in the queue + * @return received message + */ + char receive(int timeout, int retries); + + /** + * Pops all messages until the most recent one and receive the latest. + * Used in the case that a particular subscriber cares only about the very + * last message and other subscribers care about all the messages in the + * queue. + * For another use case, when you only want to store the latest message + * and all subscribers are interested only in the latest, then use + * high water mark max_msgs_num = 1. + * + * @param timeout time waiting for a message. If none arrived until the timeout then None will be returned. (secs) + * @param retries number of tries to check if a msg arrived in the queue + * @return received message + */ + char receive_to_top(int timeout, int retries); + + ~Subscriber(); + private: + std::vector identifier_; + persipubsub::queue::Queue queue_; + + /** + * Pop a message from the subscriber's lmdb. + */ + void pop(); + + }; // class Subscriber + } // namespace subscriber +} // namespace persipubsub \ No newline at end of file diff --git a/cpp/test/CMakeLists.txt b/cpp/test/CMakeLists.txt new file mode 100644 index 0000000..d1d1791 --- /dev/null +++ b/cpp/test/CMakeLists.txt @@ -0,0 +1,13 @@ +find_package (Boost COMPONENTS system filesystem unit_test_framework REQUIRED) +include_directories (${TEST_SOURCE_DIR}/src + ${Boost_INCLUDE_DIRS} + ) + +add_definitions (-DBOOST_TEST_DYN_LINK) +add_executable (Test test_queue.cpp) +target_link_libraries (Test + persipubsub + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY} + ) \ No newline at end of file diff --git a/cpp/test/test_queue.cpp b/cpp/test/test_queue.cpp new file mode 100644 index 0000000..75000cb --- /dev/null +++ b/cpp/test/test_queue.cpp @@ -0,0 +1,106 @@ +// Copyright (c) 2019 Parquery AG. All rights reserved. +// Created by Selim Naji (selim.naji@parquery.com and +// Marko Ristin (marko@parquery.com) on 2019-02-15 + +#define BOOST_TEST_DYN_LINK +#define BOOST_TEST_MODULE TestQueue +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + +#include "../src/queue.h" + +namespace fs = boost::filesystem; + +class TmpDir{ +public: + TmpDir(){ + path_ = fs::unique_path(); + fs::create_directories(path_); + } + + ~TmpDir(){ + fs::remove_all(path_); + } + fs::path path_; +private: + +}; // class TmpDir + +BOOST_AUTO_TEST_CASE(test_initialize_environment){ + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, 1024, 32UL * 1024UL * 1024UL * 1024UL); + + auto rtxn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + auto dbi = lmdb::dbi::open(rtxn, nullptr); + + MDB_envinfo info; + lmdb::env_info(env, &info); + + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(info.me_last_txnid)); + BOOST_REQUIRE_EQUAL("1", boost::lexical_cast(info.me_last_pgno)); + BOOST_REQUIRE_EQUAL("1024", boost::lexical_cast(info.me_maxreaders)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(info.me_mapaddr)); + BOOST_REQUIRE_EQUAL("34359738368", boost::lexical_cast(info.me_mapsize)); + BOOST_REQUIRE_EQUAL("1", boost::lexical_cast(info.me_numreaders)); + + MDB_stat stat = dbi.stat(rtxn); + + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_branch_pages)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_entries)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_depth)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_leaf_pages)); + BOOST_REQUIRE_EQUAL("0", boost::lexical_cast(stat.ms_overflow_pages)); + BOOST_REQUIRE_EQUAL("4096", boost::lexical_cast(stat.ms_psize)); + + rtxn.abort(); +} + +BOOST_AUTO_TEST_CASE(test_put_to_single_subscriber) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string name = "sub"; + + auto wtxn = lmdb::txn::begin(env); + auto dbi = lmdb::dbi::open(wtxn, name.c_str(), MDB_CREATE); + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(name); + queue.put(msg, subs); + + std::string result; + queue.front(name, result); + + BOOST_REQUIRE_EQUAL(msg, result); + +} From 87ae7a123e118562be0a0e04a746f5df985032bd Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 20 Feb 2019 14:55:42 +0100 Subject: [PATCH 34/35] get all queue data in one go and more changes --- py/CHANGELOG.rst | 3 ++ py/persipubsub/__init__.py | 59 ++++++++++++++++++++++++++++---- py/persipubsub/control.py | 19 +++------- py/persipubsub/queue.py | 41 ++++------------------ py/setup.py | 2 +- py/tests/__init__.py | 30 +++++++++++++--- py/tests/component_publisher.py | 10 ++---- py/tests/component_subscriber.py | 10 ------ py/tests/test_live.py | 17 +++------ 9 files changed, 98 insertions(+), 93 deletions(-) create mode 100644 py/CHANGELOG.rst diff --git a/py/CHANGELOG.rst b/py/CHANGELOG.rst new file mode 100644 index 0000000..43bb01b --- /dev/null +++ b/py/CHANGELOG.rst @@ -0,0 +1,3 @@ +1.0.0 +===== +* Initial version \ No newline at end of file diff --git a/py/persipubsub/__init__.py b/py/persipubsub/__init__.py index 1173490..a57b115 100644 --- a/py/persipubsub/__init__.py +++ b/py/persipubsub/__init__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """Set default values of persipubsub and offers encoding tools.""" -from typing import Optional +from typing import List, Optional import lmdb @@ -71,18 +71,63 @@ def bytes_to_int(array_of_bytes: bytes) -> int: return int.from_bytes(bytes=array_of_bytes, byteorder=BYTES_ORDER) -def lookup_queue_data(key: bytes, env: lmdb.Environment) -> Optional[bytes]: +class QueueData: + """Hold queue data.""" + + def __init__(self, msg_timeout_secs: int, max_msgs_num: int, + hwm_db_size_bytes: int, strategy: str, + subscriber_ids: List[str]) -> None: + """ + Initialize. + + :param msg_timeout_secs: time after which msg is classified as dangling + msg (secs) + :param max_msgs_num: maximal amount of msg + :param hwm_db_size_bytes: high water mark for total size of LMDB (bytes) + :param strategy: pruning strategy + :param subscriber_ids: List of subscribers + """ + # pylint: disable=too-many-arguments + self.msg_timeout_secs = msg_timeout_secs + self.max_msgs_num = max_msgs_num + self.hwm_db_size_bytes = hwm_db_size_bytes + self.strategy = strategy + self.subscriber_ids = subscriber_ids + + +def lookup_queue_data(env: lmdb.Environment) -> QueueData: """ - Lookup set value for a given key in named database 'queue_db'. + Lookup set value in named database 'queue_db'. Value stored in 'queue_db' are high water mark values and pruning strategy. - :param key: for lookup :param env: environment that stores queue data :return: lookup result """ with env.begin(write=False) as txn: queue_db = env.open_db(key=QUEUE_DB, txn=txn, create=False) - data = txn.get(key=key, db=queue_db) - - return data # type: ignore + msg_timeout_secs_bytes = txn.get(key=MSG_TIMEOUT_SECS_KEY, db=queue_db) + max_msgs_num_bytes = txn.get(key=MAX_MSGS_NUM_KEY, db=queue_db) + hwm_db_size_bytes = txn.get(key=HWM_DB_SIZE_BYTES_KEY, db=queue_db) + strategy_bytes = txn.get(key=STRATEGY_KEY, db=queue_db) + subscriber_ids_bytes = txn.get(key=SUBSCRIBER_IDS_KEY, db=queue_db) + + msg_timeout_secs = bytes_to_int(msg_timeout_secs_bytes) + max_msgs_num = bytes_to_int(max_msgs_num_bytes) + hwm_db_size = bytes_to_int(hwm_db_size_bytes) + + strategy = bytes_to_str(encoded_str=strategy_bytes) + + if subscriber_ids_bytes is None: + subscriber_ids = [] # type: List[str] + else: + subscriber_ids = bytes_to_str( + encoded_str=subscriber_ids_bytes).split(' ') + + queue_data = QueueData( + msg_timeout_secs=msg_timeout_secs, + max_msgs_num=max_msgs_num, + hwm_db_size_bytes=hwm_db_size, + strategy=strategy, + subscriber_ids=subscriber_ids) + return queue_data diff --git a/py/persipubsub/control.py b/py/persipubsub/control.py index 83a3927..ecb9ea3 100644 --- a/py/persipubsub/control.py +++ b/py/persipubsub/control.py @@ -109,21 +109,10 @@ def is_initialized(self) -> bool: :return: is initialized when all values for the given keys are set """ - keys = [ - persipubsub.MSG_TIMEOUT_SECS_KEY, persipubsub.MAX_MSGS_NUM_KEY, - persipubsub.HWM_DB_SIZE_BYTES_KEY, persipubsub.STRATEGY_KEY, - persipubsub.SUBSCRIBER_IDS_KEY - ] # type: List[bytes] - - for key in keys: - - try: - value = persipubsub.lookup_queue_data(key=key, env=self.env) - except lmdb.NotFoundError: - return False - - if value is None: - return False + try: + _ = persipubsub.lookup_queue_data(env=self.env) + except lmdb.NotFoundError: + return False return True diff --git a/py/persipubsub/queue.py b/py/persipubsub/queue.py index 68ccb3d..9d9234e 100644 --- a/py/persipubsub/queue.py +++ b/py/persipubsub/queue.py @@ -204,45 +204,16 @@ def init(self, _ = self.env.open_db(key=persipubsub.META_DB, txn=txn, create=True) _ = self.env.open_db(key=persipubsub.QUEUE_DB, txn=txn, create=True) - msg_timeout_secs_bytes = persipubsub.lookup_queue_data( - key=persipubsub.MSG_TIMEOUT_SECS_KEY, env=self.env) - msg_timeout_secs = msg_timeout_secs_bytes \ - if msg_timeout_secs_bytes is None else persipubsub.bytes_to_int( - msg_timeout_secs_bytes) # type: Optional[int] - - max_msgs_num_bytes = persipubsub.lookup_queue_data( - key=persipubsub.MAX_MSGS_NUM_KEY, env=self.env) - max_msgs_num = max_msgs_num_bytes \ - if max_msgs_num_bytes is None else persipubsub.bytes_to_int( - max_msgs_num_bytes) # type: Optional[int] - - hwm_lmdb_size_bytes = persipubsub.lookup_queue_data( - key=persipubsub.HWM_DB_SIZE_BYTES_KEY, env=self.env) - hwm_lmdb_size = hwm_lmdb_size_bytes \ - if hwm_lmdb_size_bytes is None else persipubsub.bytes_to_int( - hwm_lmdb_size_bytes) # type: Optional[int] + queue_data = persipubsub.lookup_queue_data(env=self.env) self.hwm = HighWaterMark( - msg_timeout_secs=msg_timeout_secs, - max_msgs_num=max_msgs_num, - hwm_lmdb_size_bytes=hwm_lmdb_size) + msg_timeout_secs=queue_data.msg_timeout_secs, + max_msgs_num=queue_data.max_msgs_num, + hwm_lmdb_size_bytes=queue_data.hwm_db_size_bytes) - strategy = persipubsub.lookup_queue_data( - key=persipubsub.STRATEGY_KEY, env=self.env) + self.strategy = _parse_strategy(strategy=queue_data.strategy) - strategy_decoded = "" if strategy is None else persipubsub.bytes_to_str( - encoded_str=strategy) - - self.strategy = _parse_strategy(strategy=strategy_decoded) - - subscriber_list = persipubsub.lookup_queue_data( - key=persipubsub.SUBSCRIBER_IDS_KEY, env=self.env) - - if subscriber_list is None: - self.subscriber_ids = [] - else: - self.subscriber_ids = persipubsub.bytes_to_str( - encoded_str=subscriber_list).split(' ') + self.subscriber_ids = queue_data.subscriber_ids def __enter__(self) -> '_Queue': """Enter the context and give the queue prepared in the constructor.""" diff --git a/py/setup.py b/py/setup.py index 6284d85..0f916ad 100644 --- a/py/setup.py +++ b/py/setup.py @@ -34,7 +34,7 @@ keywords= 'persistent publisher subscriber lmdb MQ message queue thread-safe process-safe', packages=find_packages( - exclude=['tests', 'examples', 'messages', 'obsolete']), + exclude=['tests', 'obsolete', 'messages', 'example']), install_requires=[ # yapf: disable 'typing-extensions>=3.7.2', diff --git a/py/tests/__init__.py b/py/tests/__init__.py index 9744cf7..8122f5d 100644 --- a/py/tests/__init__.py +++ b/py/tests/__init__.py @@ -40,13 +40,33 @@ def test_get_data(self) -> None: with env.begin(write=True) as txn: queue_db = env.open_db(key=QUEUE_DB, txn=txn) txn.put( - key="key".encode(ENCODING), - value="value".encode(ENCODING), + key=persipubsub.MSG_TIMEOUT_SECS_KEY, + value=persipubsub.int_to_bytes(500), db=queue_db) + txn.put( + key=persipubsub.MAX_MSGS_NUM_KEY, + value=persipubsub.int_to_bytes(1000), + db=queue_db) + txn.put( + key=persipubsub.HWM_DB_SIZE_BYTES_KEY, + value=persipubsub.int_to_bytes(1024**3), + db=queue_db) + txn.put( + key=persipubsub.STRATEGY_KEY, + value=persipubsub.str_to_bytes("prune_last"), + db=queue_db) + txn.put( + key=persipubsub.SUBSCRIBER_IDS_KEY, + value=persipubsub.str_to_bytes(""), + db=queue_db) + + queue_data = persipubsub.lookup_queue_data(env=env) - data = persipubsub.lookup_queue_data( - key="key".encode(ENCODING), env=env) - self.assertEqual(b'value', data) + self.assertEqual(500, queue_data.msg_timeout_secs) + self.assertEqual(1000, queue_data.max_msgs_num) + self.assertEqual(1024**3, queue_data.hwm_db_size_bytes) + self.assertEqual("prune_last", queue_data.strategy) + self.assertEqual([], queue_data.subscriber_ids) if __name__ == '__main__': diff --git a/py/tests/component_publisher.py b/py/tests/component_publisher.py index 43b27b7..09317db 100644 --- a/py/tests/component_publisher.py +++ b/py/tests/component_publisher.py @@ -2,28 +2,22 @@ """Publisher component for live test.""" import pathlib -import time import persipubsub.environment # pylint: disable=missing-docstring -def send_thread(env: persipubsub.environment.Environment, - num_msg: int, - sleep_time: float = 0) -> None: +def send_thread(env: persipubsub.environment.Environment, num_msg: int) -> None: pub = env.new_publisher() for _ in range(num_msg): pub.send(msg="hello subscriber".encode('utf-8')) - time.sleep(sleep_time) -def send_process(path: pathlib.Path, num_msg: int, - sleep_time: float = 0) -> None: +def send_process(path: pathlib.Path, num_msg: int) -> None: env = persipubsub.environment.Environment(path=path) pub = env.new_publisher() for _ in range(num_msg): pub.send(msg="hello subscriber".encode('utf-8')) - time.sleep(sleep_time) diff --git a/py/tests/component_subscriber.py b/py/tests/component_subscriber.py index 64b989e..5685b92 100644 --- a/py/tests/component_subscriber.py +++ b/py/tests/component_subscriber.py @@ -16,7 +16,6 @@ def receive_thread(path: pathlib.Path, num_msg: int, timeout: int = 2, retries: int = 10, - sleep_time: float = 0, method_timeout: int = 10) -> None: sub = env.new_subscriber(identifier=identifier) @@ -32,10 +31,6 @@ def receive_thread(path: pathlib.Path, with sub.receive(timeout=timeout, retries=retries) as msg: if msg is not None: received_msg += 1 - time.sleep(sleep_time) - else: - time.sleep(sleep_time) - continue result = path / identifier result.write_text('pass') @@ -47,7 +42,6 @@ def receive_process(path: pathlib.Path, num_msg: int, timeout: int = 2, retries: int = 10, - sleep_time: float = 0, method_timeout: int = 60) -> None: env = persipubsub.environment.Environment(path=path) sub = env.new_subscriber(identifier=identifier) @@ -64,10 +58,6 @@ def receive_process(path: pathlib.Path, with sub.receive(timeout=timeout, retries=retries) as msg: if msg is not None: received_msg += 1 - time.sleep(sleep_time) - else: - time.sleep(sleep_time) - continue result = path / identifier result.write_text('pass') diff --git a/py/tests/test_live.py b/py/tests/test_live.py index 23462c3..e697658 100644 --- a/py/tests/test_live.py +++ b/py/tests/test_live.py @@ -55,13 +55,10 @@ def subscriber_receive_second(sub: persipubsub.subscriber.Subscriber) -> None: result.write_text('fail') -def receive( - sub: persipubsub.subscriber.Subscriber, - num_msg: int, - timeout: int = 2, - # TODO: remove - sleep_time: float = 0, - method_timeout: int = 60) -> None: +def receive(sub: persipubsub.subscriber.Subscriber, + num_msg: int, + timeout: int = 2, + method_timeout: int = 60) -> None: received_msg = 0 start = time.time() @@ -74,7 +71,6 @@ def receive( with sub.receive(timeout=timeout) as msg: if msg is not None: received_msg += 1 - time.sleep(sleep_time) assert isinstance(sub.queue, persipubsub.queue._Queue) # pylint: disable=protected-access assert isinstance(sub.queue.env, lmdb.Environment) @@ -84,12 +80,9 @@ def receive( result.write_text('pass') -def send(pub: persipubsub.publisher.Publisher, - num_msg: int, - sleep_time: float = 0) -> None: +def send(pub: persipubsub.publisher.Publisher, num_msg: int) -> None: for _ in range(num_msg): pub.send(msg="hello subscriber".encode('utf-8')) - time.sleep(sleep_time) class TestLive(unittest.TestCase): From 59e88ea70a0e99c01a612690fdaf3cb17293cf6f Mon Sep 17 00:00:00 2001 From: slimeth Date: Wed, 20 Feb 2019 14:56:28 +0100 Subject: [PATCH 35/35] cpp in work commit --- cpp/src/library.cpp | 43 ++++++++- cpp/src/library.h | 29 +++++++ cpp/src/queue.cpp | 106 ++++++++++++----------- cpp/src/queue.h | 44 +++++----- cpp/test/test_queue.cpp | 187 ++++++++++++++++++++++++++++++++++++++-- 5 files changed, 329 insertions(+), 80 deletions(-) diff --git a/cpp/src/library.cpp b/cpp/src/library.cpp index 907f978..796924e 100644 --- a/cpp/src/library.cpp +++ b/cpp/src/library.cpp @@ -2,10 +2,6 @@ #include -void hello() { - std::cout << "Hello, World!" << std::endl; -} - namespace persipubsub { unsigned int MAX_READER_NUM = 1024; unsigned int MAX_DB_NUM = 1024; @@ -16,4 +12,43 @@ namespace persipubsub { char PENDING_DB[] = "pending_db"; // msg_id | pending subscriber char META_DB[] = "meta_db"; // msg_id | metadata char QUEUE_DB[] = "queue_db"; // queue_pth | all queue data + + char HWM_DB_SIZE_BYTES_KEY[] = "hwm_db_size_bytes"; + char MAX_MSGS_NUM_KEY[] = "max_msgs_num"; + char MSG_TIMEOUT_SECS_KEY[] = "msg_timeout_secs"; + char STRATEGY_KEY[] = "strategy"; + char SUBSCRIBER_IDS_KEY[] = "subscriber_ids"; + + QueueData lookup_queue_data(const lmdb::env& env) { + auto rtxn = lmdb::txn::begin(env, nullptr, MDB_RDONLY); + auto queue_dbi = lmdb::dbi::open(rtxn, QUEUE_DB); + + lmdb::val size_key(HWM_DB_SIZE_BYTES_KEY), size; + + queue_dbi.get(rtxn, size_key, size); + unsigned long hwm_db_size_bytes = std::stoul(size.data()); + + + lmdb::val msgs_key(MAX_MSGS_NUM_KEY), msgs_num; + queue_dbi.get(rtxn, msgs_key, msgs_num); + unsigned int max_msgs_num = std::stoi(msgs_num.data()); + + lmdb::val timeout_key(MSG_TIMEOUT_SECS_KEY), timeout; + queue_dbi.get(rtxn, timeout_key, timeout); + unsigned int msg_timeout_secs = std::stoi(timeout.data()); + + lmdb::val strategy_key(STRATEGY_KEY), strategy_val; + queue_dbi.get(rtxn, strategy_key, strategy_val); + std::string strategy_str = strategy_val.data(); + queue::Strategy strategy = queue::parse_strategy(strategy_str); + + lmdb::val subscriber_ids_key(SUBSCRIBER_IDS_KEY), subscriber_ids_val; + queue_dbi.get(rtxn, subscriber_ids_key, subscriber_ids_val); + std::string subscriber_ids_str = subscriber_ids_val.data(); + std::vector subscriber_ids; + + boost::split(subscriber_ids, subscriber_ids_str, boost::is_any_of(" "), boost::token_compress_on); + + return QueueData(msg_timeout_secs, max_msgs_num, hwm_db_size_bytes, strategy, subscriber_ids); + } } // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/library.h b/cpp/src/library.h index 946bbb8..c032a9d 100644 --- a/cpp/src/library.h +++ b/cpp/src/library.h @@ -6,9 +6,14 @@ #include #include +#include // Include boost::for is_any_of +#include // Include for boost::split #include +#include "queue.h" namespace persipubsub { + // TODO read when it is appropriate to use extern + extern unsigned int MAX_READER_NUM; extern unsigned int MAX_DB_NUM; extern unsigned long MAX_DB_SIZE_BYTES; @@ -18,4 +23,28 @@ namespace persipubsub { extern char PENDING_DB[]; // msg_id | pending subscriber extern char META_DB[]; // msg_id | metadata extern char QUEUE_DB[]; // queue_pth | all queue data + + extern char HWM_DB_SIZE_BYTES_KEY[]; + extern char MAX_MSGS_NUM_KEY[]; + extern char MSG_TIMEOUT_SECS_KEY[]; + extern char STRATEGY_KEY[]; + extern char SUBSCRIBER_IDS_KEY[]; + + struct QueueData{ + unsigned int msg_timeout_secs_; + unsigned int max_msgs_num_; + unsigned long hwm_db_size_bytes_; + persipubsub::queue::Strategy strategy_; + std::vector subscriber_ids_; + + QueueData(unsigned int msg_timeout_secs, unsigned int max_msgs_num, + unsigned long hwm_db_size_bytes, + persipubsub::queue::Strategy strategy, + std::vector subscriber_ids) : + msg_timeout_secs_(msg_timeout_secs), + max_msgs_num_(max_msgs_num), + hwm_db_size_bytes_(hwm_db_size_bytes), strategy_(strategy), + subscriber_ids_(std::move(subscriber_ids)) {}; + }; + QueueData lookup_queue_data(const lmdb::env& env); } // namespace persipubsub \ No newline at end of file diff --git a/cpp/src/queue.cpp b/cpp/src/queue.cpp index 941e9d7..5119528 100644 --- a/cpp/src/queue.cpp +++ b/cpp/src/queue.cpp @@ -2,6 +2,8 @@ // Created by selim on 13.02.19. // +// TODO nicer headers in files + #include "queue.h" // @@ -27,7 +29,7 @@ namespace fs = boost::filesystem; std::map strategyMap; persipubsub::queue::Strategy persipubsub::queue::parse_strategy(const std::string &strategy) { - strategyMap["prune_first"] = persipubsub::queue::Strategy::prune_first; + strategyMap["prune_first"] = persipubsub::queue::Strategy::prune_first; // TODO use consts for strings like prune_first - please google how (const staticexpr string????) strategyMap["prune_last"] = persipubsub::queue::Strategy::prune_last; auto result = strategyMap.find(strategy); @@ -40,9 +42,9 @@ persipubsub::queue::Strategy persipubsub::queue::parse_strategy(const std::strin lmdb::env persipubsub::queue::initialize_environment(const fs::path& queue_dir, - const unsigned int& max_reader_num, - const unsigned int& max_db_num, - const unsigned long& max_db_size_bytes) { + const unsigned int max_reader_num, + const unsigned int max_db_num, + const unsigned long max_db_size_bytes) { if (not fs::exists(queue_dir)) throw std::runtime_error("The queue directory does not exist: " + @@ -58,10 +60,10 @@ lmdb::env persipubsub::queue::initialize_environment(const fs::path& queue_dir, } -void persipubsub::queue::prune_dangling_messages_for(persipubsub::queue::Queue * queue, - std::vector &subscriber_ids){ +void persipubsub::queue::prune_dangling_messages_for(const persipubsub::queue::Queue& queue, + const std::vector &subscriber_ids){ - auto wtxn = lmdb::txn::begin(queue -> env_); + auto wtxn = lmdb::txn::begin(queue.env_); auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB); auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB); @@ -78,6 +80,7 @@ void persipubsub::queue::prune_dangling_messages_for(persipubsub::queue::Queue * if (std::stoi(pending_subscribers_num.data()) == 0) msgs_to_delete.insert(std::move(pending_key)); } + pending_cursor.close(); time_t timer; time(&timer); @@ -89,10 +92,10 @@ void persipubsub::queue::prune_dangling_messages_for(persipubsub::queue::Queue * lmdb::val meta_key, timestamp; while (meta_cursor.get(meta_key, timestamp, MDB_NEXT)) { - if (timestamp_now - std::stoi(timestamp.data()) > queue -> hwm_.msg_timeout_secs_) + if (timestamp_now - std::stoi(timestamp.data()) > queue.hwm_.msg_timeout_secs_) msgs_to_delete_timeout.insert(std::move(meta_key)); } - + meta_cursor.close(); for (auto timeout_msg = msgs_to_delete_timeout.begin(); timeout_msg != msgs_to_delete_timeout.end(); ++timeout_msg){ lmdb::val new_timeout_msg((*timeout_msg).data()); msgs_to_delete.insert(std::move(new_timeout_msg)); @@ -104,15 +107,15 @@ void persipubsub::queue::prune_dangling_messages_for(persipubsub::queue::Queue * data_dbi.del(wtxn, *delete_it); } - for (auto sub_it = queue -> subscriber_ids_.begin(); sub_it != queue -> subscriber_ids_.end(); ++sub_it){ + for (auto sub_it = queue.subscriber_ids_.begin(); sub_it != queue.subscriber_ids_.end(); ++sub_it){ auto sub_dbi = lmdb::dbi::open(wtxn, (*sub_it).c_str()); for (auto key_it = msgs_to_delete_timeout.begin(); key_it != msgs_to_delete_timeout.end(); ++key_it){ sub_dbi.del(wtxn, *key_it); } } - // todo replace with this style? - // for (auto sub_id : queue -> subscriber_ids_){ + // todo replace with this style? YES + // for (const auto& sub_id : queue -> subscriber_ids_){ // auto sub_dbi = lmdb::dbi::open(wtxn, sub_id); // for (auto key_id : msgs_to_delete_timeout){ // sub_dbi.del(wtxn, key_id); @@ -123,47 +126,46 @@ void persipubsub::queue::prune_dangling_messages_for(persipubsub::queue::Queue * } -void persipubsub::queue::Queue::init(boost::filesystem::path path, lmdb::env env) { +void persipubsub::queue::Queue::init(const boost::filesystem::path& path, lmdb::env env) { path_ = path; if (env) - // todo check if swap works - std::swap(env_, env); + // todo check if move works YES + env_ = std::move(env); else env_ = persipubsub::queue::initialize_environment(path, persipubsub::MAX_READER_NUM, persipubsub::MAX_DB_NUM, persipubsub::MAX_DB_SIZE_BYTES); // todo continue init - auto wtxn = lmdb::txn::begin(env_); - auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB, MDB_CREATE); - auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); - auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB, MDB_CREATE); - auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + { + auto wtxn = lmdb::txn::begin(env_); + + auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB, MDB_CREATE); + auto meta_dbi = lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); + // lmdb::dbi::open(wtxn, persipubsub::META_DB, MDB_CREATE); // TODO ask marko how to deal with it + auto data_dbi = lmdb::dbi::open(wtxn, persipubsub::DATA_DB, MDB_CREATE); + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + wtxn.commit(); // TODO this will not be needed, jumping out of scope + } - wtxn.commit(); // todo lookup hwm data - unsigned int msg_timeout_secs = 500; - unsigned int max_msgs_num = 1000; - unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; - HighWaterMark hwm = HighWaterMark(msg_timeout_secs, max_msgs_num, hwm_lmdb_size); - std::swap(hwm_, hwm); + persipubsub::QueueData queue_data = persipubsub::lookup_queue_data(env_); - // todo lookup strategy data - std::string strategy_str = "prune_first"; + hwm_ = HighWaterMark(queue_data.msg_timeout_secs_, + queue_data.max_msgs_num_, queue_data.hwm_db_size_bytes_); - strategy_ = persipubsub::queue::parse_strategy(strategy_str); + strategy_ = queue_data.strategy_; - // todo lookup subscriber list - char sub[] = "sub"; - subscriber_ids_.push_back(sub); + subscriber_ids_ = std::move(queue_data.subscriber_ids_); } -void persipubsub::queue::Queue::put(std::string msg, std::vector &subscriber_ids){ +void persipubsub::queue::Queue::put(const std::string& msg, const std::vector &subscriber_ids) const{ // todo uncomment - // vacuum(); + vacuum(); // todo check UTC time_t timer; time(&timer); @@ -199,8 +201,8 @@ void persipubsub::queue::Queue::put(std::string msg, std::vector &s } -void persipubsub::queue::Queue::put_many_flush_once(std::vector &msgs, - std::vector &subscriber_ids){ +void persipubsub::queue::Queue::put_many_flush_once(const std::vector &msgs, + const std::vector &subscriber_ids) const { vacuum(); // todo check UTC @@ -241,13 +243,12 @@ void persipubsub::queue::Queue::put_many_flush_once(std::vector &ms sub_dbi_it -> put(wtxn, msg_id, sub_data); } } - wtxn.commit(); } -// todo msg as pointer or address? -void persipubsub::queue::Queue::front(std::string identifier, std::string& msg){ +// todo msg as pointer or address? POINTER +void persipubsub::queue::Queue::front(const std::string& identifier, std::string* msg) const{ auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); auto sub_dbi = lmdb::dbi::open(rtxn, identifier.c_str()); @@ -261,7 +262,7 @@ void persipubsub::queue::Queue::front(std::string identifier, std::string& msg){ bool found = data_cursor.get(key, data_value, MDB_FIRST); if (found){ std::string tmp_str = data_value.data(); - std::swap(msg, tmp_str); + std::swap(*msg, tmp_str); } else{ cursor.close(); @@ -273,36 +274,41 @@ void persipubsub::queue::Queue::front(std::string identifier, std::string& msg){ rtxn.abort(); } -void persipubsub::queue::Queue::pop(std::string identifier){ +void persipubsub::queue::Queue::pop(const std::string& identifier) const { auto wtxn = lmdb::txn::begin(env_); auto sub_dbi = lmdb::dbi::open(wtxn, identifier.c_str()); auto pending_dbi = lmdb::dbi::open(wtxn, persipubsub::PENDING_DB); auto cursor = lmdb::cursor::open(wtxn, sub_dbi); + lmdb::val key, value; if (cursor.get(key, value, MDB_FIRST)) { sub_dbi.del(wtxn, key); - lmdb::val pending_value; pending_dbi.get(wtxn, key, pending_value); auto pending_num = boost::numeric_cast(std::stoi(pending_value.data())); pending_num--; lmdb::val pending_new_value((boost::lexical_cast(pending_num)).c_str()); pending_dbi.put(wtxn, key, pending_new_value); + cursor.close(); + wtxn.commit(); } - else + else { + cursor.close(); + wtxn.commit(); throw std::runtime_error("No message to pop"); + } } -void persipubsub::queue::Queue::prune_dangling_messages(){ - persipubsub::queue::prune_dangling_messages_for(this, subscriber_ids_); +void persipubsub::queue::Queue::prune_dangling_messages() const { + persipubsub::queue::prune_dangling_messages_for(*this, subscriber_ids_); } // todo size_t instead of long? -unsigned long persipubsub::queue::Queue::check_current_lmdb_size(){ +unsigned long persipubsub::queue::Queue::check_current_lmdb_size() const{ auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); auto data_dbi = lmdb::dbi::open(rtxn, persipubsub::DATA_DB); @@ -318,18 +324,17 @@ unsigned long persipubsub::queue::Queue::check_current_lmdb_size(){ } -unsigned int persipubsub::queue::Queue::count_msgs(){ +unsigned int persipubsub::queue::Queue::count_msgs() const { auto rtxn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY); auto meta_dbi = lmdb::dbi::open(rtxn, persipubsub::META_DB); auto msgs_num = boost::numeric_cast(meta_dbi.stat(rtxn).ms_entries); rtxn.abort(); - return msgs_num; } -void persipubsub::queue::Queue::vacuum(){ +void persipubsub::queue::Queue::vacuum() const { prune_dangling_messages(); unsigned int msgs_num = count_msgs(); @@ -342,7 +347,7 @@ void persipubsub::queue::Queue::vacuum(){ } -void persipubsub::queue::Queue::prune_messages(){ +void persipubsub::queue::Queue::prune_messages() const { std::set messages_to_delete; @@ -386,6 +391,7 @@ void persipubsub::queue::Queue::prune_messages(){ dbis.push_back(std::move(meta_dbi)); dbis.push_back(std::move(data_dbi)); + // TODO iterate nicely, always try to use const auto& for (auto sub_it = subscriber_ids_.begin(); sub_it != subscriber_ids_.end(); ++sub_it){ auto sub_dbi = lmdb::dbi::open(wtxn, (*sub_it).c_str()); dbis.push_back(std::move(sub_dbi)); diff --git a/cpp/src/queue.h b/cpp/src/queue.h index 1b76f13..7780be5 100644 --- a/cpp/src/queue.h +++ b/cpp/src/queue.h @@ -16,7 +16,10 @@ namespace persipubsub { /** * Store possible strategies. */ - enum Strategy {prune_first, prune_last}; + enum Strategy { + prune_first = 0, + prune_last + }; /** * Parse overflow strategy. @@ -37,14 +40,15 @@ namespace persipubsub { * @param max_masgs_num maximal amount of msg * @param hwm_lmdb_size_bytes high water mark for total size of lmdb (bytes) */ - unsigned int msg_timeout_secs_; + unsigned int msg_timeout_secs_; // TODO set to default unsigned int max_msgs_num_; unsigned long hwm_lmdb_size_bytes_; - HighWaterMark() : msg_timeout_secs_(), max_msgs_num_(), hwm_lmdb_size_bytes_() {} + HighWaterMark() : msg_timeout_secs_(), max_msgs_num_(), hwm_lmdb_size_bytes_() {} // TODO dont do this () - HighWaterMark(unsigned int msg_timeout_secs, unsigned int max_msgs_num, - unsigned long hwm_lmdb_size) : msg_timeout_secs_(msg_timeout_secs), max_msgs_num_(max_msgs_num), hwm_lmdb_size_bytes_(hwm_lmdb_size) {} + HighWaterMark(const unsigned int msg_timeout_secs, + const unsigned int max_msgs_num, + const unsigned long hwm_lmdb_size) : msg_timeout_secs_(msg_timeout_secs), max_msgs_num_(max_msgs_num), hwm_lmdb_size_bytes_(hwm_lmdb_size) {} }; /** @@ -56,7 +60,7 @@ namespace persipubsub { * @param max_db_size_bytes maximal size of database (bytes) * @return Load or if needed create LMDB queue from directory */ - lmdb::env initialize_environment(const boost::filesystem::path& queue_dir, const unsigned int& max_reader_num, const unsigned int& max_db_num, const unsigned long& max_db_size_bytes); + lmdb::env initialize_environment(const boost::filesystem::path& queue_dir, const unsigned int max_reader_num, const unsigned int max_db_num, const unsigned long max_db_size_bytes); /** * Prune all dangling messages for subscribers of a queue from lmdb. @@ -64,8 +68,8 @@ namespace persipubsub { * @param queue of which dangling messages should be pruned * @param subscriber_ids subscribers of which dangling messages should be pruned */ - void prune_dangling_messages_for(persipubsub::queue::Queue * queue, std::vector& subscriber_ids); - + void prune_dangling_messages_for(const persipubsub::queue::Queue& queue, const std::vector& subscriber_ids); + // todo string inside vector reference or normal? /** * Queue messages persistently from many publishers for many subscribers. */ @@ -88,7 +92,7 @@ namespace persipubsub { * @param max_db_num maximal number of databases * @param max_db_size_bytes maximal size of database (bytes) */ - void init(boost::filesystem::path path, lmdb::env env = nullptr); + void init(const boost::filesystem::path& path, lmdb::env env= nullptr); /** * Put message to lmdb queue. @@ -96,15 +100,15 @@ namespace persipubsub { * @param msg message send from publisher to subscribers * @param subscriber_ids List of subscribers */ - void put(std::string msg, std::vector& subscriber_ids); - + void put(const std::string& msg, const std::vector& subscriber_ids) const; // TODO make const functions const + // todo same string&? /** * Put many message to lmdb queue. * * @param msgs messages send from publisher to subscribers * @param subscriber_ids List of subscribers */ - void put_many_flush_once(std::vector& msgs, std::vector& subscriber_ids); + void put_many_flush_once(const std::vector& msgs, const std::vector& subscriber_ids) const; /** * Peek at next message in lmdb queue. @@ -113,19 +117,19 @@ namespace persipubsub { * @param identifier Subscriber ID * @param message peek on the message */ - void front(std::string identifier, std::string& msg); - + void front(const std::string& identifier, std::string* msg) const; + // todo change to pointer /** * Remove msg from the subscriber's queue and reduce pending subscribers. * * @param identifier Subscriber ID */ - void pop(std::string identifier); + void pop(const std::string& identifier) const; /** * Prune dangling messages in the queue. */ - void prune_dangling_messages(); + void prune_dangling_messages() const; /** * Check current lmdb size in bytes. @@ -133,7 +137,7 @@ namespace persipubsub { * Check size of data database by approximating size with multiplying page size with number of pages. * @return data database size in bytes */ - unsigned long check_current_lmdb_size(); + unsigned long check_current_lmdb_size() const; /** * Count number of messages in database. @@ -142,19 +146,19 @@ namespace persipubsub { * * @return number of messages in database */ - unsigned int count_msgs(); + unsigned int count_msgs() const; /** * Clean database when needed. */ - void vacuum(); + void vacuum() const; /** * Prune one half of the messages stored. * * Depending on the strategy the first or the last will be deleted. */ - void prune_messages(); + void prune_messages() const; boost::filesystem::path path_; lmdb::env env_; diff --git a/cpp/test/test_queue.cpp b/cpp/test/test_queue.cpp index 75000cb..7f66b19 100644 --- a/cpp/test/test_queue.cpp +++ b/cpp/test/test_queue.cpp @@ -1,6 +1,6 @@ // Copyright (c) 2019 Parquery AG. All rights reserved. // Created by Selim Naji (selim.naji@parquery.com and -// Marko Ristin (marko@parquery.com) on 2019-02-15 +// Selim Naji (selim.naji@parquery.com & marko@parquery.com) on 2019-02-15 #define BOOST_TEST_DYN_LINK #define BOOST_TEST_MODULE TestQueue @@ -23,6 +23,7 @@ #include #include "../src/queue.h" +#include "../src/library.h" namespace fs = boost::filesystem; @@ -41,6 +42,34 @@ class TmpDir{ }; // class TmpDir + +void setup_queue(const lmdb::txn& wtxn, const unsigned int msg_timeout_secs, + const unsigned int max_msgs_num, const unsigned long hwm_lmdb_size, + const std::string& strategy, const std::string& sub_ids){ + auto queue_dbi = lmdb::dbi::open(wtxn, persipubsub::QUEUE_DB, MDB_CREATE); + + lmdb::val msg_timeout_secs_key(persipubsub::MSG_TIMEOUT_SECS_KEY); + lmdb::val msg_timeout_secs_val(boost::lexical_cast(msg_timeout_secs)); + queue_dbi.put(wtxn, msg_timeout_secs_key, msg_timeout_secs_val); + + lmdb::val max_msgs_num_key(persipubsub::MAX_MSGS_NUM_KEY); + lmdb::val max_msgs_num_val(boost::lexical_cast(max_msgs_num)); + queue_dbi.put(wtxn, max_msgs_num_key, max_msgs_num_val); + + lmdb::val hwm_lmdb_size_key(persipubsub::HWM_DB_SIZE_BYTES_KEY); + lmdb::val hwm_lmdb_size_val(boost::lexical_cast(hwm_lmdb_size)); + queue_dbi.put(wtxn, hwm_lmdb_size_key, hwm_lmdb_size_val); + + lmdb::val strategy_key(persipubsub::STRATEGY_KEY); + lmdb::val strategy_val(strategy); + queue_dbi.put(wtxn, strategy_key, strategy_val); + + lmdb::val subscriber_ids_key(persipubsub::SUBSCRIBER_IDS_KEY); + lmdb::val subscriber_ids_val(sub_ids); + queue_dbi.put(wtxn, subscriber_ids_key, subscriber_ids_val); +} + + BOOST_AUTO_TEST_CASE(test_initialize_environment){ TmpDir tmp_dir = TmpDir(); @@ -81,11 +110,19 @@ BOOST_AUTO_TEST_CASE(test_put_to_single_subscriber) { 32UL * 1024UL * 1024UL * 1024UL); // todo ask Adam if '.' will be a problem. - std::string msg = "I'm a message.\n"; - std::string name = "sub"; + std::string msg = "I'm a message.\n"; // TODO read about carriage return, /0 and other funky issues + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; auto wtxn = lmdb::txn::begin(env); - auto dbi = lmdb::dbi::open(wtxn, name.c_str(), MDB_CREATE); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, sub_ids); + wtxn.commit(); // The Environment is closed automatically. For testing purpose the // environment is here forcefully closed. @@ -95,12 +132,150 @@ BOOST_AUTO_TEST_CASE(test_put_to_single_subscriber) { queue.init(queue_dir); std::vector subs; - subs.push_back(name); + subs.push_back(sub_ids); queue.put(msg, subs); std::string result; - queue.front(name, result); + queue.front(sub_ids, &result); BOOST_REQUIRE_EQUAL(msg, result); } + +BOOST_AUTO_TEST_CASE(test_put_multiple_subscriber) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub1 sub2"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub1_dbi = lmdb::dbi::open(wtxn, "sub1", MDB_CREATE); + auto sub2_dbi = lmdb::dbi::open(wtxn, "sub2", MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.emplace_back("sub1"); + subs.emplace_back("sub2"); + queue.put(msg, subs); + + std::string result1; + queue.front("sub1", &result1); + BOOST_REQUIRE_EQUAL(msg, result1); + std::string result2; + queue.front("sub2", &result2); + BOOST_REQUIRE_EQUAL(msg, result2); +} + +BOOST_AUTO_TEST_CASE(test_put_many) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, + sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + + std::vector msgs; + unsigned int test_num_msgs = 10; + for (int num_msgs = test_num_msgs; num_msgs > 0; num_msgs--) + msgs.push_back(msg); + + queue.put_many_flush_once(msgs, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + BOOST_REQUIRE_EQUAL(test_num_msgs, queue.count_msgs()); +} + +BOOST_AUTO_TEST_CASE(test_pop) { + + TmpDir tmp_dir = TmpDir(); + const fs::path queue_dir = tmp_dir.path_; + lmdb::env env = persipubsub::queue::initialize_environment(queue_dir, 1024, + 1024, + 32UL * 1024UL * + 1024UL * 1024UL); + // todo ask Adam if '.' will be a problem. + std::string msg = "I'm a message.\n"; + std::string sub_ids = "sub"; + std::string strategy = "prune_first"; + + unsigned int msg_timeout_secs = 500; + unsigned int max_msgs_num = 1000; + unsigned long hwm_lmdb_size = 30UL * 1024UL * 1024UL * 1024UL; + + auto wtxn = lmdb::txn::begin(env); + auto sub_dbi = lmdb::dbi::open(wtxn, sub_ids.c_str(), MDB_CREATE); + + setup_queue(wtxn, msg_timeout_secs, max_msgs_num, hwm_lmdb_size, strategy, + sub_ids); + + wtxn.commit(); + // The Environment is closed automatically. For testing purpose the + // environment is here forcefully closed. + env.close(); + + persipubsub::queue::Queue queue = persipubsub::queue::Queue(); + queue.init(queue_dir); + + std::vector subs; + subs.push_back(sub_ids); + queue.put(msg, subs); + + std::string result; + queue.front(sub_ids, &result); + + BOOST_REQUIRE_EQUAL(msg, result); + + queue.pop(sub_ids); + std::string empty; + queue.front(sub_ids, &empty); + BOOST_REQUIRE_EQUAL("", empty.c_str()); +} \ No newline at end of file