diff --git a/.github/workflows/check-pull-request.yml b/.github/workflows/check-pull-request.yml new file mode 100644 index 0000000..3cbab44 --- /dev/null +++ b/.github/workflows/check-pull-request.yml @@ -0,0 +1,42 @@ +name: Check-pull-request + +on: [pull_request] + +jobs: + Check: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@master + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + pip3 install -e .[dev] + + - name: Run checks + run: ./precommit.py + + - name: Coveralls Parallel + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.github_token }} + flag-name: run-${{ matrix.test_number }} + parallel: true + + Finish: + needs: Check + runs-on: ubuntu-latest + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@master + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel-finished: true \ No newline at end of file diff --git a/.github/workflows/check-push.yml b/.github/workflows/check-push.yml new file mode 100644 index 0000000..3a3c63b --- /dev/null +++ b/.github/workflows/check-push.yml @@ -0,0 +1,35 @@ +name: Check-push + +on: + push: + branches: + - master + +jobs: + Execute: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8] + + steps: + - uses: actions/checkout@master + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python3 -m pip install --upgrade pip + pip3 install -e .[dev] + pip3 install coveralls + + - name: Run checks + run: ./precommit.py + + - name: Upload coverage to coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: coveralls diff --git a/.gitignore b/.gitignore index 2e5d36d..995be76 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ venv *.egg-info .tox dist/ +.coverage diff --git a/README.rst b/README.rst index cc3bfc2..dbe92e7 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,22 @@ logthis ======= +.. image:: https://github.com/Parquery/logthis/workflows/Check-push/badge.svg + :target: https://github.com/Parquery/logthis/actions?query=workflow%3ACheck-push + :alt: Check status + +.. image:: https://coveralls.io/repos/github/Parquery/logthis/badge.svg?branch=master + :target: https://coveralls.io/github/Parquery/logthis + :alt: Test coverage + +.. image:: https://badge.fury.io/py/logthis.svg + :target: https://pypi.org/project/logthis/ + :alt: PyPI - version + +.. image:: https://img.shields.io/pypi/pyversions/logthis.svg + :target: https://pypi.org/project/logthis/ + :alt: PyPI - Python Version + logthis is a singleton, two-level, colorful, thread-safe, knob-free, logging library for in-house software. * **singleton**: There is no object to create. There are only two logging functions, ``say()`` and ``err()``. diff --git a/logthis/__init__.py b/logthis/__init__.py index eab652a..fab19b6 100644 --- a/logthis/__init__.py +++ b/logthis/__init__.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -""" -Provide singleton, two-level, colorful, thread-safe, knob-free, logging for in-house software. - -""" +"""Provide singleton, two-level, colorful, thread-safe, knob-free, logging for in-house software.""" import datetime import inspect import os @@ -14,9 +11,14 @@ class State: """Represent the global logging state.""" - lock = threading.Lock() - stdout = sys.stdout - stderr = sys.stderr + def __init__(self) -> None: + """Initialize with the default values.""" + self.lock = threading.Lock() + self.stdout = sys.stdout + self.stderr = sys.stderr + + +_GLOBAL_STATE = State() class Colors: @@ -82,26 +84,28 @@ def filename_line(skip: int = 2) -> Tuple[str, int]: return filename, parentframe.f_lineno -def say(message: str) -> None: +def say(message: str, state: State = _GLOBAL_STATE, utcnow: datetime.datetime = datetime.datetime.utcnow()) -> None: """ Print a formatted log message to STDOUT and flush. :param message: to be displayed - + :param state: state object to use; the default is the singleton + :param utcnow: current timestamp """ - with State.lock: + with state.lock: filename, line = filename_line() - State.stdout.write(say_as_text(filename=filename, line=line, message=message)) - State.stdout.flush() + state.stdout.write(say_as_text(filename=filename, line=line, message=message, utcnow=utcnow)) + state.stdout.flush() -def say_as_text(filename: str, line: int, message: str) -> str: +def say_as_text(filename: str, line: int, message: str, utcnow: datetime.datetime = datetime.datetime.utcnow()) -> str: """ Generate 'say' message as a string. :param filename: path to the script, usually you want to pass __file__ :param line: line number in the script, usually you want to pass inspect.currentframe().f_lineno :param message: to be displayed + :param utcnow: current timestamp :return: whole formatted message which the 'say' function will display """ @@ -109,32 +113,35 @@ def say_as_text(filename: str, line: int, message: str) -> str: blue=Colors.kForeBlue, fname=filename, line=line, - dt=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ"), + dt=utcnow.strftime("%Y-%m-%d %H:%M:%SZ"), msg=message, endcolor=Colors.kConsoleDefault) -def err(message: str) -> None: +def err(message: str, state: State = _GLOBAL_STATE, utcnow: datetime.datetime = datetime.datetime.utcnow()) -> None: """ Print a formatted log message to STDERR and flush. :param message: to be displayed + :param state: state object to use; default is the singleton + :param utcnow: current timestamp """ filename, line = filename_line() - with State.lock: - State.stderr.write(err_as_text(filename=filename, line=line, message=message)) - State.stderr.flush() + with state.lock: + state.stderr.write(err_as_text(filename=filename, line=line, message=message, utcnow=utcnow)) + state.stderr.flush() -def err_as_text(filename: str, line: int, message: str) -> str: +def err_as_text(filename: str, line: int, message: str, utcnow: datetime.datetime = datetime.datetime.utcnow()) -> str: """ Generate 'err' message as a string. :param filename: path to the script, usually you want to pass __file__ :param line: line number in the script, usually you want to pass inspect.currentframe().f_lineno :param message: to be displayed + :param utcnow: current timestamp :return: whole formatted message which the 'err' function will display """ @@ -142,6 +149,6 @@ def err_as_text(filename: str, line: int, message: str) -> str: red=Colors.kForeRed, fname=filename, line=line, - dt=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ"), + dt=utcnow.strftime("%Y-%m-%d %H:%M:%SZ"), msg=message, endcolor=Colors.kConsoleDefault) diff --git a/precommit.py b/precommit.py index f6219bd..afecb11 100755 --- a/precommit.py +++ b/precommit.py @@ -1,204 +1,105 @@ #!/usr/bin/env python3 -""" -Runs precommit checks on the repository. -""" +"""Run precommit checks on the repository.""" import argparse -import concurrent.futures -import hashlib import os import pathlib +import re import subprocess import sys -from typing import List, Union, Tuple # pylint: disable=unused-import -import yapf.yapflib.yapf_api - - -def compute_hash(text: str) -> str: - """ - :param text: to hash - :return: hash digest - """ - md5 = hashlib.md5() - md5.update(text.encode()) - return md5.hexdigest() - - -class Hasher: - """ - Hashes the source code files and reports if they differed to one of the previous hashings. - """ - - def __init__(self, source_dir: pathlib.Path, hash_dir: pathlib.Path) -> None: - self.source_dir = source_dir - self.hash_dir = hash_dir - - def __hash_dir(self, path: pathlib.Path) -> pathlib.Path: - """ - :param path: to a source file - :return: path to the file holding the hash of the source text - """ - if self.source_dir not in path.parents: - raise ValueError("Expected the path to be beneath the source directory {!r}, got: {!r}".format( - str(self.source_dir), str(path))) - - return self.hash_dir / path.relative_to(self.source_dir).parent / path.name - - def hash_differs(self, path: pathlib.Path) -> bool: - """ - :param path: to the source file - :return: True if the hash of the content differs to one of the previous hashings. - """ - hash_dir = self.__hash_dir(path=path) - - if not hash_dir.exists(): - return True - - prev_hashes = set([pth.name for pth in hash_dir.iterdir()]) - - new_hsh = compute_hash(text=path.read_text()) - - return not new_hsh in prev_hashes - - def update_hash(self, path: pathlib.Path) -> None: - """ - Hashes the file content and stores it on disk. - - :param path: to the source file - :return: - """ - hash_dir = self.__hash_dir(path=path) - hash_dir.mkdir(exist_ok=True, parents=True) - - new_hsh = compute_hash(text=path.read_text()) - - pth = hash_dir / new_hsh - pth.write_text('passed') +def main() -> int: + """Execute the main routine.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + help="Overwrites the unformatted source files with the " + "well-formatted code in place. If not set, " + "an exception is raised if any of the files do not conform " + "to the style guide.", + action='store_true') -def check(path: pathlib.Path, py_dir: pathlib.Path, overwrite: bool) -> Union[None, str]: - """ - Runs all the checks on the given file. + args = parser.parse_args() - :param path: to the source file - :param py_dir: path to the source files - :param overwrite: if True, overwrites the source file in place instead of reporting that it was not well-formatted. - :return: None if all checks passed. Otherwise, an error message. - """ - style_config = py_dir / 'style.yapf' + overwrite = bool(args.overwrite) - report = [] + repo_root = pathlib.Path(__file__).parent - # yapf - if not overwrite: - formatted, _, changed = yapf.yapflib.yapf_api.FormatFile( - filename=str(path), style_config=str(style_config), print_diff=True) + # yapf: disable + source_files = ( + sorted((repo_root / "logthis").glob("**/*.py")) + + sorted((repo_root / "tests").glob("**/*.py"))) + # yapf: enable - if changed: - report.append("Failed to yapf {}:\n{}".format(path, formatted)) + if overwrite: + print('Removing trailing whitespace...') + for pth in source_files: + pth.write_text(re.sub(r'[ \t]+$', '', pth.read_text(), flags=re.MULTILINE)) + + print("YAPF'ing...") + yapf_targets = ["tests", "logthis", "setup.py", "precommit.py"] + if overwrite: + # yapf: disable + subprocess.check_call( + ["yapf", "--in-place", "--style=style.yapf", "--recursive"] + + yapf_targets, + cwd=str(repo_root)) + # yapf: enable else: - yapf.yapflib.yapf_api.FormatFile(filename=str(path), style_config=str(style_config), in_place=True) - - # mypy - env = os.environ.copy() - env['PYTHONPATH'] = ":".join([py_dir.as_posix(), env.get("PYTHONPATH", "")]) - - proc = subprocess.Popen( - ['mypy', str(path), '--ignore-missing-imports'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - universal_newlines=True) - stdout, stderr = proc.communicate() - if proc.returncode != 0: - report.append("Failed to mypy {}:\nOutput:\n{}\n\nError:\n{}".format(path, stdout, stderr)) - - # pylint - proc = subprocess.Popen( - ['pylint', str(path), '--rcfile={}'.format(py_dir / 'pylint.rc')], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - - stdout, stderr = proc.communicate() - if proc.returncode != 0: - report.append("Failed to pylint {}:\nOutput:\n{}\n\nError:\n{}".format(path, stdout, stderr)) + # yapf: disable + subprocess.check_call( + ["yapf", "--diff", "--style=style.yapf", "--recursive"] + + yapf_targets, + cwd=str(repo_root)) + # yapf: enable - if len(report) > 0: - return "\n".join(report) - - return None + print("Mypy'ing...") + subprocess.check_call(["mypy", "--strict", "logthis", "tests"], cwd=str(repo_root)) + print("Isort'ing...") + # yapf: disable + isort_files = map(str, source_files) + # yapf: enable -def main() -> int: - """" - Main routine - """ - # pylint: disable=too-many-locals - parser = argparse.ArgumentParser() - parser.add_argument( - "--overwrite", - help="Overwrites the unformatted source files with the well-formatted code in place. " - "If not set, an exception is raised if any of the files do not conform to the style guide.", - action='store_true') + # yapf: disable + subprocess.check_call( + ["isort", "--project", "logthis", '--line-width', '120'] + + ([] if overwrite else ['--check-only']) + + [str(pth) for pth in source_files]) + # yapf: enable - parser.add_argument("--all", help="checks all the files even if they didn't change", action='store_true') + print("Pydocstyle'ing...") + subprocess.check_call(["pydocstyle", "logthis"], cwd=str(repo_root)) - args = parser.parse_args() + print("Pylint'ing...") + subprocess.check_call(["pylint", "--rcfile=pylint.rc", "tests", "logthis"], cwd=str(repo_root)) - overwrite = bool(args.overwrite) - check_all = bool(args.all) + print("Testing...") + env = os.environ.copy() + env['ICONTRACT_SLOW'] = 'true' - py_dir = pathlib.Path(__file__).parent + # yapf: disable + subprocess.check_call( + ["coverage", "run", + "--source", "logthis", + "-m", "unittest", "discover", "tests"], + cwd=str(repo_root), + env=env) + # yapf: enable - hash_dir = py_dir / '.precommit_hashes' - hash_dir.mkdir(exist_ok=True) + subprocess.check_call(["coverage", "report"]) - hasher = Hasher(source_dir=py_dir, hash_dir=hash_dir) + print("Doctesting...") + doctest_files = ([repo_root / "README.rst"] + sorted((repo_root / "logthis").glob("**/*.py"))) - # yapf: disable - pths = sorted( - list(py_dir.glob("*.py")) + - list((py_dir / 'tests').glob("*.py"))) - # yapf: enable + for pth in doctest_files: + subprocess.check_call([sys.executable, "-m", "doctest", str(pth)]) - # see which files changed: - pending_pths = [] # type: List[pathlib.Path] + print("Checking setup.py sdist ...") + subprocess.check_call([sys.executable, "setup.py", "sdist"], cwd=str(repo_root)) - if check_all: - pending_pths = pths - else: - for pth in pths: - if hasher.hash_differs(path=pth): - pending_pths.append(pth) - - print("There are {} file(s) that need to be individually checked...".format(len(pending_pths))) - - success = True - - futures_paths = [] # type: List[Tuple[concurrent.futures.Future, pathlib.Path]] - with concurrent.futures.ThreadPoolExecutor() as executor: - for pth in pending_pths: - future = executor.submit(fn=check, path=pth, py_dir=py_dir, overwrite=overwrite) - futures_paths.append((future, pth)) - - for future, pth in futures_paths: - report = future.result() - if report is None: - print("Passed all checks: {}".format(pth)) - hasher.update_hash(path=pth) - else: - print("One or more checks failed for {}:\n{}".format(pth, report)) - success = False - - print("Running unit tests...") - source_dir = pathlib.Path(__file__).resolve().parent - retcode = subprocess.call(['python3', '-m', 'unittest', 'discover', str(source_dir / 'tests')]) - success = success and retcode == 0 - - if not success: - print("One or more checks failed.") - return 1 + print("Checking with twine...") + subprocess.check_call(["twine", "check", "dist/*"], cwd=str(repo_root)) return 0 diff --git a/setup.py b/setup.py index 7e6da5d..d194266 100644 --- a/setup.py +++ b/setup.py @@ -25,17 +25,32 @@ author_email='marko.ristin@parquery.com', license='MIT License', classifiers=[ + # yapf: disable 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8' + # yapf: enable ], keywords='logging log colorful color simple plain straightforward', packages=find_packages(exclude=['tests']), install_requires=[], extras_require={ - 'dev': ['mypy==0.600', 'pylint==1.8.4', 'yapf==0.20.2', 'tox>=3.0.0', 'temppathlib==1.0.1'], - 'test': ['tox==3.0.0', 'temppathlib==1.0.1'] + 'dev': [ + # yapf: disable + 'mypy==0.790', + 'pylint==2.6.0', + 'yapf==0.20.2', + 'coverage>=5,<6', + 'pydocstyle>=5,<6', + 'tox>=3.0.0,<4', + 'temppathlib>=1.0.3,<2', + 'twine' + # yapf: enable + ] }, py_modules=['logthis'], package_data={"logthis": ["py.typed"]}) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..b9d4584 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test logthis.""" diff --git a/tests/component_test_logthis.py b/tests/component_test_logthis.py deleted file mode 100755 index cf68ff3..0000000 --- a/tests/component_test_logthis.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -""" -Test logthis as a component by generating a temporary Python script. -""" -import datetime -import subprocess -import sys - -import temppathlib - - -def main() -> int: - """ - executes the main routine. - """ - with temppathlib.NamedTemporaryFile(mode="wt", prefix="logthis_temporary", suffix=".py") as tmp: - tmp.file.write('#!/usr/bin/env python3\n' - 'import logthis\n' - 'logthis.say("Hello!")\n' - 'logthis.err("Wrong.")\n') - tmp.file.flush() - tmp.file.close() - - tmp.path.chmod(0o700) - - proc = subprocess.Popen([tmp.path.as_posix()], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - out, err = proc.communicate() - if proc.returncode != 0: - raise RuntimeError("Temporary script failed. Stdout:\n{}\nStderr:\n{}\n".format(out, err)) - - name = tmp.path.name - now = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%SZ") - - expected_out = b'\x1b[34m' + name.encode() + b': 3: ' + now.encode() + b':\x1b[0m Hello!\n' - - if out != expected_out: - for i, (expected, got) in enumerate(zip(expected_out, out)): - if expected != got: - arrow = " " * (10 + len("{}".format(out[:i - 1]))) + "^" - raise AssertionError("Unexpected STDOUT:\nExpected: {}\nBut got: {}\n{}".format( - expected_out, out, arrow)) - - expected_err = b'\x1b[31m' + name.encode() + b': 4: ' + now.encode() + b':\x1b[0m Wrong.\n' - if err != expected_err: - for i, (expected, got) in enumerate(zip(expected_err, err)): - if expected != got: - arrow = " " * (10 + len("{}".format(err[:i - 1]))) + "^" - raise AssertionError("Unexpected STDERR:\nExpected: {}\nBut got: {}\n{}".format( - expected_err, err, arrow)) - - sys.stdout.write(expected_out.decode()) - sys.stderr.write(expected_err.decode()) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/example.py b/tests/example.py index d301141..71082f8 100755 --- a/tests/example.py +++ b/tests/example.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 -""" -Make an example of how to use logthis. -""" +"""Make an example of how to use logthis.""" import sys import logthis diff --git a/tests/test.py b/tests/test.py new file mode 100644 index 0000000..e63e33c --- /dev/null +++ b/tests/test.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Test logthis as a component by generating a temporary Python script.""" +import datetime +import io +import os +import pathlib +import subprocess +import sys +import tempfile +import textwrap +import unittest +import unittest.mock + +import logthis + +# pylint: disable=missing-class-docstring,missing-function-docstring,too-many-locals,no-self-use + + +class TestMocked(unittest.TestCase): + def test_say(self) -> None: + utcnow = datetime.datetime(1901, 12, 21) + + state = logthis.State() + state.stdout = io.StringIO() + state.stderr = io.StringIO() + + logthis.say("hello!", state=state, utcnow=utcnow) + + out = state.stdout.getvalue() + expected_out = '\x1b[34mtest.py: 27: 1901-12-21 00:00:00Z:\x1b[0m hello!\n' + self.assertEqual(expected_out, out) + + err = state.stderr.getvalue() + expected_err = '' + self.assertEqual(expected_err, err) + + def test_err(self) -> None: + utcnow = datetime.datetime(1901, 12, 21) + + state = logthis.State() + state.stdout = io.StringIO() + state.stderr = io.StringIO() + + logthis.err("Wrong!", state=state, utcnow=utcnow) + + out = state.stdout.getvalue() + expected_out = '' + self.assertEqual(expected_out, out) + + err = state.stderr.getvalue() + expected_err = '\x1b[31mtest.py: 44: 1901-12-21 00:00:00Z:\x1b[0m Wrong!\n' + self.assertEqual(expected_err, err) + + +class TestComponent(unittest.TestCase): + def test_say_and_err(self) -> None: + text = textwrap.dedent('''\ + #!/usr/bin/env python3 + import logthis + logthis.say("Hello!") + logthis.err("Wrong.") + ''') + + with tempfile.TemporaryDirectory() as tmpdir: + pth = pathlib.Path(tmpdir) / "logthis_out.py" + pth.write_text(text) + + proc = subprocess.Popen([sys.executable, str(pth)], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = proc.communicate() + if proc.returncode != 0: + raise RuntimeError("Temporary script failed. Stdout:\n{!r}\nStderr:\n{!r}\n".format(out, err)) + + name = pth.name + + ## + # Check out + ## + + expected_out_prefix = b'\x1b[34m' + name.encode() + b': 3: ' + # We need to ignore the datetime as this is too hard to mock. + expected_out_suffix = b':\x1b[0m Hello!' + os.linesep.encode() + + self.assertTrue(out.startswith(expected_out_prefix)) + self.assertTrue(out.endswith(expected_out_suffix)) + + self.assertEqual(len(expected_out_prefix) + 20 + len(expected_out_suffix), len(out)) + + ## + # Check err + ## + + expected_err_prefix = b'\x1b[31m' + name.encode() + b': 4: ' + # We need to ignore the datetime as this is too hard to mock. + expected_err_suffix = b':\x1b[0m Wrong.' + os.linesep.encode() + + self.assertTrue(err.startswith(expected_err_prefix)) + self.assertTrue(err.endswith(expected_err_suffix)) + + self.assertEqual(len(expected_err_prefix) + 20 + len(expected_err_suffix), len(err)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tox.ini b/tox.ini index 5ba7a1c..6efb544 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35 +envlist = py35,py36,py37,py38 [testenv] deps =