From a9784e077f4b0b833034e425aa9e652a07170393 Mon Sep 17 00:00:00 2001 From: Hynek Schlawack Date: Thu, 30 Jul 2026 17:31:02 +0200 Subject: [PATCH] Add codspeed benchmarks If we want to judge PRs like #821 we need some kind of objective baseline. --- .github/workflows/codspeed.yml | 46 ++++++++ bench/test_benchmarks.py | 199 +++++++++++++++++++++++++++++++++ pyproject.toml | 9 +- tox.ini | 11 ++ 4 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/codspeed.yml create mode 100644 bench/test_benchmarks.py diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000..75214d81 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,46 @@ +--- +name: CodSpeed Benchmarks + +on: + push: + branches: [main] + tags: ["*"] + paths: + - src/**.py + - bench/** + - .github/workflows/codspeed.yml + pull_request: + paths: + - src/**.py + - bench/** + - .github/workflows/codspeed.yml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + FORCE_COLOR: "1" + +permissions: {} + +jobs: + codspeed: + name: Run CodSpeed benchmarks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version-file: .python-version-default + - uses: hynek/setup-cached-uv@34e35d30f1ebc7421a5cc733bca38dcc62603960 # v2.6.0 + + - name: Run CodSpeed benchmarks + uses: CodSpeedHQ/action@88472375d0a4572cf70a9f1fe3a4e0ab8da1b924 # v5.0.1 + with: + mode: simulation + run: uvx --with tox-uv tox run -e codspeed diff --git a/bench/test_benchmarks.py b/bench/test_benchmarks.py new file mode 100644 index 00000000..79bd4516 --- /dev/null +++ b/bench/test_benchmarks.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: MIT OR Apache-2.0 +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the MIT License. See the LICENSE file in the root of this +# repository for complete details. + +""" +Benchmark structlog using CodSpeed. +""" + +from __future__ import annotations + +import logging + +import pytest + +import structlog + + +pytestmark = pytest.mark.benchmark() + +ROUNDS = 1_000 + +# A typical set of key/value pairs that gets bound to a logger in a web +# application. +KWARGS = { + "user_id": 42, + "request_id": "b1f2b3c4-d5e6-4f7a-8b9c-0d1e2f3a4b5c", + "path": "/api/v1/orders", + "method": "POST", + "status": 201, + "duration_ms": 12.34, +} + +INFO_LOGGER_CLASS = structlog.make_filtering_bound_logger(logging.INFO) + + +def make_logger(*processors): + """ + Create a bound logger that filters at info level, runs *processors*, and + logs into the void. + """ + return structlog.wrap_logger( + structlog.testing.ReturnLogger(), + processors=list(processors), + wrapper_class=INFO_LOGGER_CLASS, + ).bind() + + +def test_create_bound_logger(): + """ + Benchmark wrapping a logger and binding initial values to it. + """ + for _ in range(ROUNDS): + structlog.wrap_logger( + structlog.testing.ReturnLogger(), + processors=[structlog.processors.JSONRenderer()], + wrapper_class=INFO_LOGGER_CLASS, + ).bind(**KWARGS) + + +def test_bind(): + """ + Benchmark binding key/value pairs to an existing bound logger. + """ + log = make_logger(structlog.processors.JSONRenderer()) + + for _ in range(ROUNDS): + log.bind(**KWARGS) + + +def test_log_event_json(): + """ + Benchmark logging an event through a typical production processor chain + that renders to JSON. + """ + log = make_logger( + structlog.processors.add_log_level, + structlog.processors.StackInfoRenderer(), + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.JSONRenderer(), + ).bind(**KWARGS) + + for _ in range(ROUNDS): + log.info("request handled") + + +def test_log_event_console(): + """ + Benchmark logging an event using the development ConsoleRenderer. + """ + log = make_logger( + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S", utc=True), + structlog.dev.ConsoleRenderer(colors=True), + ).bind(**KWARGS) + + for _ in range(ROUNDS): + log.info("request handled") + + +def test_log_event_key_value(): + """ + Benchmark logging an event using the classic KeyValueRenderer. + """ + log = make_logger( + structlog.processors.add_log_level, + structlog.processors.TimeStamper(fmt="iso", utc=True), + structlog.processors.KeyValueRenderer( + key_order=["timestamp", "level", "event"] + ), + ).bind(**KWARGS) + + for _ in range(ROUNDS): + log.info("request handled") + + +def test_log_event_filtered_out(): + """ + Benchmark a log call that gets dropped by the level filter. + """ + log = make_logger(structlog.processors.JSONRenderer()).bind(**KWARGS) + + for _ in range(ROUNDS): + log.debug("nobody will ever see this") + + +def test_log_event_with_contextvars(): + """ + Benchmark logging an event that merges context-local values. + """ + log = make_logger( + structlog.contextvars.merge_contextvars, + structlog.processors.JSONRenderer(), + ) + + structlog.contextvars.bind_contextvars(**KWARGS) + + for _ in range(ROUNDS): + log.info("request handled") + + structlog.contextvars.clear_contextvars() + + +def test_bind_and_clear_contextvars(): + """ + Benchmark binding and clearing context-local values, like a middleware + does once per request. + """ + for _ in range(ROUNDS): + structlog.contextvars.bind_contextvars(**KWARGS) + structlog.contextvars.clear_contextvars() + + +def _bottom(): + raise ValueError("d'oh") + + +def _middle(): + _bottom() + + +def _top(): + _middle() + + +def test_log_exception_as_dict(): + """ + Benchmark extracting and rendering an exception into structured data + using dict_tracebacks. + """ + log = make_logger( + structlog.processors.add_log_level, + structlog.processors.dict_tracebacks, + structlog.processors.JSONRenderer(), + ).bind(**KWARGS) + + try: + _top() + except ValueError: + for _ in range(ROUNDS): + log.exception("cannot compute") + + +def test_log_exception_formatted(): + """ + Benchmark formatting an exception into a traceback string using + format_exc_info. + """ + log = make_logger( + structlog.processors.add_log_level, + structlog.processors.format_exc_info, + structlog.processors.JSONRenderer(), + ).bind(**KWARGS) + + try: + _top() + except ValueError: + for _ in range(ROUNDS): + log.exception("cannot compute") diff --git a/pyproject.toml b/pyproject.toml index b9878eb6..aafb08ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,11 @@ tests = [ # Need Twisted & Rich for stubs. # Otherwise mypy fails in tox. typing = ["mypy>=1.4", "rich", "twisted"] +benchmark = [ + {include-group = "tests"}, + "pytest-codspeed", + "pytest-xdist[psutil]", +] docs = [ "cogapp", "myst-parser", @@ -186,9 +191,9 @@ ignore_errors = false # Analyze everything for LSP, but only complain about typing examples. [[tool.ty.overrides]] -include = ["src", "tests"] -exclude = ["tests/typing/"] rules.all = "ignore" +include = ["src", "tests", "bench"] +exclude = ["tests/typing/"] [tool.pyrefly] diff --git a/tox.ini b/tox.ini index 526a2677..8a32988d 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,17 @@ deps = coverage commands = coverage report +[testenv:codspeed] +dependency_groups = benchmark +pass_env = + CODSPEED_TOKEN + CODSPEED_ENV + ARCH + PYTHONHASHSEED + PYTHONMALLOC +commands = pytest --codspeed -n auto bench/test_benchmarks.py + + [testenv:docs-{build,doctests,linkcheck}] # Keep base_python in sync with ci.yml/docs and .readthedocs.yaml. base_python = 3.14