diff --git a/AGENTS.md b/AGENTS.md index 90aba780..fd1fa947 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,9 +36,12 @@ uv run ruff format . --check # Format check uv run lint-imports # Hexagonal architecture boundary validation uv run mypy . # Type checking (strict, targets Python 3.10) -# Run all tests +# Run all tests (parallel by default: 4 xdist workers, one Postgres container each) uv run pytest +# Run tests serially (disable pytest-xdist) +uv run pytest -n 0 + # Run a single test file uv run pytest test/test_queries.py @@ -225,7 +228,9 @@ When a comment explains a non-obvious workaround, keep it short and put it adjac ## Testing Conventions -- **pytest config**: `asyncio_mode = "auto"`, 20s timeout per test, `--durations=15` +- **pytest config**: `asyncio_mode = "auto"`, 20s timeout per test, `--durations=15`, `-n 4` (pytest-xdist) +- Tests run in parallel via **pytest-xdist**: session-scoped fixtures are per-worker, so each worker starts its own Postgres container and template database. Pass `-n 0` to run serially, or `-n ` to change worker count. +- Keep tests xdist-safe: no fixed ports, no shared files, no cross-test state. Database isolation comes from the per-test `CREATE DATABASE ... TEMPLATE` fixture. - Declare async tests as `async def test_...` -- no `@pytest.mark.asyncio` decorator needed - Annotate all test function parameters: `async def test_foo(apgdriver: db.Driver, N: int) -> None:` - Use `@pytest.mark.parametrize` for multi-value testing diff --git a/pyproject.toml b/pyproject.toml index 59d707fb..18eb3151 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ dev = [ "opentelemetry-semantic-conventions>=0.41b0", "pytest-mock>=3.14.0", "pytest-timeout>=2.3.1", + "pytest-xdist>=3.8.0", "ruff", "tqdm", "types-croniter", @@ -154,7 +155,9 @@ explicit_package_bases = true asyncio_mode = "auto" timeout = "20" timeout_func_only = true -addopts = "--durations=15" +# -n 4: modest parallelism; each xdist worker gets its own session-scoped +# Postgres container, so this scales the suite across a few extra servers. +addopts = "--durations=15 -n 4" # --------------------------------------------------------------------------- # Import-linter: enforce hexagonal architecture boundaries diff --git a/test/conftest.py b/test/conftest.py index 179b8c6e..fd765aa3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,7 +21,7 @@ uvloop = None # type: ignore[assignment] from testcontainers.core.container import DockerContainer -from testcontainers.core.wait_strategies import LogMessageWaitStrategy +from testcontainers.core.wait_strategies import HealthcheckWaitStrategy from pgqueuer.db import SyncPsycopgDriver @@ -62,7 +62,21 @@ def __init__( self.with_env("POSTGRES_USER", self.username) self.with_env("POSTGRES_PASSWORD", self.password) self.with_env("POSTGRES_DB", self.dbname) - self.waiting_for(LogMessageWaitStrategy("database system is ready to accept connections")) + # Readiness comes from docker's own healthcheck (as in ci.yml service + # containers), probing pg_isready over TCP: the image's initdb + # bootstrap server listens only on the unix socket and logs a + # misleading "ready to accept connections" line, so the container + # cannot report healthy before the real server accepts TCP. + self._kwargs.setdefault( + "healthcheck", + { + "test": ["CMD", "pg_isready", "-h", "127.0.0.1", "-U", self.username], + "interval": 500_000_000, # docker durations are nanoseconds + "timeout": 2_000_000_000, + "retries": 120, + }, + ) + self.waiting_for(HealthcheckWaitStrategy()) def get_connection_url(self, host: str | None = None) -> str: if self._container is None: @@ -94,6 +108,10 @@ async def postgres_container() -> AsyncGenerator[str, None]: # https://postgresqlco.nf/doc/en/param/vacuum_buffer_usage_limit/ # Assume worst case for backwards compatibility + # + # Memory is bounded per container (shared_buffers, max_wal_size, tmpfs size) + # because each xdist worker starts its own container and the data dir lives + # on tmpfs (RAM). commands = [ "postgres", "-c", @@ -107,19 +125,22 @@ async def postgres_container() -> AsyncGenerator[str, None]: "-c", "checkpoint_timeout=30min", "-c", - "max_wal_size=4GB", + "max_wal_size=512MB", "-c", "checkpoint_completion_target=0.9", "-c", - "shared_buffers=256MB", + "shared_buffers=128MB", "-c", "autovacuum_naptime=5s", ] + (["-c", "vacuum_buffer_usage_limit=8MB"] if int(postgres_version) >= 16 else []) container = ( - PgQueuerPostgresContainer(f"postgres:{postgres_version}", driver=None) + PgQueuerPostgresContainer( + f"postgres:{postgres_version}", + driver=None, + tmpfs={"/var/lib/pg/data": "rw,size=1g"}, + ) .with_command(commands) - .with_kwargs(tmpfs={"/var/lib/pg/data": "rw"}) .with_envs(PGDATA="/var/lib/pg/data") ) diff --git a/uv.lock b/uv.lock index 818f054c..2406bb60 100644 --- a/uv.lock +++ b/uv.lock @@ -572,6 +572,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -1372,6 +1381,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-mock" }, { name = "pytest-timeout" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "testcontainers" }, { name = "time-machine" }, @@ -1451,6 +1461,7 @@ requires-dist = [ { name = "pytest-cov", marker = "extra == 'dev'" }, { name = "pytest-mock", marker = "extra == 'dev'", specifier = ">=3.14.0" }, { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.3.1" }, + { name = "pytest-xdist", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "ruff", marker = "extra == 'dev'" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.0.0" }, { name = "tabulate", specifier = ">=0.9.0" }, @@ -1775,6 +1786,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0"