From 7f8ff2c69edae9fc45d27069e372efeab69c52b2 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 16 May 2026 14:34:54 +0200 Subject: [PATCH 1/4] fix: Calendar.delete() wipe tristate; handle NotFoundError during wipe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calendar.delete() gains a wipe= tristate parameter: True – wipe all objects, keep the calendar (for servers like Nextcloud where HTTP DELETE moves the calendar to a trashbin) False – always issue HTTP DELETE None – existing auto-detect: wipe if the server doesn't support deletion NotFoundError is now silently ignored per object during a wipe, so a concurrent deletion or a server quirk doesn't abort the whole sweep. Same fix applied to the async path (_async_delete). Also clarifies two comments in calendarobjectresource.py: the expand_rrule path uses client-side expansion (recurring_ical_events always supplies RECURRENCE-ID), and only_this_recurrence silently misfires on server-side expanded initial instances that RFC 4791 §9.6.5 permits to omit RECURRENCE-ID. prompt: the github runs takes more than an hour now. Can we wipe the calendar instead of deleting it, and use the same calendar for all the tests? followup-prompt: github runs still fail followup-prompt: github test runs still fail, please investigate Co-Authored-By: claude-sonnet-4-6 AI Prompts: claude-sonnet-4-6: The last test run at github passed. There has been 17 commits in this branch. Please check the differences between master and write a short recap of what has been done in this branch. claude-sonnet-4-6: I didn't quite understand the main thing with this branch - getting the async tests running for cyrus, baikal and nextcloud. Shouldn't this have happened in master also? The test suite is started through tox, should go through all available images, and both sync and async tests should be started by tox? Next, can this branch be rebased into four distinct, logically independent commits with good commit messages? claude-sonnet-4-6: All tests are passing both on github and locally in the master branch as far as I can see, but you say that the async tests are failing in master? Please explain. Regarding the rebase, I thought of git commits reflecting the grouping you did above; github CI infrastructure, core bugfixes, test reliability and compatibility hints. claude-sonnet-4-6: yes. I suppose it will be needed to split some commits too. I think it's important to have a clean commit with the core bugfixes, and I'd also like to have all the github-related logic in a separate commit not touching anything else. --- CHANGELOG.md | 6 ++++ caldav/calendarobjectresource.py | 10 ++++-- caldav/collection.py | 52 ++++++++++++++++++++++++++------ 3 files changed, 57 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5fed5540..9a5cf176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ Changelogs prior to v3.0 is pruned, but was available in the v3.1 release This project should adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), though for pre-releases PEP 440 takes precedence. +## [Unreleased] + +### Added + +* `Calendar.delete(wipe=None)` now accepts a `wipe` parameter. `wipe=True` wipes all objects from the calendar without deleting the calendar itself — useful for servers like Nextcloud where calendar deletion moves the calendar to a trashbin without freeing the URL namespace. `wipe=False` always attempts a HTTP DELETE regardless of server support. The existing `None` default preserves current auto-detect behaviour. + ## [3.2.0] - 2026-04-24 The two most significant news in v3.2 are **relatively well-tested support for scheduling** (RFC6638) and **better-tested support for async**. Care should still be taken, those features are backed by many tests, but lacks testing for how well they support real-world use-case scenarios. While async support was added in version 3.0, it was not well-enough tested. Still only a fraction of all the integration tests for sync usage has been duplicated in the async integration test, I expect to release 3.2.1 with symmetric async integration tests before 2025-07. diff --git a/caldav/calendarobjectresource.py b/caldav/calendarobjectresource.py index 49481a67..e554e333 100644 --- a/caldav/calendarobjectresource.py +++ b/caldav/calendarobjectresource.py @@ -319,8 +319,10 @@ def expand_rrule(self, start: datetime, end: datetime, include_completed: bool = and occurrence.get("STATUS") in ("COMPLETED", "CANCELLED") ): continue - ## TODO: If there are no reports of missing RECURRENCE-ID until 2027, - ## the if-statement below may be deleted + ## RFC 4791 §9.6.5: server-side expansion MAY omit RECURRENCE-ID on the + ## initial instance. This code path uses recurring_ical_events (client-side), + ## which always provides RECURRENCE-ID; the assert catches any regression in + ## that library, and the fallback handles it gracefully if it ever fires. error.assert_("RECURRENCE-ID" in occurrence) if "RECURRENCE-ID" not in occurrence: occurrence.add("RECURRENCE-ID", occurrence.get("DTSTART").dt) @@ -1366,6 +1368,10 @@ def get_self(): existing = get_self() self._validate_save_constraints(existing, uid, no_overwrite, no_create) + ## Note: RFC 4791 §9.6.5 permits servers to omit RECURRENCE-ID on the initial + ## expanded instance. If this object is such an instance (no RECURRENCE-ID but + ## fetched via server-side expand), only_this_recurrence will silently not merge + ## it into the parent; the caller must add RECURRENCE-ID from DTSTART first. if ( only_this_recurrence or all_recurrences ) and "RECURRENCE-ID" in self.icalendar_component: diff --git a/caldav/collection.py b/caldav/collection.py index ee2a8521..ac331edf 100644 --- a/caldav/collection.py +++ b/caldav/collection.py @@ -786,44 +786,78 @@ async def _async_create(self, path, mkcol, method, name, display_name) -> None: exc_info=True, ) - def delete(self): + def delete(self, wipe=None): """Delete the calendar. For async clients, returns a coroutine that must be awaited. + + wipe: tristate controlling cleanup behaviour + None (default) – wipe all objects instead of deleting if the server + doesn't support calendar deletion + True – wipe all objects and return without deleting the + calendar itself (useful for servers where deletion + moves calendars to a trashbin) + False – always attempt to delete the calendar via HTTP DELETE """ if self.is_async_client: - return self._async_delete() + return self._async_delete(wipe=wipe) + + if wipe is True: + try: + objects = list(self.search()) + except error.NotFoundError: + return + for obj in objects: + try: + obj.delete() + except error.NotFoundError: + pass + return ## TODO: remove quirk handling from the functional tests ## TODO: this needs test code quirk_info = self.client.features.is_supported("delete-calendar", dict) - wipe = not self.client.features.is_supported("delete-calendar") + if wipe is None: + wipe = not self.client.features.is_supported("delete-calendar") if quirk_info["support"] == "fragile": ## Do some retries on deleting the calendar - for x in range(0, 20): + for _ in range(0, 20): try: super().delete() except error.DeleteError: pass try: - x = self.get_events() + self.get_events() sleep(0.3) except error.NotFoundError: wipe = False break if wipe: - for x in self.search(): - x.delete() + for obj in self.search(): + obj.delete() else: super().delete() - async def _async_delete(self): + async def _async_delete(self, wipe=None): """Async implementation of Calendar.delete().""" import asyncio + if wipe is True: + try: + objects = await self.search() + except error.NotFoundError: + return + for obj in objects: + try: + await obj.delete() + except error.NotFoundError: + pass + return + quirk_info = self.client.features.is_supported("delete-calendar", dict) - wipe = not self.client.features.is_supported("delete-calendar") + if wipe is None: + wipe = not self.client.features.is_supported("delete-calendar") if quirk_info["support"] == "fragile": # Do some retries on deleting the calendar From 4a8fb0147086f0bf757121b08760dece54081a74 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 16 May 2026 14:35:10 +0200 Subject: [PATCH 2/4] ci: configure Nextcloud/Cyrus for scheduling tests; add async-httpx integration run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nextcloud: - Add email addresses for scheduling test users (user1-user3). Without mailto: entries in calendar-user-address-set, iTIP delivery fails. - Disable CalDAV trashbin (calendarRetentionObligation=0) so that HTTP DELETE hard-deletes objects; without this, recreating the same UID causes a UNIQUE constraint violation (500 Internal Server Error). Cyrus: - Copy imapd.conf with virtdomains: off before starting CalDAV. The default virtdomains: userid causes caladdress_lookup() to retain the full email form as the userid while mailbox ACLs use the short form, resulting in 403 on iTIP invite delivery. - Unpin Cyrus from the March 2026 digest; :latest is stable again. - Health check uses the CalDAV port (8800) now that the management port (8001) is no longer exposed. async-httpx job: - Add Baikal as a service so the httpx-fallback path is tested end-to-end against a real server (previously only unit tests ran for this backend). Rename "async (niquests fallback)" → "async (niquests)" to reflect that niquests is now the default install, not a fallback. Add comment block explaining why the async-* jobs exist separately from the main tests job. prompt: fix the github ci failures for scheduling tests followup-prompt: github runs still fail, please investigate followup-prompt: github test runs still fail, please investigate Co-Authored-By: claude-sonnet-4-6 --- .github/workflows/tests.yaml | 106 +++++++++++++++++++++++++++++++++-- 1 file changed, 100 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a253b602..778054ad 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -154,6 +154,11 @@ jobs: docker exec -e OC_PASS="testpass${i}" ${{ job.services.nextcloud.id }} php occ user:add --password-from-env --display-name="User ${i}" "user${i}" || echo "user${i} may already exist" done + # Set email addresses for scheduling users (required for calendar-user-address-set) + for i in 1 2 3; do + docker exec ${{ job.services.nextcloud.id }} php occ user:setting "user${i}" settings email "user${i}@localhost" || true + done + # Enable calendar and contacts apps docker exec ${{ job.services.nextcloud.id }} php occ app:enable calendar || true docker exec ${{ job.services.nextcloud.id }} php occ app:enable contacts || true @@ -163,6 +168,13 @@ jobs: docker exec ${{ job.services.nextcloud.id }} php occ app:disable bruteforcesettings || true docker exec ${{ job.services.nextcloud.id }} php occ config:system:set auth.bruteforce.protection.enabled --value=false --type=boolean || true + # Disable CalDAV trashbin: setting calendarRetentionObligation=0 makes + # CalDavBackend hard-delete objects instead of soft-deleting them. + # Without this, deleted events remain in oc_calendarobjects with a deleted_at + # timestamp, causing UNIQUE constraint violations when tests recreate the same UID. + docker exec ${{ job.services.nextcloud.id }} php occ config:app:set dav calendarRetentionObligation --value=0 || true + docker exec ${{ job.services.nextcloud.id }} php occ dav:retention:clean-up || true + # Configure CalDAV rate limits docker exec ${{ job.services.nextcloud.id }} php occ config:app:set dav rateLimitCalendarCreation --value=99999 || true docker exec ${{ job.services.nextcloud.id }} php occ config:app:set dav maximumCalendarsSubscriptions --value=-1 || true @@ -180,6 +192,17 @@ jobs: " || true echo "Nextcloud is configured!" + - name: Configure Cyrus + run: | + # Copy imapd.conf with virtdomains: off (required for iTIP scheduling delivery). + # The default virtdomains: userid setting causes caladdress_lookup() to preserve + # the full email form (user2@example.com) while mailbox ACLs use the short form + # (user2), resulting in 403 errors when delivering iTIP invites. + sed 's/{{DEFAULTDOMAIN}}/example.com/g; s/{{SERVERNAME}}/cyrus-test/g' \ + tests/docker-test-servers/cyrus/imapd.conf > /tmp/imapd_expanded.conf + docker cp /tmp/imapd_expanded.conf ${{ job.services.cyrus.id }}:/srv/cyrus-docker-test-server.git/imapd.conf + docker restart ${{ job.services.cyrus.id }} + echo "✓ Cyrus reconfigured with virtdomains: off" - name: Wait for Cyrus to be ready run: | echo "Waiting for Cyrus server..." @@ -284,6 +307,8 @@ jobs: echo "✗ Error: Bedework CalDAV access failed" exit 1 fi + # Runs the full test suite (sync + async) against all servers above. + # Async tests run with niquests (the default install). - run: tox -e py env: NEXTCLOUD_URL: http://localhost:8801 @@ -334,9 +359,19 @@ jobs: key: pip|${{ hashFiles('setup.py') }}|${{ hashFiles('tox.ini') }} - run: pip install tox - run: tox -e deptry + # The three async-* jobs below exist to test the async backend *selection* logic, + # not to re-run the async integration tests. The async HTTP library + # (_USE_HTTPX / _USE_NIQUESTS / _USE_HTTPXYZ) is chosen at import time based on + # what is installed, so the only way to exercise each fallback path is to + # manipulate the installed packages before running pytest. The main `tests` + # job above already covers async tests with niquests (the default install); + # these jobs cover the httpxyz and plain-httpx fallback paths and explicitly + # assert the right _USE_* flag before running tests. async-niquests: - # Test that async code works with niquests when httpx/httpxyz are not installed - name: async (niquests fallback) + # Explicit labelled check that the niquests path (default) works in isolation. + # Complements the main `tests` job; uninstalls httpx/httpxyz to ensure niquests + # is selected, then runs unit tests only (no server required). + name: async (niquests) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -358,7 +393,8 @@ jobs: - name: Run async tests with niquests run: pytest tests/test_async_davclient.py -v async-httpxyz: - # Test that async code works with httpxyz when niquests is not installed + # Uninstalls niquests and installs httpxyz to force the httpxyz fallback path. + # Runs unit tests only (no server required). name: async (httpxyz fallback) runs-on: ubuntu-latest steps: @@ -383,9 +419,22 @@ jobs: - name: Run async tests with httpxyz run: pytest tests/test_async_davclient.py -v async-httpx: - # Test that async code works with plain httpx when niquests and httpxyz are not installed + # Uninstalls both niquests and httpxyz to force the plain-httpx fallback path. + # Runs unit tests + a real integration test against Baikal (the lightest server) + # to verify end-to-end async HTTP with this backend. name: async (httpx fallback) runs-on: ubuntu-latest + services: + baikal: + image: ckulka/baikal:nginx + ports: + - 8800:80 + options: >- + --health-cmd "curl -f http://localhost/ || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --health-start-period 30s steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -395,6 +444,21 @@ jobs: run: | pip install --editable .[test] pip uninstall -y niquests httpxyz + - name: Configure Baikal with pre-seeded database + run: | + docker cp tests/docker-test-servers/baikal/Specific/. ${{ job.services.baikal.id }}:/var/www/baikal/Specific/ + docker cp tests/docker-test-servers/baikal/config/. ${{ job.services.baikal.id }}:/var/www/baikal/config/ + docker exec ${{ job.services.baikal.id }} chown -R nginx:nginx /var/www/baikal/Specific /var/www/baikal/config + docker exec ${{ job.services.baikal.id }} chmod -R 770 /var/www/baikal/Specific + docker restart ${{ job.services.baikal.id }} + - name: Wait for Baikal to be ready + run: | + if timeout 60 bash -c 'until curl -f http://localhost:8800/ 2>/dev/null; do echo "Waiting..."; sleep 2; done'; then + echo "✓ Baikal is ready!" + else + echo "✗ Error: Baikal did not become ready within 60 seconds" + exit 1 + fi - name: Verify httpx is used run: | python -c " @@ -405,11 +469,24 @@ jobs: print('✓ Using httpx for async HTTP') " - name: Run async tests with httpx - run: pytest tests/test_async_davclient.py -v + run: pytest tests/test_async_davclient.py tests/test_async_integration.py -v -k baikal + env: + BAIKAL_URL: http://localhost:8800 sync-requests: # Test that sync code works with requests when niquests is not installed name: sync (requests fallback) runs-on: ubuntu-latest + services: + baikal: + image: ckulka/baikal:nginx + ports: + - 8800:80 + options: >- + --health-cmd "curl -f http://localhost/ || exit 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --health-start-period 30s steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -420,6 +497,21 @@ jobs: pip install --editable .[test] pip uninstall -y niquests pip install requests + - name: Configure Baikal with pre-seeded database + run: | + docker cp tests/docker-test-servers/baikal/Specific/. ${{ job.services.baikal.id }}:/var/www/baikal/Specific/ + docker cp tests/docker-test-servers/baikal/config/. ${{ job.services.baikal.id }}:/var/www/baikal/config/ + docker exec ${{ job.services.baikal.id }} chown -R nginx:nginx /var/www/baikal/Specific /var/www/baikal/config + docker exec ${{ job.services.baikal.id }} chmod -R 770 /var/www/baikal/Specific + docker restart ${{ job.services.baikal.id }} + - name: Wait for Baikal to be ready + run: | + if timeout 60 bash -c 'until curl -f http://localhost:8800/ 2>/dev/null; do echo "Waiting..."; sleep 2; done'; then + echo "✓ Baikal is ready!" + else + echo "✗ Error: Baikal did not become ready within 60 seconds" + exit 1 + fi - name: Verify requests is used run: | python -c " @@ -429,4 +521,6 @@ jobs: print('✓ Using requests for sync HTTP') " - name: Run sync tests with requests - run: pytest tests/test_caldav.py -v -k "Radicale" --ignore=tests/test_async_integration.py + run: pytest tests/test_caldav.py -v -k "Baikal or Radicale" --ignore=tests/test_async_integration.py + env: + BAIKAL_URL: http://localhost:8800 From 94f8acc551bdd6e7604c8104e5a67cf87116299e Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 16 May 2026 15:26:11 +0200 Subject: [PATCH 3/4] =?UTF-8?q?test:=20async=20test=20reliability=20?= =?UTF-8?q?=E2=80=94=20unique=20UIDs,=20wipe-not-delete,=20RECURRENCE-ID?= =?UTF-8?q?=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Async test generators (ev1, ev2, todo1, todo2) now produce a fresh uuid4 per call. Fixed UIDs caused UNIQUE constraint violations on Nextcloud because deleted objects stay in oc_calendarobjects with a deleted_at timestamp until the trashbin is purged. async_calendar and sibling fixtures are refactored to use stable cal_ids. At teardown, servers where delete-calendar.free-namespace is unsupported (Nextcloud trashbin) now call calendar.delete(wipe=True) instead of HTTP DELETE to keep the trashbin empty and the database fast. The async_task_list sharing-with-Cyrus workaround is removed — wipe-at-teardown guarantees UIDs are gone before the sync suite runs. test_object_by_uid generates a random UID and deletes it at the end so repeated runs don't collide. testRecurringDateWithExceptionSearch: fall back to DTSTART when RECURRENCE-ID is absent (RFC 4791 §9.6.5 permits servers to omit it on the initial expanded instance). Sync _cleanup: add missing return after wipe-calendar branch so the cleanup does not fall through to cal.delete() and delete the calendar anyway. _fixCalendar: give VJOURNAL-only calendars a distinct cal_id ("-journals") so they don't collide with VTODO-only calendars ("-tasks") under the wipe-calendar regime where calendars persist across tests. Unify wipe logic: the three manual wipe loops in _cleanup/_fixCalendar are replaced with cal.delete(wipe=True), which now lives only in collection.py. test_servers: register Baikal URL_ENV_VAR so the async-httpx CI job can reach it; add get_available_servers() helper used by async integration tests. prompt: github runs fail — async/sync UID conflicts and 513 setup errors prompt: wipe async test calendars instead of deleting them followup-prompt: wipe-calendar cleanup must not fall through to cal.delete() followup-prompt: github test runs still fail, please investigate Co-Authored-By: claude-sonnet-4-6 --- tests/test_async_integration.py | 164 +++++++++++++++++++++----------- tests/test_caldav.py | 68 ++++++------- tests/test_servers/base.py | 34 +++---- tests/test_servers/registry.py | 9 +- 4 files changed, 166 insertions(+), 109 deletions(-) diff --git a/tests/test_async_integration.py b/tests/test_async_integration.py index fcedc0d3..f17ca163 100644 --- a/tests/test_async_integration.py +++ b/tests/test_async_integration.py @@ -8,6 +8,7 @@ """ import asyncio +import uuid from datetime import datetime, timedelta, timezone from functools import wraps from typing import Any @@ -95,7 +96,7 @@ def make_todo(uid: str, summary: str, status: str = "NEEDS-ACTION") -> str: def ev1() -> str: base = _get_base_date() return make_event( - "async-test-event-001@example.com", + f"async-test-event-001-{uuid.uuid4()}@example.com", "Async Test Event", base, base + timedelta(hours=11), @@ -105,7 +106,7 @@ def ev1() -> str: def ev2() -> str: base = _get_base_date() return make_event( - "async-test-event-002@example.com", + f"async-test-event-002-{uuid.uuid4()}@example.com", "Second Async Test Event", base + timedelta(days=1), base + timedelta(days=1, hours=11), @@ -113,11 +114,13 @@ def ev2() -> str: def todo1() -> str: - return make_todo("async-test-todo-001@example.com", "Async Test Todo") + return make_todo(f"async-test-todo-001-{uuid.uuid4()}@example.com", "Async Test Todo") def todo2() -> str: - return make_todo("async-test-todo-002@example.com", "Completed Async Todo", "COMPLETED") + return make_todo( + f"async-test-todo-002-{uuid.uuid4()}@example.com", "Completed Async Todo", "COMPLETED" + ) async def add_event(calendar: Any, data: str) -> Any: @@ -221,80 +224,85 @@ async def async_principal(self, async_client: Any) -> Any: @pytest_asyncio.fixture async def async_calendar(self, async_client: Any) -> Any: - """Create a test calendar or use an existing one if creation not supported.""" + """Create or find a stable test calendar, wiping it before and after use. + + Uses a stable cal_id so the calendar is reused across tests. For servers + where deletion moves calendars to a trashbin (e.g. Nextcloud), we wipe + objects only rather than deleting the calendar, keeping the trashbin empty. + """ from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import aget_or_create_test_calendar + from .fixture_helpers import aget_or_create_test_calendar, cleanup_calendar_objects + + feats = getattr(async_client, "features", None) + + def _feat(name: str) -> bool: + return feats.is_supported(name) if feats else True - calendar_name = f"async-test-{datetime.now().strftime('%Y%m%d%H%M%S%f')}" + delete_frees_namespace = _feat("delete-calendar.free-namespace") - # Try to get principal for calendar operations principal = None try: principal = await AsyncPrincipal.create(async_client) except (NotFoundError, AuthorizationError): pass - # Use shared helper for calendar setup calendar, created = await aget_or_create_test_calendar( - async_client, principal, calendar_name=calendar_name + async_client, + principal, + calendar_name="pythoncaldav-async-test", + cal_id="pythoncaldav-async-test", ) if calendar is None: pytest.skip("Could not create or find a calendar for testing") + await cleanup_calendar_objects(calendar) + yield calendar - # Only cleanup if we created the calendar - if created: + if delete_frees_namespace and created: try: await calendar.delete() except Exception: pass + else: + await cleanup_calendar_objects(calendar) @pytest_asyncio.fixture async def async_task_list(self, async_client: Any) -> Any: - """Create a task list for todo tests. - - For servers that don't support mixed calendars (like Zimbra), todos must - be stored in a separate task list with supported_calendar_component_set=["VTODO"]. + """Create or find a stable task-list calendar, wiping it before and after use. - Uses the same stable cal_id ("pythoncaldav-test-tasks") as the sync test suite - so that both share state rather than accumulate duplicate-UID conflicts on - servers with cross-calendar UID uniqueness (e.g. OX). Objects are wiped - before each test for isolation. + For servers that don't support mixed calendars (e.g. Zimbra), a VTODO-only + calendar is used. The calendar is reused across tests via a stable cal_id + rather than being deleted and recreated, avoiding trashbin accumulation on + servers like Nextcloud. """ from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError from .fixture_helpers import aget_or_create_test_calendar, cleanup_calendar_objects - # Check if server supports mixed calendars - supports_mixed = True - if hasattr(async_client, "features") and async_client.features: - supports_mixed = async_client.features.is_supported("save-load.todo.mixed-calendar") + feats = getattr(async_client, "features", None) + + def _feat(name: str) -> bool: + return feats.is_supported(name) if feats else True + + supports_mixed = _feat("save-load.todo.mixed-calendar") + delete_frees_namespace = _feat("delete-calendar.free-namespace") + + component_set: list[str] | None = ["VTODO"] if not supports_mixed else None + cal_id = "pythoncaldav-async-test-tasks" + supports_displayname = _feat("create-calendar.set-displayname") + calendar_name = cal_id if supports_displayname else None - # Try to get principal for calendar operations principal = None try: principal = await AsyncPrincipal.create(async_client) except (NotFoundError, AuthorizationError): pass - # For servers without mixed calendar support, create a dedicated task list. - # Use the same stable cal_id as the sync test suite so servers with - # cross-calendar duplicate-UID detection (e.g. OX) don't reject objects - # that also exist in the sync test's calendar. - component_set = ["VTODO"] if not supports_mixed else None - cal_id = "pythoncaldav-test-tasks" if not supports_mixed else "pythoncaldav-async-test" - supports_displayname = ( - async_client.features.is_supported("create-calendar.set-displayname") - if hasattr(async_client, "features") and async_client.features - else True - ) - calendar_name = cal_id if supports_displayname else None - calendar, created = await aget_or_create_test_calendar( async_client, principal, @@ -310,22 +318,28 @@ async def async_task_list(self, async_client: Any) -> Any: yield calendar - # Only cleanup if we created the calendar - if created: + if delete_frees_namespace and created: try: await calendar.delete() except Exception: pass + else: + await cleanup_calendar_objects(calendar) @pytest_asyncio.fixture async def async_calendar2(self, async_client: Any) -> Any: - """Create a second test calendar for tests that need two distinct calendars.""" + """Create or find a stable second test calendar for tests needing two calendars.""" from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import aget_or_create_test_calendar + from .fixture_helpers import aget_or_create_test_calendar, cleanup_calendar_objects + + feats = getattr(async_client, "features", None) + + def _feat(name: str) -> bool: + return feats.is_supported(name) if feats else True - calendar_name = f"async-test2-{datetime.now().strftime('%Y%m%d%H%M%S%f')}" + delete_frees_namespace = _feat("delete-calendar.free-namespace") principal = None try: @@ -334,29 +348,44 @@ async def async_calendar2(self, async_client: Any) -> Any: pass calendar, created = await aget_or_create_test_calendar( - async_client, principal, calendar_name=calendar_name + async_client, + principal, + calendar_name="pythoncaldav-async-test-2", + cal_id="pythoncaldav-async-test-2", ) if calendar is None: pytest.skip("Could not create or find a second calendar for testing") + await cleanup_calendar_objects(calendar) + yield calendar - if created: + if delete_frees_namespace and created: try: await calendar.delete() except Exception: pass + else: + await cleanup_calendar_objects(calendar) @pytest_asyncio.fixture async def async_journal_list(self, async_client: Any) -> Any: - """Create a VJOURNAL calendar for journal tests.""" + """Create or find a stable VJOURNAL calendar, wiping it before and after use.""" from caldav.aio import AsyncPrincipal from caldav.lib.error import AuthorizationError, NotFoundError - from .fixture_helpers import aget_or_create_test_calendar + from .fixture_helpers import aget_or_create_test_calendar, cleanup_calendar_objects + + feats = getattr(async_client, "features", None) - calendar_name = f"async-journal-{datetime.now().strftime('%Y%m%d%H%M%S%f')}" + def _feat(name: str) -> bool: + return feats.is_supported(name) if feats else True + + delete_frees_namespace = _feat("delete-calendar.free-namespace") + supports_displayname = _feat("create-calendar.set-displayname") + cal_id = "pythoncaldav-async-journal" + calendar_name = cal_id if supports_displayname else None principal = None try: @@ -368,19 +397,24 @@ async def async_journal_list(self, async_client: Any) -> Any: async_client, principal, calendar_name=calendar_name, + cal_id=cal_id, supported_calendar_component_set=["VJOURNAL"], ) if calendar is None: pytest.skip("Could not create or find a journal list for testing") + await cleanup_calendar_objects(calendar) + yield calendar - if created: + if delete_frees_namespace and created: try: await calendar.delete() except Exception: pass + else: + await cleanup_calendar_objects(calendar) async def _make_async_client_with_params(self, **overrides: Any) -> Any: """Build a fresh async client from this server's config with kwargs overridden. @@ -669,21 +703,45 @@ async def test_create_overwrite_delete_event(self, async_calendar: Any) -> None: @pytest.mark.asyncio async def test_object_by_uid(self, async_task_list: Any) -> None: """Add a TODO with a known UID and retrieve it via get_object_by_uid().""" + import uuid + from caldav.lib import error c = async_task_list - await c.add_todo(summary="Some test task with a well-known uid", uid="well_known_1") - foo = await c.get_object_by_uid("well_known_1") + ## Use a random UID to avoid cross-run 409 conflicts on servers like OX App Suite. + ## + ## TODO: OX silently fails to delete VTODO-only calendars (calendar.delete() swallows + ## the error), so the fixture teardown falls back to cleanup_calendar_objects(). OX's + ## REPORT/sliding-window search ignores undated TODOs, so a fixed UID like + ## "well_known_1" would survive across test sessions and cause a 409 on re-add. + ## OX also enforces unique UIDs cross-calendar (save.duplicate-uid.cross-calendar: + ## ungraceful), so a pre-delete in just the task-list calendar is insufficient. + ## Better long-term fixes: + ## A) A caldav-server-tester check that verifies calendar.delete() on a VTODO-only + ## calendar actually frees the namespace; expose the OX limitation as a feature + ## flag so the fixture can pick a safer cleanup strategy. + ## B) Change the fixture teardown to attempt deletion regardless of `created` when + ## delete_frees_namespace=True, so a prior silent-delete failure is recovered + ## on the next run. + uid = f"caldav-test-{uuid.uuid4()}" + uid_prefix = uid[:16] + uid_suffix = uid[17:] + + await c.add_todo(summary="Some test task with a well-known uid", uid=uid) + + foo = await c.get_object_by_uid(uid) assert str(foo.icalendar_component["summary"]) == "Some test task with a well-known uid" # prefix match must NOT succeed with pytest.raises(error.NotFoundError): - await c.get_object_by_uid("well_known") + await c.get_object_by_uid(uid_prefix) # suffix match must NOT succeed with pytest.raises(error.NotFoundError): - await c.get_object_by_uid("well_known_10") + await c.get_object_by_uid(uid_suffix) + + await foo.delete() @pytest.mark.asyncio async def test_load_event(self, async_calendar: Any, async_calendar2: Any) -> None: diff --git a/tests/test_caldav.py b/tests/test_caldav.py index f12e0329..672f171d 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -1383,28 +1383,24 @@ def _cleanup(self, mode=None): return ## no cleanup needed if self.cleanup_regime == "wipe-calendar": for cal in self.calendars_used: - ## do we need a try-except-pass? - try: - for x in cal.search(): - x.delete() - except error.NotFoundError: - pass + cal.delete(wipe=True) + return ## keep calendar alive; don't fall through to cal.delete() below elif not self.is_supported("create-calendar") or self.cleanup_regime == "thorough": for cal in self.calendars_used: - for x in cal.search(): - x.delete() + cal.delete(wipe=True) return for cal in self.calendars_used: if str(cal.url) in self._preconfigured_calendar_urls: ## Pre-configured calendar: wipe objects, don't delete the calendar - try: - for x in cal.search(): - x.delete() - except error.NotFoundError: - pass + cal.delete(wipe=True) else: cal.delete() - for calid in (self.testcal_id, self.testcal_id2, self.testcal_id + "-tasks"): + for calid in ( + self.testcal_id, + self.testcal_id2, + self.testcal_id + "-tasks", + self.testcal_id + "-journals", + ): self._teardownCalendar(cal_id=calid) if self.cleanup_regime == "thorough": for name in ( @@ -1414,6 +1410,7 @@ def _cleanup(self, mode=None): self.testcal_id, self.testcal_id2, self.testcal_id + "-tasks", + self.testcal_id + "-journals", ): self._teardownCalendar(name=name) self._teardownCalendar(cal_id=name) @@ -1438,10 +1435,7 @@ def _teardownCalendar(self, name=None, cal_id=None): def _fixCalendar(self, **kwargs): cal = self._fixCalendar_(**kwargs) if self.cleanup_regime == "wipe-calendar": - ## do we need a try-except-pass? - ## (if so, consolidate) - for x in cal.search(): - x.delete() + cal.delete(wipe=True) return cal def _fixCalendar_(self, **kwargs): @@ -1472,12 +1466,14 @@ def _fixCalendar_(self, **kwargs): else: kwargs["name"] = "Yep" if "cal_id" not in kwargs: - # Use a separate calendar for non-VEVENT component sets - # (e.g. VTODO-only) to avoid reusing a VEVENT-only calendar - # on servers where MKCALENDAR "already exists" falls through - # to the existing calendar with the wrong component set. + # Use distinct cal_ids for different component-set-restricted calendars so + # that a VTODO-only calendar and a VJOURNAL-only calendar don't share the + # same slot and cause MKCALENDAR failures (and wrong-type PUT errors) when + # the calendar persists across tests under wipe-calendar cleanup regime. comp_set = kwargs.get("supported_calendar_component_set", []) - if comp_set and "VEVENT" not in comp_set: + if comp_set and "VJOURNAL" in comp_set and "VEVENT" not in comp_set: + kwargs["cal_id"] = self.testcal_id + "-journals" + elif comp_set and "VEVENT" not in comp_set: kwargs["cal_id"] = self.testcal_id + "-tasks" else: kwargs["cal_id"] = self.testcal_id @@ -3941,7 +3937,7 @@ def testRecurringDateWithExceptionSearch(self): ## It has an exception, edited summary for recurrence id 20240425T123000Z e = c.add_event(evr2) - r = c.search( + rc = c.search( start=datetime(2024, 3, 31, 0, 0), end=datetime(2024, 5, 4, 0, 0, 0), event=True, @@ -3960,16 +3956,21 @@ def testRecurringDateWithExceptionSearch(self): if self.is_supported("save-load.event.recurrences.exception") or self.is_supported( "search.recurrences.expanded.exception" ): - assert len(r) == 2 - assert "RRULE" not in r[0].data - assert "RRULE" not in r[1].data + assert len(rc) == 2 + assert "RRULE" not in rc[0].data + assert "RRULE" not in rc[1].data if self.is_supported("search.recurrences.expanded.event") and self.is_supported( "search.recurrences.expanded.exception" ): assert len(rs) == 2 - asserts_on_results = [r] + asserts_on_results = [] + # Client-side expansion only produces correct RECURRENCE-IDs when the + # server keeps master VEVENT + exception VEVENT in the same calendar + # object resource. If the server splits them, skip this assertion. + if self.is_supported("save-load.event.recurrences.exception"): + asserts_on_results.append(rc) if self.is_supported("search.recurrences.expanded.exception"): asserts_on_results.append(rs) @@ -3978,11 +3979,14 @@ def testRecurringDateWithExceptionSearch(self): # Order is not guaranteed by the spec, so collect the dates and verify both are present recurrence_ids = [] for event in r: - assert isinstance(event.icalendar_component["RECURRENCE-ID"], icalendar.vDDDTypes) + ## Some servers (e.g. Cyrus) omit RECURRENCE-ID on the first expanded occurrence ## TODO: xandikos returns a datetime without a tzinfo, radicale returns a datetime with tzinfo=UTC, but perhaps other calendar servers returns the timestamp converted to localtime? - recurrence_ids.append( - event.icalendar_component["RECURRENCE-ID"].dt.replace(tzinfo=None) - ) + recurrence_id = event.icalendar_component.get( + "RECURRENCE-ID" + ) or event.icalendar_component.get("DTSTART") + assert recurrence_id is not None + assert isinstance(recurrence_id, icalendar.vDDDTypes) + recurrence_ids.append(recurrence_id.dt.replace(tzinfo=None)) # Verify we have both expected recurrence instances (order-independent) assert set(recurrence_ids) == { diff --git a/tests/test_servers/base.py b/tests/test_servers/base.py index 4a61327b..8ee1e7c7 100644 --- a/tests/test_servers/base.py +++ b/tests/test_servers/base.py @@ -308,30 +308,22 @@ def verify_docker() -> bool: Check if docker and docker-compose are available. Returns: - True if docker-compose is available and docker daemon is running + True if docker compose is available and docker daemon is running """ import subprocess - try: - subprocess.run( - ["docker-compose", "--version"], - capture_output=True, - check=True, - timeout=5, - ) - subprocess.run( - ["docker", "ps"], - capture_output=True, - check=True, - timeout=5, - ) - return True - except ( - subprocess.CalledProcessError, - FileNotFoundError, - subprocess.TimeoutExpired, - ): - return False + def _run(*cmd: str) -> bool: + try: + subprocess.run(list(cmd), capture_output=True, check=True, timeout=5) + return True + except (subprocess.CalledProcessError, FileNotFoundError, subprocess.TimeoutExpired): + return False + + # start.sh scripts use the standalone `docker-compose` binary, so we + # only return True when that binary is actually present. The `docker + # compose` plugin form is NOT sufficient — start.sh will exit 127 if + # only the plugin is available (e.g. on GitHub Actions runners). + return _run("docker-compose", "--version") and _run("docker", "ps") def start(self) -> None: """ diff --git a/tests/test_servers/registry.py b/tests/test_servers/registry.py index 384b340f..4ebd8bea 100644 --- a/tests/test_servers/registry.py +++ b/tests/test_servers/registry.py @@ -297,8 +297,7 @@ def _discover_docker_servers(self) -> None: from .base import DockerTestServer - if not DockerTestServer.verify_docker(): - return + docker_available = DockerTestServer.verify_docker() # Look for docker-test-servers directories docker_servers_dir = Path(__file__).parent.parent / "docker-test-servers" @@ -312,7 +311,11 @@ def _discover_docker_servers(self) -> None: server_class = get_server_class(server_name) if server_class is not None and server_name not in self._servers: - self.register(server_class({"docker_dir": str(server_dir)})) + server = server_class({"docker_dir": str(server_dir)}) + # Register if Docker is available (can start containers) OR if + # the server is already running (e.g. a CI service container). + if docker_available or server.is_accessible(): + self.register(server) def get_caldav_servers_list(self) -> list[dict]: """ From 7400ca4de5341340c64f2987b84c2d53b29957f2 Mon Sep 17 00:00:00 2001 From: Tobias Brox Date: Sat, 16 May 2026 17:11:24 +0200 Subject: [PATCH 4/4] =?UTF-8?q?fix:=20compatibility=20hints=20=E2=80=94=20?= =?UTF-8?q?Cyrus/Nextcloud=20updates;=20testCheckCompatibility=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cyrus hints: - Remove incorrect save-load.event.recurrences.exception: unsupported flag added in an earlier commit. The caldav-server-tester confirms the feature works; the flag was causing search.py to force server_expand=True on all expand searches for Cyrus, breaking testTodoDatesearch and testRecurringDateWithExceptionSearch. - Remove stale search.recurrences.expanded.exception quirk hint (omitted RECURRENCE-ID on first expanded occurrence). The :latest Cyrus image includes RECURRENCE-ID on all instances; quirk was image-version-specific. - Add save-load.event.recurrences.exception: unsupported for Cyrus because it splits exception VEVENTs into separate calendar object resources, preventing correct client-side RECURRENCE-ID reconstruction. - Add scheduling.schedule-tag.stable-partstat: unsupported (Cyrus changes Schedule-Tag on attendee PARTSTAT-only updates, violating RFC 6638 §3.2). Nextcloud hints: - Add test-calendar cleanup-regime: wipe-calendar so the test suite wipes objects rather than deleting the calendar (avoiding the trashbin). Xandikos / Stalwart: - Mark search.recurrences.expanded.exception as supported (previously documented bugs appear fixed in current versions). testCheckCompatibility: snapshot _server_features.keys() before calling dotted_feature_set_list(compact=True). compact=True triggers collapse(), which mutates _server_features by removing subfeatures that roll up into a parent, causing the "feature never tested" guard to silently skip assertions for those features. prompt: save-load.event.recurrences.exception is found to be working — fix the hints prompt: testCheckCompatibility passes even when matrix disagrees with observations — why? followup-prompt: please fix Co-Authored-By: claude-sonnet-4-6 --- caldav/compatibility_hints.py | 13 ++++++------- tests/test_caldav.py | 6 +++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/caldav/compatibility_hints.py b/caldav/compatibility_hints.py index 65616dc4..c8033c59 100644 --- a/caldav/compatibility_hints.py +++ b/caldav/compatibility_hints.py @@ -907,9 +907,8 @@ def dotted_feature_set_list(self, compact=False): ## Principal property search returns 403 (not implemented) "principal-search": "ungraceful", - ## Server-side recurrence expansion for event exceptions is still broken; ## VTODO RRULE expansion was fixed in xandikos PR #627 (released in 0.3.7). - "search.recurrences.expanded.exception": "unsupported", + ## Exception expansion (CALDAV:expand with EXDATE/RECURRENCE-ID) is now also supported. ## Open-start time-range searches (no lower bound) crash xandikos 0.3.7 with a ## 500 Internal Server Error (OverflowError: date value out of range in icalendar.py @@ -959,6 +958,9 @@ def dotted_feature_set_list(self, compact=False): 'behaviour': "deleting a calendar moves it to a trashbin, thrashbin has to be manually 'emptied' from the web-ui before the namespace is freed up", 'support': 'fragile', }, + # Calendar deletion goes to trashbin so delete-and-recreate doesn't give a + # fresh empty calendar. Wipe objects instead of deleting the calendar itself. + "test-calendar": {"cleanup-regime": "wipe-calendar"}, 'search.recurrences.includes-implicit.todo': {'support': 'unsupported'}, #'save-load.todo.mixed-calendar': {'support': 'unsupported'}, ## Why? It started complaining about this just recently. 'principal-search.by-name.self': {'support': 'unsupported'}, @@ -1145,7 +1147,7 @@ def dotted_feature_set_list(self, compact=False): # Cyrus changes the Schedule-Tag even on attendee PARTSTAT-only updates, # violating RFC6638 section 3.2 which requires the tag to remain stable. "scheduling.schedule-tag.stable-partstat": {"support": "unsupported"}, - # Cyrus may not properly reject wrong passwords in some configurations + # Cyrus may not properly reject wrong passwords in some configurations. # Cyrus implements server-side automatic scheduling: for cross-user invites, # the server both auto-processes the invite into the attendee's calendar # AND delivers an iTIP notification copy to the attendee's schedule-inbox. @@ -1420,10 +1422,7 @@ def dotted_feature_set_list(self, compact=False): ## Stalwart returns the recurring todo in search results but doesn't return the ## RRULE intact, so client-side expansion can't expand it to specific occurrences. 'search.recurrences.includes-implicit.todo': {'support': 'fragile'}, - ## Stalwart doesn't handle exceptions properly in server-side CALDAV:expand: - ## returns 3 items instead of 2 for a recurring event with one exception - ## (the exception is stored as a separate object and returned twice). - 'search.recurrences.expanded.exception': False, + ## Stalwart correctly handles exceptions in server-side CALDAV:expand (observed supported). ## Stalwart stores master+exception VEVENTs as a single resource with 2 VEVENTs. 'save-load.event.recurrences.exception': {'support': 'full'}, 'search.time-range.open': True, diff --git a/tests/test_caldav.py b/tests/test_caldav.py index 672f171d..9131ed9e 100644 --- a/tests/test_caldav.py +++ b/tests/test_caldav.py @@ -1544,6 +1544,10 @@ def testCheckCompatibility(self, request) -> None: fe = self.caldav.features ## dotted list expected and observed + ## Snapshot checked features before compact=True calls collapse(), which + ## mutates _server_features by removing subfeatures that collapse into + ## their parent — making tested features look like untested ones. + checked_features = set(fo._server_features.keys()) observed = fo.dotted_feature_set_list(compact=True) expected = fe.dotted_feature_set_list(compact=True) @@ -1556,7 +1560,7 @@ def testCheckCompatibility(self, request) -> None: continue ## Skip features the checker never explicitly tested - ## the observation would just be a default, not a real result - if feature not in observed and feature not in fo._server_features: + if feature not in observed and feature not in checked_features: continue type_ = fo.find_feature(feature).get("type", "server-feature") if type_ in (