diff --git a/README.md b/README.md index a496c5df..5aafbf71 100644 --- a/README.md +++ b/README.md @@ -1,411 +1,90 @@ # Horde ActiveSync -PHP library implementing the Microsoft Exchange ActiveSync (EAS) protocol. It -decodes WBXML requests from mobile clients, drives synchronization through a -pluggable backend driver, and encodes WBXML responses. +PHP library implementing the Microsoft Exchange ActiveSync (EAS) protocol, +versions **2.5 through 16.1**. It decodes WBXML requests from mobile clients, +drives synchronization through a pluggable backend driver, and encodes WBXML +responses — including streamed `Sync` response delivery for clients with hard +read timeouts. In a typical Horde deployment, this package is the **protocol engine**. The **data backend** lives in `horde/core` as `Horde_Core_ActiveSync_Driver`, which talks to IMAP (mail), Kronolith (calendar), Turba (contacts), Nag (tasks), and -Mnemo (notes). +Mnemo (notes). The library itself has no Horde application dependencies at +runtime and can be embedded with a custom driver. -Open work and the Horde 6 roadmap are tracked in -[`doc/Horde/ActiveSync/todo.md`](doc/Horde/ActiveSync/todo.md). +## Documentation -## How it fits together +The documentation is split by audience: + +| You are… | You want to… | Read | +|----------|--------------|------| +| **Administrator / end user** | Enable ActiveSync, configure protocol versions, streaming, logging, the web server endpoint, and per-user policy | [`doc/configuration.md`](doc/configuration.md) | +| **Library user / integrator** | Embed the library in your own product: server object, driver API, state backends, custom backends | [`doc/integration.md`](doc/integration.md) | +| **ActiveSync developer** | Understand the internals: components, request lifecycle, Sync anatomy, state machine, tests | [`doc/architecture.md`](doc/architecture.md) | + +Cross-cutting references, useful to all three: + +| Topic | Read | +|-------|------| +| Supported EAS protocol versions, negotiation, and what each version adds | [`doc/protocol-versions.md`](doc/protocol-versions.md) | +| Streamed `Sync` response delivery — design, error model, tuning | [`doc/sync-streaming.md`](doc/sync-streaming.md) | +| Open work and the Horde 6 roadmap | [`doc/todo.md`](doc/todo.md) | + +## At a glance ``` -Mobile client (Outlook, iOS Mail, …) +Mobile client (Outlook, iOS Mail, Gmail, …) │ HTTPS POST, WBXML body ▼ Web server → /Microsoft-Server-ActiveSync (rewritten to Horde rpc.php) │ ▼ -Horde_Rpc_ActiveSync +Horde_Rpc_ActiveSync (horde/rpc) │ ▼ -Horde_ActiveSync ← this library +Horde_ActiveSync ← this library ├── Request handlers (Sync, FolderSync, Ping, …) ├── Message classes (Appointment, Mail, Contact, …) ├── WBXML encoder/decoder └── Horde_ActiveSync_Driver_Base (abstract backend API) │ ▼ -Horde_Core_ActiveSync_Driver ← horde/core (Horde deployment) +Horde_Core_ActiveSync_Driver (horde/core, in a Horde deployment) └── Horde_Core_ActiveSync_Connector → Horde apps / IMAP ``` -### Main classes - -| Class | Role | -|-------|------| -| `Horde_ActiveSync` | Server entry point: auth, version negotiation, request dispatch | -| `Horde_ActiveSync_Request_*` | One class per EAS command (`Sync`, `FolderSync`, `Find`, …) | -| `Horde_ActiveSync_Message_*` | Typed WBXML property maps for each item type | -| `Horde_ActiveSync_Driver_Base` | Abstract backend all data access goes through | -| `Horde_ActiveSync_State_Sql` / `_Mongo` | Device state, sync keys, change maps | -| `Horde_ActiveSync_Device` | Per-device metadata (type, policy key, remote wipe) | -| `Horde_ActiveSync_Wbxml_*` | Low-level WBXML encode/decode and protocol logging | - -Message objects are version-aware: constructors accept -`protocolversion` and adjust their property maps for the negotiated EAS level. - -## Protocol versions - -The library defines constants for EAS **2.5**, **12.0**, **12.1**, **14.0**, -**14.1**, **16.0** and **16.1**. - -| Version | Status in this tree | -|---------|---------------------| -| 2.5 – 14.1 | Mature; long-standing Horde support | -| **16.0** | Supported end-to-end for production use (see below) | -| **16.1** | Supported. It extends 16.0 with meeting proposals and account-only wipe (see below) | - -### How version negotiation works - -Two related values matter on each request: - -1. **Server ceiling** (`Horde_ActiveSync::$_maxVersion`, set via - `setSupportedVersion()`). Controls what the server **advertises** in - `OPTIONS` / `MS-ASProtocolVersions` and `MS-Server-ActiveSync`, and which - command sets are available. -2. **Session protocol version** (client header `MS-ASProtocolVersion`, or - `ProtVer` in GET for very old clients). The level actually used for WBXML - encoding/decoding on that request. The client must not exceed what it - offered and what the server supports. - -Clients normally negotiate down: a device that sends `MS-ASProtocolVersion: 16.0` -against a server ceiling of `14.1` will sync at 14.1. - -In a Horde deployment the ceiling is applied in **layers** (see next section). -The library itself only exposes `setSupportedVersion()`; per-user and -per-device policy is implemented in `Horde_Core_ActiveSync_Driver::versionCallback()` -and invoked at the start of every `Horde_ActiveSync::handleRequest()` call, -before authentication completes. - -### Protocol version configuration (Horde) - -Three mechanisms stack together. None of them are personal **preferences** -(users cannot change their own EAS version under Preferences); per-user limits -are **administrator permissions**. - -#### 1. Global ceiling (all users, default) - -Set in Horde administration -> ActiveSync -> *What is the highest version of EAS -that Horde should support?*, or in `conf.php`: - -```php -$conf['activesync']['version'] = '16.1'; -``` - -`Horde_Core_Factory_ActiveSyncServer` calls `setSupportedVersion()` with this -value when the server object is created. This is the baseline for every request. - -#### 2. Per-user ceiling (permissions) - -Default mode: `version_mode` is **`user`** when unset. - -Administrators can assign **Maximum ActiveSync protocol version** -(`horde:activesync:version`) per user or group under Horde administration → -Permissions → ActiveSync. Allowed values: `2.5`, `12.0`, `12.1`, `14.0`, -`14.1`, `16.0`, and `16.1`. - -On each request, `versionCallback()` resolves the authenticated Horde username -(from HTTP Basic credentials, the `User` GET parameter, or the registry) and -reads that permission. If set, it calls `setSupportedVersion()` again for this -request only. - -| Situation | Effective ceiling for this request | -|-----------|-------------------------------------| -| Permission empty / permission tree not defined | Global `conf['activesync']['version']` only | -| User permission **lower** than global (e.g. user `14.1`, global `16.0`) | User value — caps that user below the site default | -| User permission **higher** than global (e.g. user `16.0`, global `14.1`) | User value — can raise the advertised ceiling above the admin default for that user | -| User in multiple groups with different values | **Lowest** (most restrictive) allowed version | - -The last row matters for group-based permissions: if one group allows `16.0` and -another `14.1`, the user syncs at `14.1`. - -#### 3. Per-device ceiling (hook) - -For device-specific policy (pilot devices, problematic clients, lab handsets), -set in `conf.php`: - -```php -$conf['activesync']['version_mode'] = 'device'; -``` - -Then implement `activesync_device_version()` in `config/hooks.php` (see -`vendor/horde/horde/config/hooks.php.dist`): - -```php -public function activesync_device_version($deviceId, $user) -{ - // $deviceId is normalised to uppercase. - $map = [ - 'OLD-OUTLOOK-DEVICE-ID' => '14.1', - 'TEST-IPHONE-ID' => '16.0', - ]; - - return $map[$deviceId] ?? null; -} -``` - -Hook return values: - -| Return | Meaning | -|--------|---------| -| String, e.g. `'16.0'` | Use this ceiling for the device | -| Array of version strings | **Lowest** (most restrictive) entry is used | -| `null`, `false`, `''`, or `-1` | No override; fall back to global / user permission behaviour | - -`DeviceId` must be present in the request (standard on all sync commands). If -the hook is not defined or returns no override, behaviour depends on -`version_mode`: in **`device`** mode with no hook result, no permission override -is applied; switch back to **`user`** mode to use group permissions as the -primary per-principal control. - -#### Choosing `user` vs `device` mode - -| `version_mode` | Source of per-request override | -|----------------|-------------------------------| -| `user` (default) | `horde:activesync:version` permission | -| `device` | `activesync_device_version` hook | - -Only one mode is active per installation. Use **permissions** for -account/class-of-user policy; use the **hook** when the device ID is the right -key (e.g. force an old Outlook build to `14.1` while everyone else stays on -`16.0`). - -#### Custom / non-Horde drivers - -Any backend driver may implement `versionCallback(Horde_ActiveSync $server)` the -same way as `Horde_Core_ActiveSync_Driver`. The library checks -`is_callable([$driver, 'versionCallback'])` on every request. Integrators -without Horde permissions can set policy entirely inside that method. - -## EAS commands - -Commands advertised for EAS ≥ 12.0 (including 16.0): - -`Sync`, `SendMail`, `SmartForward`, `SmartReply`, `GetAttachment`, -`GetHierarchy`, `CreateCollection`, `DeleteCollection`, `MoveCollection`, -`FolderSync`, `FolderCreate`, `FolderDelete`, `FolderUpdate`, `MoveItems`, -`GetItemEstimate`, `MeetingResponse`, `Search`, `Settings`, `Ping`, -`ItemOperations`, `Provision`, `ResolveRecipients`, `ValidateCert`, **`Find`** - -EAS 2.5 omits `Settings`, `ItemOperations`, and `Find`. - -`OPTIONS` and **Autodiscover** are handled outside the normal command loop via -`Horde_Rpc_ActiveSync`. - -## Implemented feature set - -### Mail (EAS `Email` class) - -- Folder hierarchy sync, message sync, flags, categories -- Send, reply, forward (`SendMail`, `SmartReply`, `SmartForward`) -- Attachments (`GetAttachment`, `ItemOperations:Fetch`) -- Meeting requests embedded in mail -- Body preferences and truncation (`AirSyncBase:Body`) -- Draft folder sync; **EAS 16.0** draft edit detection (`CHANGE_TYPE_DRAFT`) - and draft send via `POOMMAIL2:Send` -- **EAS 16.0** `Forwardee` objects on forward/reply -- GAL search (`Search`, `ResolveRecipients`) -- **EAS 16.0** mailbox `Find` with KQL parser (`Horde_ActiveSync_Find_Kql`) - supporting boolean operators, property restrictions, dates, and size - -### Calendar (EAS `Calendar` class) - -Handled in `horde/kronolith` (`Kronolith_Event::fromASAppointment()` / -`toASAppointment()`) with logic in this library's `Message/Appointment` and -`Message/Exception` classes. - -- Create, update, delete appointments; recurrence and exceptions -- Attendees, reminders, categories, sensitivity, busy status -- Meeting responses (`MeetingResponse`) -- **EAS 16.0 instance model**: modified recurrence instances sync as separate - items with top-level `InstanceId`; masters export only deleted-instance - exceptions; `ClientUid` round-trip; bound exceptions visible in initial sync -- **EAS 16.0** `AirSyncBase:Location` (display name + coordinates) -- **EAS 16.0** inbound validation strips forbidden top-level fields (`uid`, - `dtstamp`, `organizername`, `organizeremail`) instead of rejecting the item -- All-day event rules for 16.0 (date-only, no spurious timezone conversion) - -### Contacts (`Contacts`) - -- Personal address books and GAL -- Photo support via `ResolveRecipients` / Find picture options -- Standard vCard-style field mapping - -### Tasks (`Tasks`) and Notes (`Notes`) - -- Full folder sync and item CRUD through Nag and Mnemo -- Task recurrence (basic) - -### Device management - -- Provisioning and policy keys (`Provision`, `Settings`) -- Remote wipe status tracking -- Per-device logging (`perdevice` log type in Horde config) -- Device block/allow hooks (Horde `hooks.php`) - -### State and performance - -- SQL (default) or MongoDB state backends -- Sync key / modseq change tracking -- `Ping` long-poll with configurable heartbeat bounds -- `SyncCache` for in-request collection state -- WBXML protocol logging at configurable verbosity - -## EAS 16.0 — what changed - -Microsoft reworked several areas in 16.0. The following are implemented in this -codebase: - -| Area | Behaviour | -|------|-----------| -| **Calendar instances** | Exceptions are first-class sync items, not only embedded in the series master | -| **ClientUid** | Persisted on events and exported on sync | -| **Location** | `AirSyncBase:Location` instead of plain string for 16.0+ | -| **Drafts** | Content changes reported as `CHANGE_TYPE_DRAFT`; `send=true` sends via SMTP and removes draft | -| **Find** | Mailbox/GAL search; KQL parser maps common Outlook restrictions to IMAP search | -| **SmartForward/Reply** | `Forwardee` list support | -| **Appointment validation** | Forbidden inbound fields stripped per MS-ASCAL spec | - -Horde driver details (initial calendar UID list omits bound exceptions at 16.0+, -`calendar_import()` unified return shape) live in `horde/core` and `horde/kronolith`. - -## EAS 16.1 — what changed - -EAS 16.1 is a small delta on top of 16.0. The following are implemented in this -codebase: - -| Area | Behaviour | -|------|-----------| -| **Propose new time** | `MeetingResponse` accepts `ProposedStartTime` / `ProposedEndTime`; outbound RFC5546 `METHOD=COUNTER`; inbound storage and sync of attendee proposals | -| **DisallowNewTimeProposal** | Exported on calendar appointments (≥14.0) from iCal `DISALLOW-COUNTER`; inbound proposals ignored when set | -| **Account-only remote wipe** | `Provision:AccountOnlyRemoteWipe` status flow; admin and user prefs UI (devices must negotiate ≥16.1) | - -Horde driver, Kronolith, iTip, and IMP details live in `horde/core`, `horde/kronolith`, -`horde/itip`, and `horde/imp`. - -## Using ActiveSync in a Horde deployment - -### 1. Enable and configure - -In Horde administration → ActiveSync (or `var/config/horde/conf.php`): - -```php -$conf['activesync']['enabled'] = true; -$conf['activesync']['version'] = '16.1'; // global protocol ceiling (see above) -$conf['activesync']['storage'] = 'Sql'; // or 'Nosql' (Mongo) -$conf['activesync']['emailsync'] = true; -$conf['activesync']['auth']['type'] = 'basic'; -// Optional: per-device version policy instead of per-user permissions -// $conf['activesync']['version_mode'] = 'device'; -``` - -Per-user version limits are **not** preference keys — configure them under -Permissions → ActiveSync → *Maximum ActiveSync protocol version*. See -[Protocol version configuration](#protocol-version-configuration-horde). - -Also configure IMAP/SMTP host hints for Autodiscover, logging path/level, and -Ping heartbeat bounds. Full option descriptions are in -`vendor/horde/horde/config/conf.xml` under the `activesync` tab. - -History (`$conf['history']['enabled']`) must be enabled — ActiveSync relies on -it for change timestamps. - -### 2. Web server URL - -Clients expect `/Microsoft-Server-ActiveSync`. Rewrite that path to Horde's RPC -endpoint, for example: - -``` -/Microsoft-Server-ActiveSync → /horde/rpc.php -``` - -The RPC layer selects the ActiveSync backend when `server=ActiveSync` is passed -(Apache/nginx configs usually add this; see -[Horde ActiveSync wiki](http://wiki.horde.org/ActiveSync)). - -Autodiscover is served from the same endpoint when the request URI contains -`autodiscover/autodiscover`. - -### 3. Client setup - -Point the device at your mail domain. With Autodiscover enabled -(`autodiscovery` in config), iOS and Outlook discover the ActiveSync URL -automatically. Otherwise configure the ActiveSync server URL manually. - -Authentication is HTTP Basic against Horde by default (`auth.type = basic`). - -### 4. Per-user access - -Users need the **ActiveSync** permission in Horde. They can manage enrolled -devices under Personal Preferences → ActiveSync (device list and wipe — not -protocol version). - -Administrators assign the maximum EAS version per user or group via -**Permissions**, and optionally per device via `hooks.php` when -`version_mode = device`. -## For integrators — custom backends - -To use this library outside Horde (or with a minimal test stack): - -1. Subclass `Horde_ActiveSync_Driver_Base` and implement the abstract methods for - each collection class you support. -2. Provide a `Horde_ActiveSync_State_*` implementation (or use SQL/Mongo drivers - from this package). -3. Instantiate the server: - -```php -$server = new Horde_ActiveSync( - $driver, - new Horde_ActiveSync_Wbxml_Decoder(fopen('php://input', 'r')), - new Horde_ActiveSync_Wbxml_Encoder(fopen('php://output', 'w+')), - $state, - $httpRequest -); -$server->setSupportedVersion(Horde_ActiveSync::VERSION_SIXTEEN); -$server->setLogger($logger); -// Optional: implement versionCallback() on your driver for dynamic ceilings -$server->handleRequest($cmd, $device); -``` - -`Horde_ActiveSync_Driver_Mock` plus `Horde_ActiveSync_Driver_MockConnector` in -this package provide a reference stack for unit and integration tests. - -Import/export of calendar data is normally done by converting between -`Horde_ActiveSync_Message_Appointment` and your domain model (in Horde, -`Kronolith_Event`). - -## Development and tests - -**Requirements:** PHP `^7.4 || ^8`, plus Horde packages listed in `composer.json`. -Suggested packages for a full stack: `horde/imap_client`, `horde/db`, `horde/mail`. - -Run unit tests from the package directory: - -```bash -cd vendor/horde/activesync -php ../../../vendor/bin/phpunit --bootstrap ../../../vendor/autoload.php -``` - -Calendar EAS 16.0 import/export tests live in `horde/kronolith`: - -```bash -php vendor/bin/phpunit \ - --bootstrap vendor/horde/kronolith/test/bootstrap.php \ - vendor/horde/kronolith/test/Kronolith/Unit/EventActiveSyncTest.php -``` - -WBXML fixtures and protocol-level tests are under `test/unit/` and -`test/integration/`. - -Enable protocol logging (`logging.level` / per-device log files) when debugging -client issues — the logger records command names, collection IDs, and decoded -metadata without dumping full message bodies at low levels. +### Protocol versions + +All EAS versions from 2.5 to 16.1 are implemented and production-supported. +Clients negotiate down to the server's advertised ceiling; the ceiling is +configurable globally, per user, and per device. Details, the version +negotiation mechanics, and a per-version feature delta are in +[`doc/protocol-versions.md`](doc/protocol-versions.md). + +| Version | Notes | +|---------|-------| +| 2.5 | Baseline; reduced command set (no `Settings`, `ItemOperations`, `Find`) | +| 12.0 / 12.1 | `AirSyncBase` bodies, provisioning 2, empty/hanging Sync, SyncCache | +| 14.0 / 14.1 | Conversations, reply/forward state, rights management, body parts | +| 16.0 | Calendar instance model, drafts sync, `Find`, `AirSyncBase:Location` | +| 16.1 | Meeting time proposals, account-only remote wipe | + +### Feature highlights + +- **Mail:** folder & message sync, flags, drafts (incl. EAS 16 draft + editing/sending), send/reply/forward, attachments, meeting requests, + mailbox `Find` with KQL parsing, GAL search +- **Calendar:** full recurrence + exceptions, attendees, meeting responses, + EAS 16 instance model, meeting time proposals (16.1) +- **Contacts / Tasks / Notes:** full CRUD sync incl. task recurrence +- **Device management:** provisioning & policies, remote wipe (full and + account-only), per-device protocol ceilings, per-device logging +- **State:** SQL (default) or MongoDB backends, sync-key/modseq change + tracking, `Ping` long-poll, `SyncCache` +- **Delivery:** optional streamed `Sync` responses (chunked HTTP) with + deferred up-sync import and WBXML keep-alives, so clients with ~30 s read + timeouts survive large batches — see + [`doc/sync-streaming.md`](doc/sync-streaming.md) ## Package layout @@ -416,11 +95,12 @@ lib/Horde/ActiveSync/ Message/ Item type WBXML mappings State/ Sync state persistence (SQL, Mongo) Wbxml/ Encoder, decoder, code pages + Imap/ IMAP-to-EAS message building Find/ EAS 16.0 Find command helpers Driver/ Base, Mock backends migration/ SQL schema for state tables +doc/ Documentation (see index above) test/unit/ PHPUnit tests -doc/Horde/ActiveSync/todo.md Open work and Horde 6 refactor notes ``` ## License diff --git a/doc/architecture.md b/doc/architecture.md new file mode 100644 index 00000000..e1d34c78 --- /dev/null +++ b/doc/architecture.md @@ -0,0 +1,200 @@ +# Architecture (ActiveSync developers) + +How the library works internally. Read this before changing protocol logic. +Companion documents: [`protocol-versions.md`](protocol-versions.md) for +per-version behaviour, [`sync-streaming.md`](sync-streaming.md) for the +streamed `Sync` delivery design, and [`todo.md`](todo.md) for the Horde 6 +refactor roadmap. + +## Component map + +| Component | Classes | Role | +|-----------|---------|------| +| Server front door | `Horde_ActiveSync` | Auth, version negotiation, device handling, provisioning enforcement, request dispatch; owns protocol constants | +| Request handlers | `Horde_ActiveSync_Request_*` | One class per EAS command (`Sync`, `FolderSync`, `Ping`, `ItemOperations`, …), all extending `Request_Base` | +| WBXML layer | `Horde_ActiveSync_Wbxml_Encoder` / `_Decoder` | Token-level WBXML I/O with the EAS code pages (`Wbxml.php`); protocol logging | +| Message model | `Horde_ActiveSync_Message_*` | Typed, version-aware property maps for every item type (mail, appointment, contact, task, note, attachments, …) | +| Collection state | `Horde_ActiveSync_Collections`, `Horde_ActiveSync_SyncCache`, `Horde_ActiveSync_Folder_*` | Per-request collection management and the persisted cross-request collection cache | +| Sync state | `Horde_ActiveSync_State_Base` (`_Sql`, `_Mongo`) | Devices, policy keys, sync keys, change maps, garbage collection | +| Backend API | `Horde_ActiveSync_Driver_Base` (+ `Driver_Mock`) | Abstract data access; all reads/writes to the actual store go through it | +| Import/export | `Horde_ActiveSync_Connector_Importer`, `Connector_Exporter_Sync` | Apply client changes to the backend; stream server changes to the encoder | +| Mail building | `Horde_ActiveSync_Imap_Adapter`, `Imap_EasMessageBuilder*`, `Imap_MessageBodyData`, `Imap_Strategy_*` | Turn IMAP messages into EAS `Message_Mail` objects; change detection strategies (modseq, plain, initial) | +| Device & policy | `Horde_ActiveSync_Device`, `Horde_ActiveSync_Policies` | Per-device metadata (type, policy key, wipe status, version); policy WBXML/XML generation | +| Search | `Horde_ActiveSync_Search_*`, `Horde_ActiveSync_Find_*` | `Search` and EAS 16 `Find` parameter/result objects, KQL parser | + +## Request lifecycle + +Every request enters through `Horde_ActiveSync::handleRequest($cmd, $devId)`: + +1. **Logger setup** from GET parameters (per-device log files). +2. **`versionCallback()`** — if the driver implements it, it may adjust the + advertised version ceiling for this request (per-user/per-device policy). +3. **Autodiscover** short-circuits here (it authenticates on its own). +4. **Authentication** via `Horde_ActiveSync_Credentials` and the driver. On + failure the EAS headers are still sent (clients probe with bad + credentials) and `Horde_Exception_AuthenticationFailure` is thrown. +5. **Command normalisation** — `FolderDelete`/`FolderUpdate` are handled by + `Request_FolderCreate`; the deprecated EAS 1.x collection commands map to + `Request_LegacyCollection`. +6. **Device handling** (`_handleDevice()`) — load or create the + `Horde_ActiveSync_Device`, check block/allow status, remote wipe state. +7. **Provisioning support** is announced from `driver->getProvisioning()`. +8. **WBXML header** is read from the request body; multipart support + (`MS-ASAcceptMultiPart`) is detected for `ItemOperations`. +9. **Response headers** (`MS-Server-ActiveSync`, content type) are sent + *before* dispatch, because some handlers (`GetAttachment`) start streaming + output immediately. +10. **Dispatch** to `Horde_ActiveSync_Request_::handle()`, which owns + request parsing (decoder), business logic (driver), and response + encoding (encoder). + +`OPTIONS` requests bypass dispatch and only emit the version/commands +headers. + +### Policy enforcement + +`Request_Base::checkPolicyKey()` compares the client's `X-MS-PolicyKey` +against the device's stored key; commands that require provisioning respond +with the appropriate provisioning status when the key is stale, which makes +the client run a `Provision` cycle and retry. + +## Anatomy of a Sync request + +`Request_Sync` (with `Request_SyncBase` and `Horde_ActiveSync_Collections`) +is the heart of the library. A `Sync` request runs in two phases: + +### Phase 1 — parse (decoder) + +- Read the collection list: sync keys, options (`FilterType`, body + preferences, conflict handling, `WindowSize`, EAS 16 `ClientUid` …). +- Read client-sent commands (`Add`/`Change`/`Delete`/`Fetch`) per + collection. In buffered mode they are imported inline through + `Connector_Importer` while parsing; in streaming mode they are *queued* + and imported later during output (see + [`sync-streaming.md`](sync-streaming.md)). +- For EAS ≥ 12.1 an **empty request body** is legal: the collection set is + reconstructed from the persisted `SyncCache` (`Collections` + + `SyncCache`), enabling cheap looping sync. +- `HeartbeatInterval` / `Wait` turn the request into a hanging sync: the + handler polls for changes (`Collections::pollForChanges()`) until the + heartbeat expires or changes arrive. + +### Phase 2 — respond (encoder) + +For each collection: + +1. `initCollectionState()` loads the state for the client's sync key + (throwing `StateGone` → sync-key error status when the key is unknown, + which forces the client to resync). +2. Deferred client commands are imported here when streaming. +3. `getCollectionChangeCount()` asks the backend for pending server→client + changes; the effective window (client `WindowSize`, global window, and + the streaming count cap) decides truncation, announced via + `MoreAvailable` *before* the `Commands` block (MS-ASCMD ordering rule). +4. `Connector_Exporter_Sync` streams each change: fetches the item from the + driver, encodes it (`Message_*::encodeStream()`), and records it in the + change map so the client's echo is not mirrored back. +5. Replies for client commands (`SyncReplies`: server ids for adds, status + for failures, EAS 16 `modifiedids`) are encoded. +6. A **new sync key** is written with the resulting state. The client + confirms it by using it in the next request; until then the previous + state remains recoverable (sync keys are the transaction mechanism). + +`Ping` is the lightweight sibling: no item transfer, only "did anything +change in these collections", with heartbeat bounds from configuration. + +## Sync state and the SyncCache + +State backends (`State_Sql`, `State_Mongo`) persist, per device + user: + +- **Device records** (`horde_activesync_device`, `_device_users`): device + info, policy key, wipe status, per-device version. +- **Sync state** (`horde_activesync_state`): serialized folder state per + collection and sync key — for mail, the seen UID set and modseq + (`Folder_Imap`); for other classes, timestamp state (`Folder_Collection`). +- **Change map** (`horde_activesync_map`, `_mailmap`): which changes were + sent by / imported from the client, keyed by sync key, so client-origin + changes are filtered out of the next export (loop avoidance). +- **SyncCache** (`horde_activesync_cache`): cross-request collection + metadata (folder list, per-collection options, hierarchy sync key, + pingable flags) enabling EAS ≥ 12.1 empty and looping Sync requests. + +Two sync keys per collection are kept alive so a lost response can be +replayed; garbage collection trims older generations. When state and client +diverge irrecoverably, the handler answers with a sync-key error status and +the client restarts from sync key 0 (full resync of that collection). + +## WBXML layer + +`Wbxml_Decoder` and `Wbxml_Encoder` implement EAS WBXML directly (no +external XML lib): multi-byte integers, string tables, opaque data, and the +EAS code pages defined in `Wbxml.php`. Points worth knowing: + +- The **encoder buffers start tags** and only writes them when actual + content follows (`startTag(..., $output = false)` semantics), so empty + containers never reach the wire. +- `Encoder::flushOutput()` pushes encoder bytes to the PHP output stream — + the streaming hook. `Encoder::keepAlive()` emits a redundant + `SWITCH_PAGE` to the already-active code page: a semantic no-op for any + conforming parser, used as a keep-alive byte source during long + server-side work (see [`sync-streaming.md`](sync-streaming.md)). +- The WBXML header is emitted exactly once (`outputWbxmlHeader()` is + idempotent). +- Both classes log a pretty-printed protocol trace (`I:` / `O:` lines in the + per-device logs) — the primary debugging tool for client issues. +- Large values (attachment data, bodies) are streamed through + `Horde_Stream` objects, never concatenated into strings. + +## Import and export connectors + +- `Connector_Importer` applies client changes to the backend: conflict + detection against the change map, `changeMessage()` / `deleteMessage()` / + `moveMessage()` calls, folder operations, and recording of client-origin + changes so they are not exported back. +- `Connector_Exporter_Sync` walks the change list produced by + `getServerChanges()` and encodes each change into the response, updating + state as it goes. + +## The IMAP subsystem + +Mail is the only class where the library itself contains substantial data +logic (other classes convert in the backend): + +- `Imap_Adapter` wraps a `Horde_Imap_Client` factory and implements folder + stat/changes/fetch for the driver. +- Change detection strategies in `Imap_Strategy_*`: `Modseq` + (CONDSTORE/QRESYNC servers), `Plain` (fallback polling), `Initial` (first + sync). +- `Imap_EasMessageBuilder` + `Imap_MessageBodyData` assemble + `Message_Mail` objects honoring body preferences, truncation, MIME + support, and protocol version differences. + +## Tests + +``` +test/unit/ PHPUnit unit tests (WBXML, messages, requests, state); + WBXML and MIME fixtures under …/ActiveSync/fixtures/ +test/integration/ End-to-end request tests against the mock driver +``` + +Run from the package directory: + +```bash +cd vendor/horde/activesync +php ../../../vendor/bin/phpunit --bootstrap ../../../vendor/autoload.php +``` + +`Horde_ActiveSync_Driver_Mock` + `Driver_MockConnector` provide a full fake +backend, so request handlers can be exercised end-to-end (decoder in, +encoder out) without IMAP or a database. Streaming-specific tests live in +`test/unit/Horde/ActiveSync/Request/SyncStreamingTest.php`. + +## Documentation conventions + +Cross-cutting designs (anything spanning several classes or packages) are +documented here in `doc/`, not in docblocks — e.g. +[`sync-streaming.md`](sync-streaming.md). Code comments are reserved for +*local* invariants that must move with the code (ordering constraints, +protocol quirks at a specific call site) and may point to the relevant doc +with `@see doc/.md`. Do not duplicate design prose into docblocks; it +drifts. diff --git a/doc/configuration.md b/doc/configuration.md new file mode 100644 index 00000000..fa294baf --- /dev/null +++ b/doc/configuration.md @@ -0,0 +1,219 @@ +# Configuring ActiveSync (administrators) + +How to enable and operate ActiveSync in a Horde deployment. For embedding the +library in a non-Horde product see [`integration.md`](integration.md); for +internals see [`architecture.md`](architecture.md). + +## 1. Enable and configure + +In Horde administration → ActiveSync (or `var/config/horde/conf.php`): + +```php +$conf['activesync']['enabled'] = true; +$conf['activesync']['version'] = '16.1'; // global protocol ceiling (see below) +$conf['activesync']['storage'] = 'Sql'; // or 'Nosql' (Mongo) +$conf['activesync']['emailsync'] = true; +$conf['activesync']['auth']['type'] = 'basic'; +// Optional: per-device version policy instead of per-user permissions +// $conf['activesync']['version_mode'] = 'device'; +``` + +Also configure IMAP/SMTP host hints for Autodiscover, logging path/level, and +Ping heartbeat bounds. Full option descriptions are in +`vendor/horde/horde/config/conf.xml` under the `activesync` tab. + +History (`$conf['history']['enabled']`) must be enabled — ActiveSync relies on +it for change timestamps. + +## 2. Web server URL + +Clients expect `/Microsoft-Server-ActiveSync`. Rewrite that path to Horde's RPC +endpoint, for example: + +``` +/Microsoft-Server-ActiveSync → /horde/rpc.php +``` + +The RPC layer selects the ActiveSync backend when `server=ActiveSync` is passed +(Apache/nginx configs usually add this; see the +[Horde ActiveSync wiki](http://wiki.horde.org/ActiveSync)). + +Autodiscover is served from the same endpoint when the request URI contains +`autodiscover/autodiscover`. + +**When Sync response streaming is enabled** (see below), the web server must +not buffer or compress responses on this path — lighttpd `mod_deflate`, nginx +`gzip` / `proxy_buffering`, and similar filters would hold the streamed bytes +back and re-introduce the client timeouts streaming is meant to prevent. + +## 3. Client setup + +Point the device at your mail domain. With Autodiscover enabled +(`autodiscovery` in config), iOS and Outlook discover the ActiveSync URL +automatically. Otherwise configure the ActiveSync server URL manually. + +Authentication is HTTP Basic against Horde by default (`auth.type = basic`). + +## 4. Per-user access + +Users need the **ActiveSync** permission in Horde. They can manage enrolled +devices under Personal Preferences → ActiveSync (device list and wipe — not +protocol version). + +Administrators assign the maximum EAS version per user or group via +**Permissions**, and optionally per device via `hooks.php` when +`version_mode = device` (both described next). + +## Protocol version policy + +The library supports EAS 2.5 – 16.1 (see +[`protocol-versions.md`](protocol-versions.md) for what each version adds). +Which version a device actually syncs at is bounded by a **ceiling** that +stacks in three layers. None of these are personal *preferences* — users +cannot change their own EAS version; per-user limits are **administrator +permissions**. + +### 1. Global ceiling (all users, default) + +Set in Horde administration → ActiveSync → *What is the highest version of EAS +that Horde should support?*, or in `conf.php`: + +```php +$conf['activesync']['version'] = '16.1'; +``` + +`Horde_Core_Factory_ActiveSyncServer` calls `setSupportedVersion()` with this +value when the server object is created. This is the baseline for every +request. + +### 2. Per-user ceiling (permissions) + +Default mode: `version_mode` is **`user`** when unset. + +Administrators can assign **Maximum ActiveSync protocol version** +(`horde:activesync:version`) per user or group under Horde administration → +Permissions → ActiveSync. Allowed values: `2.5`, `12.0`, `12.1`, `14.0`, +`14.1`, `16.0`, and `16.1`. + +On each request, the driver's `versionCallback()` resolves the authenticated +Horde username (from HTTP Basic credentials, the `User` GET parameter, or the +registry) and reads that permission. If set, it calls `setSupportedVersion()` +again for this request only. + +| Situation | Effective ceiling for this request | +|-----------|-------------------------------------| +| Permission empty / permission tree not defined | Global `conf['activesync']['version']` only | +| User permission **lower** than global (e.g. user `14.1`, global `16.0`) | User value — caps that user below the site default | +| User permission **higher** than global (e.g. user `16.0`, global `14.1`) | User value — can raise the advertised ceiling above the admin default for that user | +| User in multiple groups with different values | **Lowest** (most restrictive) allowed version | + +The last row matters for group-based permissions: if one group allows `16.0` +and another `14.1`, the user syncs at `14.1`. + +### 3. Per-device ceiling (hook) + +For device-specific policy (pilot devices, problematic clients, lab handsets), +set in `conf.php`: + +```php +$conf['activesync']['version_mode'] = 'device'; +``` + +Then implement `activesync_device_version()` in `config/hooks.php` (see +`vendor/horde/horde/config/hooks.php.dist`): + +```php +public function activesync_device_version($deviceId, $user) +{ + // $deviceId is normalised to uppercase. + $map = [ + 'OLD-OUTLOOK-DEVICE-ID' => '14.1', + 'TEST-IPHONE-ID' => '16.0', + ]; + + return $map[$deviceId] ?? null; +} +``` + +Hook return values: + +| Return | Meaning | +|--------|---------| +| String, e.g. `'16.0'` | Use this ceiling for the device | +| Array of version strings | **Lowest** (most restrictive) entry is used | +| `null`, `false`, `''`, or `-1` | No override; fall back to global / user permission behaviour | + +`DeviceId` must be present in the request (standard on all sync commands). If +the hook is not defined or returns no override, behaviour depends on +`version_mode`: in **`device`** mode with no hook result, no permission +override is applied; switch back to **`user`** mode to use group permissions +as the primary per-principal control. + +### Choosing `user` vs `device` mode + +| `version_mode` | Source of per-request override | +|----------------|-------------------------------| +| `user` (default) | `horde:activesync:version` permission | +| `device` | `activesync_device_version` hook | + +Only one mode is active per installation. Use **permissions** for +account/class-of-user policy; use the **hook** when the device ID is the right +key (e.g. force an old Outlook build to `14.1` while everyone else stays on +`16.0`). + +## Sync response delivery (streaming) + +Some clients — notably Gmail on Android — abort a `Sync` connection after +~30 seconds without response body bytes, and can end up in a broken sync +state. Streaming delivery sends WBXML incrementally so bytes keep flowing +while the server works, in both directions (message export *and* import of +client-sent changes). The complete design, error model, and rationale are in +[`sync-streaming.md`](sync-streaming.md); this section covers the operator +view. + +All keys live under `$conf['activesync']['sync']` (Horde administration → +ActiveSync → *Sync Response Delivery*): + +| Key | Default | Meaning | +|-----|---------|---------| +| `streaming` | `false` | Master switch for streaming Sync delivery | +| `maxmessagesperresponse` | `10` | Count cap per response when streaming; more changes are announced via `MoreAvailable`. `0` = window size only | +| `maxmessagetime` | `0` | Soft cap (seconds) for assembling a single message; stops the batch after a slow message. Streaming only. `0` = off | +| `maxrequestduration` | `0` | Whole-request wall clock cap (seconds), measured from request start (includes import of client changes). Streaming only. `0` = off | +| `keepaliveinterval` | `15` | Minimum seconds between WBXML keep-alive tokens during import of client changes. `0` = one token per imported command | +| `maxresponsetime` | `25` | **Legacy** export-phase time budget; only honored when `streaming` is `false` | + +Rollback: set `streaming = false` to restore the buffered `Content-Length` +behaviour (including the `maxresponsetime` budget) with no other changes. + +### Operator notes + +- The Sync handler logs + `SYNC: starting response output N.Ns after request start (streaming on|off)` + at INFO level — use it to verify streaming is active and to measure time + to first byte. Up-sync batches additionally log + `Queued N incoming changes for deferred import (streaming).` and + `SYNC: imported N deferred incoming change(s) for collection F… in N.Ns, + N keep-alive(s) emitted`. +- Web-server-level buffering or compression on + `/Microsoft-Server-ActiveSync` can re-introduce the timeout even with + streaming enabled — PHP cannot disable it from inside the request. Exclude + the ActiveSync path from response buffering/compression (see section 2). +- Streaming trades slightly more HTTP round-trips (smaller batches with + `MoreAvailable`) for reliability and lower peak memory. +- A device already stuck from earlier timeouts may still need one account + re-add (or server-side device state removal) — streaming prevents the + breakage, it does not repair broken client state. + +## Logging and troubleshooting + +- Enable protocol logging (`logging.level`, and the `perdevice` log type for + one file per device) when debugging client issues. The logger records + command names, collection IDs, decoded WBXML structure, and metadata + without dumping full message bodies at low levels. +- Per-device logs are the primary debugging tool: they contain the decoded + incoming request (`I:` lines), the outgoing response (`O:` lines), and + state/collection diagnostics in between. +- Device block/allow decisions can be made via Horde `hooks.php`. +- Remote wipe (full, and account-only for EAS 16.1 devices) is managed from + the admin and user device lists. diff --git a/doc/integration.md b/doc/integration.md new file mode 100644 index 00000000..7aa1763e --- /dev/null +++ b/doc/integration.md @@ -0,0 +1,147 @@ +# Using the library (integrators) + +How to embed `horde/activesync` in your own product — outside Horde, or with +a minimal test stack. For Horde deployment configuration see +[`configuration.md`](configuration.md); for internals see +[`architecture.md`](architecture.md). + +## The three things you provide + +The library handles the protocol (WBXML, command dispatch, sync state +machine, device management). You provide: + +1. **A backend driver** — subclass of `Horde_ActiveSync_Driver_Base` that + maps EAS operations onto your data store (folders, messages, search, + send-mail, policies). +2. **A state backend** — `Horde_ActiveSync_State_Sql` (any + `Horde_Db_Adapter`) or `Horde_ActiveSync_State_Mongo`, or your own + subclass of `Horde_ActiveSync_State_Base`. Persists device records, sync + keys, and change maps. SQL schema ships in `migration/`. +3. **The HTTP plumbing** — routing `/Microsoft-Server-ActiveSync` to a + script that instantiates the server and hands it the request. In Horde + this is `horde/rpc`'s `Horde_Rpc_ActiveSync`; any framework will do. + +## Minimal server setup + +```php +$state = new Horde_ActiveSync_State_Sql(['db' => $hordeDbAdapter]); + +$driver = new My_ActiveSync_Driver([ + 'state' => $state, // required + 'logger' => $logger, // optional Horde_Log_Logger + 'ping' => [ // Ping/heartbeat bounds + 'heartbeatmin' => 60, + 'heartbeatmax' => 2700, + 'heartbeatdefault' => 480, + 'deviceping' => true, + 'waitinterval' => 15, + ], + 'sync' => [ // Sync response delivery, all optional + 'streaming' => true, // see doc/sync-streaming.md + 'maxmessagesperresponse' => 10, + ], +]); + +$server = new Horde_ActiveSync( + $driver, + new Horde_ActiveSync_Wbxml_Decoder(fopen('php://input', 'r')), + new Horde_ActiveSync_Wbxml_Encoder(fopen('php://output', 'w+')), + $state, + $httpRequest // Horde_Controller_Request_Http +); +$server->setSupportedVersion(Horde_ActiveSync::VERSION_SIXTEENONE); +$server->setLogger($logger); +$server->handleRequest($cmd, $deviceId); // from GET: Cmd, DeviceId +``` + +`handleRequest()` performs authentication (via your driver), version +negotiation, device handling, provisioning enforcement, and dispatches to the +matching `Horde_ActiveSync_Request_*` handler, which reads the request body +from the decoder and writes the response through the encoder. + +**Response delivery** is your caller's responsibility: either buffer the +encoder output and send it with `Content-Length` (classic), or — for the +`Sync` command — let the handler stream it and send no `Content-Length` +(chunked). `Horde_Rpc_ActiveSync` in `horde/rpc` is the reference +implementation of both paths, including the streaming rules (no output +buffering, no zlib compression, `headers_sent()`-guarded error headers). If +you enable `sync => streaming`, replicate that behaviour; details in +[`sync-streaming.md`](sync-streaming.md). + +## Implementing a driver + +Subclass `Horde_ActiveSync_Driver_Base` and implement its abstract methods. +They group into: + +| Group | Methods (selection) | Purpose | +|-------|--------------------|---------| +| Authentication | `authenticate()`, `setup()`, `clearAuthentication()` | Map HTTP credentials to your user model | +| Folder hierarchy | `getFolders()`, `getFolderList()`, `getFolder()`, `statFolder()`, `changeFolder()`, `deleteFolder()` | Expose your folder tree as EAS collections | +| Item sync | `getServerChanges()`, `getMessage()`, `statMessage()`, `changeMessage()`, `deleteMessage()`, `moveMessage()` | Change detection and item CRUD | +| Mail specifics | `sendMail()`, `getAttachment()`, `getWasteBasket()`, `setReadFlag()`, `statMailMessage()` | SMTP send, attachments, trash semantics | +| Search | `getSearchResults()`, `getFindResults()`, `resolveRecipient()` | Mailbox/GAL search (`Search`, `Find`, `ResolveRecipients`) | +| ItemOperations | `itemOperationsGetAttachmentData()`, `itemOperationsFetchMailbox()`, `itemOperationsGetDocumentLibraryLink()` | Fetch operations | +| Device / policy | `getCurrentPolicy()`, `getProvisioning()`, `getSettings()`, `setSettings()`, `autoDiscover()`, `getUsernameFromEmail()` | Provisioning, OOF, autodiscover | +| Meetings | `meetingResponse()`, `getFreebusy()` | Meeting accept/decline/counter | + +The authoritative list with full signatures and docblocks is +`lib/Horde/ActiveSync/Driver/Base.php`. `Horde_Core_ActiveSync_Driver` in +`horde/core` is the production reference implementation; +`Horde_ActiveSync_Driver_Mock` (+ `MockConnector`) in this package is a +minimal reference stack used by the unit and integration tests. + +### Item conversion + +Sync items travel as `Horde_ActiveSync_Message_*` objects (typed, +version-aware WBXML property maps). Your driver converts between them and +your domain model: + +- `getMessage()` returns a populated message object + (e.g. `Horde_ActiveSync_Message_Appointment`) for export to the client. +- `changeMessage()` receives a message object decoded from the client and + applies it to your store. + +Message constructors accept `protocolversion` and adjust their property maps +to the negotiated EAS level, so the same driver code serves all protocol +versions. In Horde, calendar conversion lives in +`Kronolith_Event::fromASAppointment()` / `toASAppointment()`. + +### Dynamic protocol ceilings + +Optionally implement `versionCallback(Horde_ActiveSync $server)` on your +driver. The library checks `is_callable([$driver, 'versionCallback'])` at the +start of every request, before authentication completes — call +`$server->setSupportedVersion()` there to enforce per-user or per-device +ceilings (this is how Horde's permission- and hook-based version policy is +implemented). See [`protocol-versions.md`](protocol-versions.md) for the +negotiation mechanics. + +## State backends + +| Class | Storage | Notes | +|-------|---------|-------| +| `Horde_ActiveSync_State_Sql` | Any `Horde_Db_Adapter` | Default; schema in `migration/` (`horde_activesync_*` tables) | +| `Horde_ActiveSync_State_Mongo` | MongoDB | Same responsibilities, document storage | + +The state backend persists: device records and per-user device pairings, +policy keys, sync state per collection + sync key, incoming-change maps (to +avoid mirroring client changes back), and the `SyncCache`. If you implement +your own, subclass `Horde_ActiveSync_State_Base` and keep its locking and +garbage-collection semantics — see [`architecture.md`](architecture.md). + +## Requirements and tests + +PHP `^7.4 || ^8`, plus the Horde packages listed in `composer.json`. +Suggested for a full stack: `horde/imap_client`, `horde/db`, `horde/mail`. + +Run the package tests from the package directory: + +```bash +cd vendor/horde/activesync +php ../../../vendor/bin/phpunit --bootstrap ../../../vendor/autoload.php +``` + +WBXML fixtures and protocol-level tests are under `test/unit/` and +`test/integration/`; the mock driver stack lets you test request handling +end-to-end without a real backend. See [`architecture.md`](architecture.md) +for the test layout. diff --git a/doc/protocol-versions.md b/doc/protocol-versions.md new file mode 100644 index 00000000..b1d4906a --- /dev/null +++ b/doc/protocol-versions.md @@ -0,0 +1,184 @@ +# EAS protocol versions + +The library implements Exchange ActiveSync versions **2.5, 12.0, 12.1, 14.0, +14.1, 16.0 and 16.1** (constants `Horde_ActiveSync::VERSION_*`). All of them +are supported for production use; message classes, request handlers, and +policy encoding adjust automatically to the version negotiated per request. + +This document covers how negotiation works and what each version adds *as +implemented in this library*. How to configure version ceilings in a Horde +deployment is in [`configuration.md`](configuration.md); the library API for +custom drivers is in [`integration.md`](integration.md). + +## How version negotiation works + +Two related values matter on each request: + +1. **Server ceiling** (`Horde_ActiveSync::$_maxVersion`, set via + `setSupportedVersion()`). Controls what the server **advertises** in + `OPTIONS` / `MS-ASProtocolVersions` and `MS-Server-ActiveSync`, and which + command sets are available. +2. **Session protocol version** (client header `MS-ASProtocolVersion`, or + `ProtVer` in GET for very old clients). The level actually used for WBXML + encoding/decoding on that request. The client must not exceed what it + offered and what the server supports. + +Clients normally negotiate down: a device that sends +`MS-ASProtocolVersion: 16.0` against a server ceiling of `14.1` will sync at +14.1. + +The advertised version list is every supported version up to and including +the ceiling (`getSupportedVersions()`), so a ceiling of `14.1` advertises +`2.5,12.0,12.1,14.0,14.1`. + +The library itself only exposes `setSupportedVersion()`. Per-user and +per-device policy is a driver concern: if the driver implements +`versionCallback(Horde_ActiveSync $server)`, it is invoked at the start of +every `handleRequest()` call, before authentication completes, and may lower +or raise the ceiling for that request. `Horde_Core_ActiveSync_Driver` uses +this to apply Horde permissions and the `activesync_device_version` hook +(see [`configuration.md`](configuration.md)). + +## Commands per version + +Advertised in `MS-ASProtocolCommands` (`getSupportedCommands()`): + +- **All versions:** `Sync`, `SendMail`, `SmartForward`, `SmartReply`, + `GetAttachment`, `GetHierarchy`, `CreateCollection`, `DeleteCollection`, + `MoveCollection`, `FolderSync`, `FolderCreate`, `FolderDelete`, + `FolderUpdate`, `MoveItems`, `GetItemEstimate`, `MeetingResponse`, + `Search`, `Ping`, `Provision`, `ResolveRecipients`, `ValidateCert` +- **≥ 12.0 additionally:** `Settings`, `ItemOperations`, `Find` + +`OPTIONS` and **Autodiscover** are handled outside the normal command loop +(in Horde, via `Horde_Rpc_ActiveSync`). + +## Per-version deltas (as implemented) + +### 2.5 — baseline + +The oldest supported dialect, kept for legacy devices. + +- Reduced command set (no `Settings`, `ItemOperations`, `Find`). +- Message bodies are sent through the legacy per-class body properties + (truncation via `MIME` options), not `AirSyncBase`. +- Provisioning uses the XML policy format (`MS-WAP-Provisioning-XML`). +- Folder hierarchy via `GetHierarchy`/`CreateCollection`-style commands. + +### 12.0 + +- **`AirSyncBase` namespace:** unified `Body`, `BodyPreferences`, attachment + metadata across all item classes (`Horde_ActiveSync_Message_AirSyncBase*`). +- **Provisioning 2:** WBXML policy format with the extended policy-setting + vocabulary (`Horde_ActiveSync_Policies`); XML policies are rejected for + ≥ 12.0 devices and vice versa. +- `Settings` (device information, OOF) and `ItemOperations` (fetch, + attachments) become available. +- Search across mailbox and GAL. + +### 12.1 + +- **Empty/short Sync requests:** a client may send a `Sync` with no body, or + omit per-collection options; the server completes the request from the + persisted **`SyncCache`**. This is the basis of efficient looping sync. +- **Hanging Sync:** `HeartbeatInterval` / `Wait` on the `Sync` command itself + (long-poll without `Ping`). +- Policy key handling and provisioning-status vocabulary extended + (`Horde_ActiveSync_Policies` emits additional 12.1 policy settings). +- The folder class (`FolderType`) is no longer echoed per collection in Sync + responses (clients track it from `FolderSync`). + +### 14.0 + +- **Conversations:** `ConversationMode` on Sync collections and conversation + ids on mail items. +- **Reply/forward state:** `LastVerbExecuted` / `LastVerbExecutionTime` + exported on mail flag changes. +- Free/busy data in `ResolveRecipients`. +- `MeetingResponse` and meeting-request handling reworked (native WBXML + encoding of meeting metadata). + +### 14.1 + +- **Rights management:** `RightsManagementSupport` negotiated per collection + during Sync parsing. +- **`BodyPartPreference` / `BodyPart`:** partial-body sync for + conversation-style clients. +- **GAL photos:** picture options and photo data in `ResolveRecipients` + responses (and later in `Find`). +- Extended device information in `Settings`. + +### 16.0 + +Microsoft reworked several areas in 16.0. Implemented here: + +| Area | Behaviour | +|------|-----------| +| **Calendar instances** | Exceptions are first-class sync items with top-level `InstanceId`, not only embedded in the series master; masters export only deleted-instance exceptions; bound exceptions visible in initial sync | +| **ClientUid** | Client-generated UID persisted on events and round-tripped on sync | +| **Location** | `AirSyncBase:Location` object (display name + coordinates) instead of a plain string | +| **Drafts** | Draft folder sync; content changes reported as `CHANGE_TYPE_DRAFT`; `POOMMAIL2:Send` sends via SMTP and removes the draft | +| **Find** | Mailbox/GAL search; KQL parser (`Horde_ActiveSync_Find_Kql`) maps boolean operators, property restrictions, dates, and sizes to backend (IMAP) search | +| **SmartForward/Reply** | `Forwardee` list support | +| **Appointment validation** | Forbidden inbound top-level fields (`uid`, `dtstamp`, `organizername`, `organizeremail`) are stripped per MS-ASCAL instead of rejecting the item | +| **All-day events** | Date-only handling without spurious timezone conversion | + +Horde driver details (initial calendar UID list omitting bound exceptions, +`calendar_import()` unified return shape) live in `horde/core` and +`horde/kronolith`. + +### 16.1 + +A small delta on top of 16.0. Implemented here: + +| Area | Behaviour | +|------|-----------| +| **Propose new time** | `MeetingResponse` accepts `ProposedStartTime` / `ProposedEndTime`; outbound RFC 5546 `METHOD=COUNTER`; inbound storage and sync of attendee proposals | +| **DisallowNewTimeProposal** | Exported on calendar appointments (≥ 14.0) from iCal `DISALLOW-COUNTER`; inbound proposals ignored when set | +| **Account-only remote wipe** | `Provision:AccountOnlyRemoteWipe` status flow; admin and user device UI in Horde (devices must negotiate ≥ 16.1) | + +Horde driver, Kronolith, iTip, and IMP details live in `horde/core`, +`horde/kronolith`, `horde/itip`, and `horde/imp`. + +## Implemented feature set by item class + +Independent of protocol version (availability of individual features follows +the deltas above): + +### Mail (EAS `Email` class) + +- Folder hierarchy sync, message sync, flags, categories +- Send, reply, forward (`SendMail`, `SmartReply`, `SmartForward`) +- Attachments (`GetAttachment`, `ItemOperations:Fetch`) +- Meeting requests embedded in mail +- Body preferences and truncation (`AirSyncBase:Body`, `BodyPart`) +- Draft folder sync and EAS 16 draft editing/sending +- GAL search (`Search`, `ResolveRecipients`), mailbox `Find` + +### Calendar (EAS `Calendar` class) + +Conversion logic is shared between this library's `Message/Appointment` / +`Message/Exception` classes and the calendar backend (in Horde: +`Kronolith_Event::fromASAppointment()` / `toASAppointment()`). + +- Create, update, delete; recurrence and exceptions +- Attendees, reminders, categories, sensitivity, busy status +- Meeting responses, EAS 16.1 time proposals +- EAS 16 instance model (see above) + +### Contacts (`Contacts`) + +- Personal address books and GAL +- Photo support via `ResolveRecipients` / `Find` picture options +- Standard vCard-style field mapping + +### Tasks (`Tasks`) and Notes (`Notes`) + +- Full folder sync and item CRUD (in Horde: through Nag and Mnemo) +- Task recurrence (basic) + +### Device management + +- Provisioning and policy keys (`Provision`, `Settings`) +- Remote wipe (full; account-only with 16.1) with status tracking +- Per-device logging and block/allow hooks (Horde `hooks.php`) diff --git a/doc/sync-streaming.md b/doc/sync-streaming.md new file mode 100644 index 00000000..8e48c366 --- /dev/null +++ b/doc/sync-streaming.md @@ -0,0 +1,198 @@ +# Sync response streaming + +Design document for streamed `Sync` response delivery. Operator-facing +configuration is summarized in [`configuration.md`](configuration.md); +internals context is in [`architecture.md`](architecture.md). + +Tracking issues: problem report +[horde/ActiveSync#77](https://github.com/horde/ActiveSync/issues/77), +implementation [horde/ActiveSync#83](https://github.com/horde/ActiveSync/issues/83). + +## The problem + +Historically the whole `Sync` WBXML response was buffered: +`Horde_Rpc_ActiveSync` wrapped the handler in a 1 MiB output buffer and sent +the result with a `Content-Length` header. A client therefore received **no +response body bytes** until the server had fetched and encoded the entire +batch. + +Some clients — notably Gmail on Android — abort the connection after ~30 +seconds without body bytes (`SocketTimeout`), retry with the old sync key, +and can end up in a broken state that only an account re-add or a +server-side state reset resolves. + +Two distinct server-side silent periods can exceed that budget: + +1. **Export phase:** fetching and encoding a large server→client batch + (big mailboxes, large `FilterType` windows, slow IMAP). +2. **Import phase:** applying a large client→server batch *while parsing + the request*, before a single response byte exists. Example: Gmail's + `FullDraftsUpSync` re-sends hundreds of drafts as `Change` commands in + one request; the IMAP side effects took ~55 s of total silence. + +The client timeout is on *time to first/next byte*, not on total request +duration: a Sync may take minutes as long as data keeps arriving. That is +the invariant this design establishes: **once a Sync request is accepted, +response bytes keep flowing until the response is complete.** + +## Architecture + +Streaming spans three packages: + +| Layer | Behaviour when streaming is enabled | +|-------|-------------------------------------| +| `horde/horde` `rpc.php` | Passes `$conf['activesync']['sync']['streaming']` to the RPC layer | +| `horde/rpc` `Horde_Rpc_ActiveSync` | For `Cmd=Sync` POST only: skips the full-response output buffer, disables zlib compression, sends no `Content-Length` (the web server applies chunked transfer-encoding). All other commands (`GetAttachment`, `ItemOperations`, `Ping`, …) keep the buffered `Content-Length` response | +| `horde/activesync` `Request_Sync` + `Wbxml_Encoder` | Flushes WBXML incrementally (details below) | + +The feature is **opt-in** (`streaming = false` by default) and fully +reversible: disabling it restores the buffered `Content-Length` path +including the legacy `maxresponsetime` budget, with no other changes. + +### Export phase: incremental flushing + +`Request_Sync` flushes the encoder (`Encoder::flushOutput()`): + +- after the response envelope/status preamble, +- after **every exported message**, +- at each folder close. + +So even when the backend is slow per message, the client sees bytes at +message granularity instead of one buffer at the end. + +### `MoreAvailable` ordering without buffering + +MS-ASCMD requires `MoreAvailable` *before* the `Commands` block in the +folder response. The legacy time budget (`maxresponsetime`) solved this by +buffering the whole `Commands` section and deciding truncation afterwards — +which defeats streaming. + +Instead, the count cap `maxmessagesperresponse` is folded into the +**effective window size before the `Commands` section starts**, so +truncation is always known up front and `MoreAvailable` is emitted through +the existing window-exceeded path. The `Commands` buffer is never used when +streaming, and `maxresponsetime` is ignored (a log notice is emitted if +both are configured). Unsent changes stay in `sync_pending` and are +delivered on the follow-up request the client issues in response to +`MoreAvailable`. + +### Import phase: deferred commands with keep-alives + +Export-phase flushing does not help when the *incoming* side is slow, since +imports historically ran during request **parsing** — before any response +byte. When streaming is enabled: + +- During parsing, incoming `Add`/`Change`/`Delete` (and EAS 16 instance + delete) commands are **queued** per collection instead of imported + inline. Parsing then completes in a fraction of a second and the response + preamble is flushed early. +- The queued commands are imported during response output, immediately + after `initCollectionState()` — the same logical position relative to + change detection and sync-key generation that inline imports occupied, so + the state semantics are unchanged (`importedchanges`, `SyncReplies`, + conflict detection against the change map all behave as before). +- Between imports the encoder emits a **WBXML keep-alive** + (`Encoder::keepAlive()`) and flushes, so bytes flow for the whole import + phase. Emission is throttled to at most one token per interval + (`keepaliveinterval` sync setting, default 15 s — safely below the ~30 s + read timeout of the strictest known clients while keeping the token + count minimal; a 200-command batch over ~55 s produces 3 tokens instead + of 200; `0` = one token per command). + +#### The keep-alive token + +`keepAlive()` writes a `SWITCH_PAGE` token to the *already active* code +page (2 bytes: `0x00` + current page). Per the WBXML specification a +`SWITCH_PAGE` to the current page is a semantic no-op; any conforming +parser skips it without state change. This makes it a safe filler byte that +can be injected at arbitrary token boundaries inside the response. The +encoder guarantees the WBXML document header precedes the first keep-alive +(`outputWbxmlHeader()` is idempotent and called from `keepAlive()`). + +Verified transparent against this package's own `Wbxml_Decoder` +(`SyncStreamingTest::testKeepAliveTokensAreTransparentToDecoder`). +Real-world client tolerance is still being validated in +[horde/ActiveSync#77](https://github.com/horde/ActiveSync/issues/77); the +emission throttle exists so strict client parsers see only a handful of +redundant tokens per response instead of hundreds. + +## Error model: pre-commit vs post-commit + +Once the first body byte is flushed, the HTTP status line can no longer be +changed — HTTP 400/500 responses are only possible **before** streaming +starts. The design therefore splits errors at the *commit point* (first +flushed byte): + +| Phase | Error channel | +|-------|---------------| +| Pre-commit: auth, policy check, request decode, change poll | HTTP 400/500 and header-level errors — unchanged from buffered mode | +| Post-commit: export, deferred import | In-protocol only: folder `Status` elements, per-command `SyncReplies` statuses, `MoreAvailable`. A streaming abort handler catches exporter exceptions, logs them, and closes a valid WBXML envelope. Unsent changes stay in `sync_pending` | + +Deferred import errors are recorded per command (failed adds get a failure +status in `SyncReplies`, failed changes are counted as import failures) — +they never abort the response. This is strictly better client-visible +behaviour than the buffered path, where a late exception produced an +HTTP 500 and the client discarded the entire batch. + +The RPC error paths guard `header()` calls with `headers_sent()`, so a +post-commit failure degrades to a truncated (but prefix-valid) response the +client re-requests — never a mid-stream protocol violation. + +## Configuration + +All keys under `$conf['activesync']['sync']` (for library embedders: the +`sync` array in the driver parameters, exposed via +`Driver_Base::getSyncConfig()`): + +| Key | Default | Meaning | +|-----|---------|---------| +| `streaming` | `false` | Master switch for streaming Sync delivery | +| `maxmessagesperresponse` | `10` | Count cap per response when streaming; more changes are announced via `MoreAvailable`. `0` = window size only | +| `maxmessagetime` | `0` | Soft cap (seconds) for assembling a single message; stops the batch after a slow message. Streaming only. `0` = off | +| `maxrequestduration` | `0` | Whole-request wall clock cap (seconds), measured from request start (includes the deferred import phase). Streaming only. `0` = off | +| `keepaliveinterval` | `15` | Minimum seconds between WBXML keep-alive tokens during deferred import. `0` = one token per imported command. Keep below the strictest client read timeout (~30 s) | +| `maxresponsetime` | `25` | **Legacy** export-phase time budget; only honored when `streaming` is `false` | + +`maxmessagesperresponse` is folded into the effective window size, so it can +only *lower* the client's `WindowSize`, never raise it. + +## Observability + +INFO-level log lines to verify and measure streaming: + +- `SYNC: starting response output N.Ns after request start (streaming on|off)` + — time to first byte; the primary health signal. +- `Queued N incoming changes for deferred import (streaming).` — up-sync + batch detected during parsing. +- `SYNC: imported N deferred incoming change(s) for collection F… in N.Ns, + N keep-alive(s) emitted` — duration of the deferred import phase and how + many keep-alive tokens went to the wire. + +## Deployment prerequisites + +PHP can disable its own buffering and zlib compression, but **not** the web +server's. Response buffering or compression on +`/Microsoft-Server-ActiveSync` (lighttpd `mod_deflate`, nginx `gzip` / +`proxy_buffering`, reverse proxies) re-introduces the timeout even with +streaming enabled — exclude the ActiveSync path from such filters. + +A device already stuck from earlier timeouts may still need one account +re-add (or server-side device state removal): streaming prevents the +breakage, it does not repair already-broken client state. + +## Testing + +- `test/unit/Horde/ActiveSync/Request/SyncStreamingTest.php` — keep-alive + transparency, WBXML header idempotence, deferred-import execution and + bookkeeping, streaming config plumbing. +- Manual validation: iOS Mail (regression, buffered semantics preserved), + Gmail on Android (the originally failing client, large export batches and + `FullDraftsUpSync` up-sync batches). + +## Non-goals + +- Client pacing of `MoreAvailable` follow-up requests (client behaviour). +- Client state self-repair after an already-broken sync relationship. +- The full Horde 6 request/response pipeline refactor + ([`todo.md`](todo.md)) — streaming is a tactical subset; the + Changes-object and response-object work remains on the roadmap. diff --git a/doc/todo.md b/doc/todo.md index 16d17502..2a5a3121 100644 --- a/doc/todo.md +++ b/doc/todo.md @@ -1,10 +1,11 @@ # ActiveSync TODO -Last reviewed: 2026-07-09 +Last reviewed: 2026-07-14 -This file tracks **remaining** work. For what the library already supports -(protocol versions, commands, EAS 16.0 behaviour, deployment setup), see the -package `README.md` at the repository root. +This file tracks **remaining** work. For what the library already supports, +see the doc index in the package `README.md` — in particular +[`protocol-versions.md`](protocol-versions.md) (versions, commands, per-version +behaviour) and [`configuration.md`](configuration.md) (deployment setup). Items are grouped by intent. The **Horde 6** section is a breaking-change roadmap — do not implement those entries piecemeal on the FRAMEWORK_6_0 / @@ -30,7 +31,43 @@ roadmap — do not implement those entries piecemeal on the FRAMEWORK_6_0 / Not supported — Nag uses fixed RRULE + `completions[]`, not post-completion regenerated due dates. Phase 0 (2026-06-23) saw `Regenerate=0` only on - Outlook weekly-series traffic. Documented in `README.md` (Tasks section). + Outlook weekly-series traffic. Documented in `doc/protocol-versions.md` + (Tasks section). + +## Near-term reliability (FRAMEWORK_6_0) + +- **Sync response streaming ([#83](https://github.com/horde/ActiveSync/issues/83)) + — implemented 2026-07-14, awaiting client validation** + + Streams Sync WBXML incrementally (chunked HTTP for `Cmd=Sync` only) so + clients with ~30s read timeouts (Gmail Android) receive body bytes while + work continues. Covers both directions: per-message flush on export, and + — since the 2026-07-14 reporter test exposed a 55.7s silent + `FullDraftsUpSync` import phase — deferred import of client-sent commands + during response output with WBXML keep-alive tokens + (`Encoder::keepAlive()`) flushed between imports. Behaviour, config keys + (`streaming`, `maxmessagesperresponse`, `maxmessagetime`, + `maxrequestduration`, legacy `maxresponsetime`), error model, and operator + notes are documented in [`sync-streaming.md`](sync-streaming.md). Companion + changes live in `horde/rpc` (`Horde_Rpc_ActiveSync` streaming path) and + `horde/horde` (`rpc.php` config passthrough, `conf.xml` keys). Motivated + by [#77](https://github.com/horde/ActiveSync/issues/77). + + **Remaining before #83 can close:** + + - [x] Author deployment soak (streaming on): iOS Mail account re-add / + fresh mail sync clean (2026-07-14). + - [x] Reporter test round 1 (2026-07-14): export-path streaming confirmed + working (stalled Inbox catch-up completed immediately); found the + up-sync gap (Drafts import before first byte), fixed via deferred + import + keep-alives. + - [ ] Gmail Android re-validation of the up-sync path (reporter, feature + branch) — includes keep-alive token tolerance of Gmail's WBXML parser. + - [ ] Flip `streaming` default to `true` in `conf.xml` after validation. + - [ ] Move this entry to **Recently completed** when #83 closes. + + Partial step toward “Request / response pipeline” and “Changes object” + below; not a substitute for the full Horde 6 response-object refactor. ## Operations and monitoring (out of library scope) @@ -96,6 +133,9 @@ when a migration plan is written. - Replace `Horde_Controller_Request_Http` with a library-local HTTP request object; add a matching response object and move header logic out of `Horde_Rpc_ActiveSync`. +- **Interim (FRAMEWORK_6_0):** Sync-only chunked streaming in + `Horde_Rpc_ActiveSync` ([#83](https://github.com/horde/ActiveSync/issues/83)); + full `horde/http` response object remains Horde 6. - Move non-server helpers out of `Horde_ActiveSync` (truncation helpers, version negotiation utilities, etc.). - Leverage horde/version @@ -188,11 +228,19 @@ when a migration plan is written. - Collection object replacing the associative collection array in `Sync.php`. - Changes object (array, `SplFixedArray`, or temp stream) to cap memory on large initial mailbox syncs and to unify the `add` / `modify` / `delete` - shapes between email and PIM collections. + shapes between email and PIM collections. Streaming v1 ([#83](https://github.com/horde/ActiveSync/issues/83)) + uses count-based `maxmessagesperresponse` and per-message flush instead; a + proper Changes object can follow in Horde 6. - Sync-reply objects per collection type; move logic out of `Horde_ActiveSync_Connector_Exporter_Sync`. - Configuration builder for server/driver construction (Ping-related settings are no longer Ping-only). +- **Unify batch-size vectors:** `maximumwindowsize` (client WindowSize + override, ping settings) and `maxmessagesperresponse` (streaming batch + cap, #83) both feed the same export-loop bound via `min()`. They stay + separate on FRAMEWORK_6_0 for rollback semantics (streaming cap must not + leak into buffered mode), but the Sync options / configuration-builder + refactor should collapse them into one batching policy. ### SMS @@ -230,7 +278,7 @@ active backlog; kept here so this file does not resurrect settled work. MS-ASCMD properties) is out of scope for the IMAP-backed implementation. - **Autodiscover**, **ItemOperations** (fetch/move/empty; not Schema), **Settings**, **Provision**, **Ping**, **Search**, **ValidateCert** — all - present for supported versions (see `README.md`). + present for supported versions (see `doc/protocol-versions.md`). ### EAS 16.0 calendar (library + `horde/kronolith` + `horde/core`) diff --git a/lib/Horde/ActiveSync/Request/Sync.php b/lib/Horde/ActiveSync/Request/Sync.php index 5d629ffc..3733f885 100644 --- a/lib/Horde/ActiveSync/Request/Sync.php +++ b/lib/Horde/ActiveSync/Request/Sync.php @@ -23,12 +23,19 @@ /** * Handle Sync requests * + * Supports two delivery modes: classic buffered responses, and streamed + * (chunked) responses with incremental WBXML flushing and deferred import + * of client-sent commands. The streaming design is documented in + * doc/sync-streaming.md. + * * @license http://www.horde.org/licenses/gpl GPLv2 * * @copyright 2009-2020 Horde LLC (http://www.horde.org) * @author Michael J Rubinsky + * @author Torben Dannhauer * @package ActiveSync * @internal + * @see doc/sync-streaming.md */ class Horde_ActiveSync_Request_Sync extends Horde_ActiveSync_Request_SyncBase { @@ -60,6 +67,50 @@ class Horde_ActiveSync_Request_Sync extends Horde_ActiveSync_Request_SyncBase */ protected $_collections; + /** + * Whether this Sync response is streamed to the client while the + * handler is still running (opt-in via the 'streaming' sync setting). + * + * @var boolean + */ + protected $_streaming = false; + + /** + * Client-sent Sync commands queued during request parsing for deferred + * import during response output (streaming only), keyed by collection + * id. Kept outside the collection arrays since entries contain message + * objects that must not end up in the sync cache or be serialized for + * partial-sync comparison. + * + * Each entry: array of + * ['type' => SYNC_ADD|SYNC_MODIFY, 'serverid' => string|false, + * 'clientid' => string|false, 'appdata' => message object|null] + * + * @var array + */ + protected $_deferredCommands = []; + + /** + * Minimum seconds between WBXML keep-alive tokens during deferred + * import (streaming only). Keeps the token count low (a handful per + * import phase instead of one per command) while still staying safely + * below the ~30 second read timeout of the strictest known clients + * (Gmail Android). 0 = emit after every imported command. Tunable via + * the 'keepaliveinterval' sync setting for client-compatibility + * diagnostics. + * + * @var integer + */ + protected $_keepAliveInterval = 15; + + /** + * Timestamp of the last emitted keep-alive (or of the flushed response + * preamble). @see _emitKeepAlive() + * + * @var float + */ + protected $_lastKeepAlive = 0.0; + /** * Handle the sync request * @@ -86,6 +137,17 @@ protected function _handle() $this->_statusCode = self::STATUS_SUCCESS; $partial = false; + // Needed before parsing: when streaming, client-sent Sync commands + // are queued during parse and imported during response output (so + // response bytes flow while the server works; see + // _runDeferredSyncCommands()). + $syncSettings = $this->_driver->getSyncConfig(); + $this->_streaming = !empty($syncSettings['streaming']); + $streaming = $this->_streaming; + if (isset($syncSettings['keepaliveinterval'])) { + $this->_keepAliveInterval = max(0, (int)$syncSettings['keepaliveinterval']); + } + try { $this->_collections = $this->_activeSync->getCollectionsObject(); } catch (Horde_ActiveSync_Exception $e) { @@ -242,10 +304,35 @@ protected function _handle() } $pingSettings = $this->_driver->getHeartbeatConfig(); - $syncSettings = $this->_driver->getSyncConfig(); $syncTimeBudget = !empty($syncSettings['maxresponsetime']) ? (int) $syncSettings['maxresponsetime'] : 0; + if ($streaming && $syncTimeBudget > 0) { + /* Streaming supersedes the export-phase time budget: the + * Commands buffering the budget needs for MOREAVAILABLE + * reordering would defeat per-message flushing. */ + $this->_logger->meta('Ignoring maxresponsetime; Sync response streaming is enabled.'); + $syncTimeBudget = 0; + } + /* Count-based batch cap (streaming only). Capping the window before + * the Commands section starts lets truncation reuse the + * window-exceeded path, keeping MOREAVAILABLE before Commands + * without buffering. */ + $maxMessagesPerResponse = $streaming + ? (int) ($syncSettings['maxmessagesperresponse'] ?? 10) + : 0; + /* Soft per-message assembly cap and whole-request wall clock + * (streaming only); both leave unsent changes in sync_pending. */ + $maxMessageTime = $streaming + ? (int) ($syncSettings['maxmessagetime'] ?? 0) + : 0; + $maxRequestDuration = $streaming + ? (int) ($syncSettings['maxrequestduration'] ?? 0) + : 0; + $requestServerVars = $this->_activeSync->request->getServerVars(); + $requestStart = !empty($requestServerVars['REQUEST_TIME_FLOAT']) + ? (float) $requestServerVars['REQUEST_TIME_FLOAT'] + : microtime(true); // Override the total, per-request, WINDOWSIZE? if (!empty($pingSettings['maximumrequestwindowsize'])) { @@ -308,11 +395,22 @@ protected function _handle() // Start output to client $syncOutputStart = microtime(true); + $this->_logger->info(sprintf( + 'SYNC: starting response output %.1fs after request start (streaming %s).', + $syncOutputStart - $requestStart, + $streaming ? 'on' : 'off' + )); $this->_encoder->startWBXML(); $this->_encoder->startTag(Horde_ActiveSync::SYNC_SYNCHRONIZE); $this->_encoder->startTag(Horde_ActiveSync::SYNC_STATUS); $this->_encoder->content(self::STATUS_SUCCESS); $this->_encoder->endTag(); + if ($streaming) { + /* First body bytes on the wire; keeps clients with hard read + * timeouts (Gmail ~30s) from aborting while messages are + * assembled below. */ + $this->_encoder->flushOutput(); + } // Start SYNC_FOLDERS $this->_encoder->startTag(Horde_ActiveSync::SYNC_FOLDERS); @@ -332,6 +430,23 @@ protected function _handle() $changecount = 0; if ($over_window || $cnt_global > $this->_collections->getDefaultWindowSize()) { + // Client-sent commands must still be imported (matching the + // non-streaming flow, where imports happen during parsing + // even for over-window collections). Replies are skipped, + // exactly like the non-streaming over-window response; the + // client re-sends and duplicate detection resolves it. + if (!empty($this->_deferredCommands[$id])) { + try { + $this->_collections->initCollectionState($collection); + $this->_runDeferredSyncCommands($collection); + } catch (Horde_ActiveSync_Exception $e) { + $this->_logger->err(sprintf( + 'Unable to import deferred commands for over-window collection %s: %s', + $id, + $e->getMessage() + )); + } + } $this->_sendOverWindowResponse($collection); continue; } @@ -365,6 +480,18 @@ protected function _handle() return false; } + // Import client-sent commands deferred during parsing + // (streaming only). Runs at the same logical position as the + // legacy inline imports - before change detection and synckey + // generation - but now with response bytes already on the wire + // and keep-alive tokens flushed between imports, so clients + // with hard read timeouts (Gmail Android ~30s) do not abort + // while a large batch (e.g. a full Drafts up-sync) is written + // to the backend. + if ($statusCode == self::STATUS_SUCCESS) { + $this->_runDeferredSyncCommands($collection); + } + // Clients are allowed to NOT request changes. We still must check // for them since this would otherwise screw up conflict detection // (we can't update sync_ts until we actually check for changes). In @@ -466,6 +593,11 @@ protected function _handle() $max_windowsize = !empty($pingSettings['maximumwindowsize']) ? min($collection['windowsize'], $pingSettings['maximumwindowsize']) : $collection['windowsize']; + $max_windowsize = $this->_streamingMaxWindowSize( + $max_windowsize, + $streaming, + $maxMessagesPerResponse + ); $countExceedsWindow = !empty($changecount) && (($changecount > $max_windowsize) @@ -476,7 +608,8 @@ protected function _handle() $changecount, $max_windowsize, $cnt_global, - $this->_collections->getDefaultWindowSize() + $this->_collections->getDefaultWindowSize(), + $streaming ); // MOREAVAILABLE? @@ -529,10 +662,31 @@ protected function _handle() break; } - $progress = $exporter->sendNextChange(); + $msgStart = microtime(true); + try { + $progress = $exporter->sendNextChange(); + } catch (Horde_Exception $e) { + if (!$streaming) { + throw $e; + } + /* Post-commit abort: body bytes are already + * on the wire, so finish a valid WBXML + * envelope instead of letting the RPC layer + * attempt an HTTP 500. Unsent changes stay + * in sync_pending. */ + $this->_logger->err(sprintf( + 'Streaming SYNC: aborting export for collection %s after error: %s', + $collection['id'], + $e->getMessage() + )); + break; + } if ($progress !== true) { break; } + if ($streaming) { + $this->_encoder->flushOutput(); + } $this->_logger->meta( sprintf( 'Peak memory usage after message: %d', @@ -541,6 +695,29 @@ protected function _handle() ); ++$cnt_collection; ++$cnt_global; + + if (!$exporter->hasPendingChanges()) { + continue; + } + if ($this->_isGuardExceeded($msgStart, $maxMessageTime)) { + $this->_logger->warn(sprintf( + 'SYNC: single message took %.1fs (cap %ds) in collection %s; stopping batch, remaining changes stay in sync_pending.', + microtime(true) - $msgStart, + $maxMessageTime, + $collection['id'] + )); + break; + } + if ($this->_isGuardExceeded($requestStart, $maxRequestDuration)) { + $this->_logger->warn(sprintf( + 'SYNC: request duration %.1fs exceeds cap (%ds) after %d change(s) in collection %s; stopping batch, remaining changes stay in sync_pending.', + microtime(true) - $requestStart, + $maxRequestDuration, + $cnt_collection, + $collection['id'] + )); + break; + } } $this->_encoder->endTag(); @@ -618,6 +795,9 @@ protected function _handle() // End SYNC_FOLDER $this->_encoder->endTag(); + if ($streaming) { + $this->_encoder->flushOutput(); + } $this->_logger->meta( sprintf( 'Collection output peak memory usage: %d', @@ -631,6 +811,9 @@ protected function _handle() // End SYNC_SYNCHRONIZE $this->_encoder->endTag(); + if ($streaming) { + $this->_encoder->flushOutput(); + } if ($this->_device->version >= Horde_ActiveSync::VERSION_TWELVEONE) { if ($this->_collections->checkStaleRequest()) { @@ -651,11 +834,17 @@ protected function _handle() * Should Sync Commands be buffered so MOREAVAILABLE can precede Commands * when a time budget stops the batch early? * + * Never buffer when streaming: buffering would hold all Commands bytes + * back until the batch completes, defeating per-message flushing. + * Truncation ordering is handled up front via the count-capped window + * instead (@see _streamingMaxWindowSize()). + * * @param integer $syncTimeBudget * @param integer $changecount * @param integer $maxWindowsize * @param integer $cntGlobal * @param integer $defaultWindowSize + * @param boolean $streaming * * @return boolean */ @@ -664,17 +853,59 @@ protected function _useSyncCommandsBuffer( $changecount, $maxWindowsize, $cntGlobal, - $defaultWindowSize + $defaultWindowSize, + $streaming = false ) { $countExceedsWindow = !empty($changecount) && (($changecount > $maxWindowsize) || $cntGlobal + $changecount > $defaultWindowSize); - return $syncTimeBudget > 0 + return !$streaming + && $syncTimeBudget > 0 && !empty($changecount) && !$countExceedsWindow; } + /** + * Reduce the effective per-collection window when streaming with a + * count-based response cap (maxmessagesperresponse). + * + * Capping the window before the Commands section starts means the + * truncation decision is known up front, so MOREAVAILABLE can be + * emitted before Commands (the only MS-ASCMD-valid ordering) without + * buffering the Commands output. + * + * @param integer $maxWindowsize Effective window so far. + * @param boolean $streaming Streaming enabled? + * @param integer $maxMessagesPerResponse Count cap, 0 = disabled. + * + * @return integer + */ + protected function _streamingMaxWindowSize( + $maxWindowsize, + $streaming, + $maxMessagesPerResponse + ) { + if ($streaming && $maxMessagesPerResponse > 0) { + return min($maxWindowsize, $maxMessagesPerResponse); + } + + return $maxWindowsize; + } + + /** + * Has an elapsed-time guard been exceeded? + * + * @param float $start Start time (microtime). + * @param integer $limit Limit in seconds, 0 = disabled. + * + * @return boolean + */ + protected function _isGuardExceeded($start, $limit) + { + return $limit > 0 && (microtime(true) - $start) >= $limit; + } + /** * Has the per-response sync time budget been reached? * @@ -698,6 +929,268 @@ protected function _isSyncTimeBudgetExceeded( && (microtime(true) - $syncOutputStart) >= $budget; } + /** + * Import a single client-sent ADD or MODIFY command and record the + * result in the collection array (replies, failures, atchash, ...). + * + * Shared by the inline (non-streaming) parse-phase import and the + * deferred (streaming) output-phase import. + * + * @param Horde_ActiveSync_Connector_Importer $importer The importer. + * @param array $collection The collection array, updated in place. + * @param string $commandType SYNC_ADD or SYNC_MODIFY. + * @param string|boolean $serverid Server id for MODIFY, false for ADD. + * @param string|boolean $clientid Client id for ADD, false for MODIFY. + * @param Horde_ActiveSync_Message_Base $appdata The message data. + */ + protected function _importSyncCommand( + $importer, + array &$collection, + $commandType, + $serverid, + $clientid, + $appdata + ) { + switch ($commandType) { + case Horde_ActiveSync::SYNC_MODIFY: + $ires = $importer->importMessageChange( + $serverid, + $appdata, + $this->_device, + false, + $collection['class'], + $collection['synckey'] + ); + if (is_array($ires) && !empty($ires['error'])) { + $collection['importfailures'][$ires[0]] = $ires['error']; + } elseif (is_array($ires)) { + $collection['importedchanges'] = true; + if (empty($collection['modifiedids'])) { + $collection['modifiedids'] = []; + } + $collection['modifiedids'][] = $ires['id']; + $collection['atchash'][$ires['id']] = !empty($ires['atchash']) + ? $ires['atchash'] + : []; + // Keep the conversation data like the SYNC_ADD path does. + // Without it, EAS 16 email MODIFYs (e.g. Gmail's Drafts + // up-sync) get no SYNC_MODIFY reply at all and the client + // rejects the response and re-sends the same batch. + if (!empty($ires['conversationid'])) { + $collection['conversations'][$ires['id']] + = [$ires['conversationid'], + $ires['conversationindex']]; + } + } + break; + + case Horde_ActiveSync::SYNC_ADD: + $ires = $importer->importMessageChange( + false, + $appdata, + $this->_device, + $clientid, + $collection['class'] + ); + if (!$ires || !empty($ires['error'])) { + $collection['clientids'][$clientid] = false; + } elseif ($clientid && is_array($ires)) { + $collection['clientids'][$clientid] = $ires['id']; + $collection['atchash'][$ires['id']] = !empty($ires['atchash']) + ? $ires['atchash'] + : []; + if (!empty($ires['conversationid'])) { + $collection['conversations'][$ires['id']] + = [$ires['conversationid'], + $ires['conversationindex']]; + } + $collection['importedchanges'] = true; + } elseif ($clientid && is_string($ires)) { + // Duplicate addition; client never received UID. + $collection['clientids'][$clientid] = $ires; + $collection['importedchanges'] = true; + } elseif ($clientid) { + $collection['clientids'][$clientid] = false; + } + break; + } + } + + /** + * Import a batch of client-sent REMOVE commands. + * + * @param Horde_ActiveSync_Connector_Importer $importer The importer. + * @param array $collection The collection array, updated in place. + * @param array $removes Server uids to remove. + * @param boolean $deletesasmoves Move to trash instead of deleting. + */ + protected function _importRemoves( + $importer, + array &$collection, + array $removes, + $deletesasmoves + ) { + if ($deletesasmoves + && $folderid = $this->_driver->getWasteBasket($collection['class'])) { + $results = $importer->importMessageMove($removes, $folderid); + } else { + // Mirror the ['results' => ..., 'missing' => ...] shape of + // importMessageMove(); importMessageDeletion() returns the + // plain list of successfully deleted uids. + $deleted = $importer->importMessageDeletion($removes, $collection['class']); + $results = is_array($deleted) + ? ['results' => $deleted, 'missing' => array_diff($removes, $deleted)] + : []; + } + if (!empty($results['missing'])) { + $collection['missing'] = $results['missing']; + } + $collection['importedchanges'] = true; + } + + /** + * Import client-sent EAS 16.0 instance deletions. + * + * @param Horde_ActiveSync_Connector_Importer $importer The importer. + * @param array $collection The collection array. + * @param array $instanceidRemoves Hash of uid => instanceid. + */ + protected function _importInstanceIdRemoves( + $importer, + array &$collection, + array $instanceidRemoves + ) { + foreach ($instanceidRemoves as $uid => $instanceid) { + $importer->importMessageDeletion([$uid => $instanceid], $collection['class'], true); + } + } + + /** + * Import client-sent Sync commands that were queued during request + * parsing (streaming only). + * + * Runs during response output, after the WBXML preamble has been + * flushed, at the same logical position the inline imports of the + * non-streaming flow occupy: before server-change detection and synckey + * generation for the collection. Between imports a WBXML keep-alive + * token is flushed at most once per $_keepAliveInterval seconds so + * clients with hard read timeouts keep receiving response body bytes + * during large up-sync batches without flooding strict WBXML parsers + * with redundant tokens. + * + * Import errors are recorded per command (reply status), never thrown: + * response bytes are already on the wire, so the request must finish + * with a valid WBXML envelope. + * + * @param array $collection The collection array, updated in place. + */ + protected function _runDeferredSyncCommands(array &$collection) + { + if (empty($this->_deferredCommands[$collection['id']])) { + return; + } + $deferred = $this->_deferredCommands[$collection['id']]; + unset($this->_deferredCommands[$collection['id']]); + + $importer = $this->_activeSync->getImporter(); + $importer->init($this->_state, $collection['id'], $collection['conflict']); + + $start = microtime(true); + // The preamble was just flushed; keep-alive pacing starts here. + $this->_lastKeepAlive = $start; + $keepAlives = 0; + $count = 0; + foreach ($deferred['commands'] ?? [] as $command) { + try { + $this->_importSyncCommand( + $importer, + $collection, + $command['type'], + $command['serverid'], + $command['clientid'], + $command['appdata'] + ); + } catch (Horde_Exception $e) { + $this->_logger->err(sprintf( + 'Deferred import failed for collection %s: %s', + $collection['id'], + $e->getMessage() + )); + if ($command['type'] == Horde_ActiveSync::SYNC_ADD + && $command['clientid']) { + $collection['clientids'][$command['clientid']] = false; + } elseif ($command['serverid']) { + $collection['importfailures'][$command['serverid']] + = self::STATUS_SERVERERROR; + } + } + ++$count; + $keepAlives += $this->_emitKeepAlive(); + } + + if (!empty($deferred['removes'])) { + try { + $this->_importRemoves( + $importer, + $collection, + $deferred['removes'], + !empty($deferred['deletesasmoves']) + ); + } catch (Horde_Exception $e) { + $this->_logger->err(sprintf( + 'Deferred remove failed for collection %s: %s', + $collection['id'], + $e->getMessage() + )); + } + $count += count($deferred['removes']); + $keepAlives += $this->_emitKeepAlive(); + } + if (!empty($deferred['instanceid_removes'])) { + try { + $this->_importInstanceIdRemoves( + $importer, + $collection, + $deferred['instanceid_removes'] + ); + } catch (Horde_Exception $e) { + $this->_logger->err(sprintf( + 'Deferred instance remove failed for collection %s: %s', + $collection['id'], + $e->getMessage() + )); + } + $count += count($deferred['instanceid_removes']); + $keepAlives += $this->_emitKeepAlive(); + } + + $this->_logger->info(sprintf( + 'SYNC: imported %d deferred incoming change(s) for collection %s in %.1fs, %d keep-alive(s) emitted (streaming).', + $count, + $collection['id'], + microtime(true) - $start, + $keepAlives + )); + } + + /** + * Emit a WBXML keep-alive if the configured interval has elapsed since + * the last one (or since the flushed response preamble). + * + * @return integer 1 if a keep-alive was emitted, 0 otherwise. + */ + protected function _emitKeepAlive() + { + $now = microtime(true); + if ($now - $this->_lastKeepAlive < $this->_keepAliveInterval) { + return 0; + } + $this->_lastKeepAlive = $now; + $this->_encoder->keepAlive(); + + return 1; + } + protected function _sendOverWindowResponse($collection) { $this->_logger->meta('Over window maximum, skip polling for this request.'); @@ -915,7 +1408,12 @@ protected function _parseSyncFolders() return false; } - if (!empty($collection['importedchanges'])) { + // Deferred (not yet imported) commands count as imported changes + // here: the flags gate looping sync and the empty-response + // shortcut, and a request carrying client commands must always + // produce a full response with replies. + if (!empty($collection['importedchanges']) + || !empty($this->_deferredCommands[$collection['id']])) { $this->_collections->importedChanges = true; } if ($this->_collections->collectionExists($collection['id']) && !empty($collection['windowsize'])) { @@ -982,6 +1480,12 @@ protected function _parseSyncCommands(&$collection) $importer = $this->_activeSync->getImporter(); $importer->init($this->_state, $collection['id'], $collection['conflict']); } + + /* When streaming, queue ADD/MODIFY imports and REMOVE batches for + * execution during response output (_runDeferredSyncCommands()), so + * the potentially slow backend writes happen while response bytes + * are already flowing to the client. */ + $deferring = $this->_streaming && !empty($collection['synckey']); $nchanges = 0; while (1) { // SYNC_MODIFY, SYNC_REMOVE, SYNC_ADD or SYNC_FETCH @@ -1115,58 +1619,24 @@ protected function _parseSyncCommands(&$collection) if (!empty($collection['synckey'])) { switch ($commandType) { case Horde_ActiveSync::SYNC_MODIFY: - if (isset($appdata)) { - $ires = $importer->importMessageChange( - $serverid, - $appdata, - $this->_device, - false, - $collection['class'], - $collection['synckey'] - ); - if (is_array($ires) && !empty($ires['error'])) { - $collection['importfailures'][$ires[0]] = $ires['error']; - } elseif (is_array($ires)) { - $collection['importedchanges'] = true; - if (empty($collection['modifiedids'])) { - $collection['modifiedids'] = []; - } - $collection['modifiedids'][] = $ires['id']; - $collection['atchash'][$ires['id']] = !empty($ires['atchash']) - ? $ires['atchash'] - : []; - } - } - break; - case Horde_ActiveSync::SYNC_ADD: if (isset($appdata)) { - $ires = $importer->importMessageChange( - false, - $appdata, - $this->_device, - $clientid, - $collection['class'] - ); - if (!$ires || !empty($ires['error'])) { - $collection['clientids'][$clientid] = false; - } elseif ($clientid && is_array($ires)) { - $collection['clientids'][$clientid] = $ires['id']; - $collection['atchash'][$ires['id']] = !empty($ires['atchash']) - ? $ires['atchash'] - : []; - if (!empty($ires['conversationid'])) { - $collection['conversations'][$ires['id']] - = [$ires['conversationid'], - $ires['conversationindex']]; - } - $collection['importedchanges'] = true; - } elseif ($clientid && is_string($ires)) { - // Duplicate addition; client never received UID. - $collection['clientids'][$clientid] = $ires; - $collection['importedchanges'] = true; - } elseif ($clientid) { - $collection['clientids'][$clientid] = false; + if ($deferring) { + $this->_deferredCommands[$collection['id']]['commands'][] = [ + 'type' => $commandType, + 'serverid' => $serverid, + 'clientid' => $clientid, + 'appdata' => $appdata, + ]; + } else { + $this->_importSyncCommand( + $importer, + $collection, + $commandType, + $serverid, + $clientid, + $appdata + ); } } break; @@ -1194,33 +1664,49 @@ protected function _parseSyncCommands(&$collection) } } - // Do all the SYNC_REMOVE requests at once - if (!empty($collection['removes']) - && !empty($collection['synckey'])) { - if (!empty($collection['deletesasmoves']) && $folderid = $this->_driver->getWasteBasket($collection['class'])) { - $results = $importer->importMessageMove($collection['removes'], $folderid); - } else { - $results = $importer->importMessageDeletion($collection['removes'], $collection['class']); - if (is_array($results)) { - $results['results'] = $results; - $results['missing'] = array_diff($collection['removes'], $results['results']); - } + if ($deferring) { + // Hand REMOVE batches to the deferred runner as well. + if (!empty($collection['removes'])) { + $this->_deferredCommands[$collection['id']]['removes'] + = $collection['removes']; + $this->_deferredCommands[$collection['id']]['deletesasmoves'] + = !empty($collection['deletesasmoves']); + unset($collection['removes']); } - if (!empty($results['missing'])) { - $collection['missing'] = $results['missing']; + if (!empty($collection['instanceid_removes'])) { + $this->_deferredCommands[$collection['id']]['instanceid_removes'] + = $collection['instanceid_removes']; + unset($collection['instanceid_removes']); } - unset($collection['removes']); - $collection['importedchanges'] = true; - } - // EAS 16.0 instance deletions. - if (!empty($collection['instanceid_removes']) && !empty($collection['synckey'])) { - foreach ($collection['instanceid_removes'] as $uid => $instanceid) { - $importer->importMessageDeletion([$uid => $instanceid], $collection['class'], true); + $this->_logger->info(sprintf( + 'Queued %d incoming changes for deferred import (streaming).', + $nchanges + )); + } else { + // Do all the SYNC_REMOVE requests at once + if (!empty($collection['removes']) + && !empty($collection['synckey'])) { + $this->_importRemoves( + $importer, + $collection, + $collection['removes'], + !empty($collection['deletesasmoves']) + ); + unset($collection['removes']); + } + // EAS 16.0 instance deletions. + if (!empty($collection['instanceid_removes']) + && !empty($collection['synckey'])) { + $this->_importInstanceIdRemoves( + $importer, + $collection, + $collection['instanceid_removes'] + ); + unset($collection['instanceid_removes']); } - unset($collection['instanceid_removes']); - } - $this->_logger->info(sprintf('Processed %d incoming changes', $nchanges)); + $this->_logger->info(sprintf('Processed %d incoming changes', $nchanges)); + } if (!$this->_decoder->getElementEndTag()) { // end commands diff --git a/lib/Horde/ActiveSync/Wbxml/Encoder.php b/lib/Horde/ActiveSync/Wbxml/Encoder.php index 61d638dc..c84e6c80 100644 --- a/lib/Horde/ActiveSync/Wbxml/Encoder.php +++ b/lib/Horde/ActiveSync/Wbxml/Encoder.php @@ -28,6 +28,7 @@ * * @copyright 2009-2020 Horde LLC (http://www.horde.org) * @author Michael J Rubinsky + * @author Torben Dannhauer * @package ActiveSync */ use Horde\Util\HordeString; @@ -66,6 +67,17 @@ class Horde_ActiveSync_Wbxml_Encoder extends Horde_ActiveSync_Wbxml */ protected $_tempStream; + /** + * Whether the WBXML document header was already written. + * + * Streaming Sync responses may pre-send the header (and keep-alive + * tokens) while the incoming request is still being processed; the + * regular startWBXML() call later must not emit it a second time. + * + * @var boolean + */ + protected $_headerSent = false; + /** * Const'r * @@ -113,9 +125,15 @@ public function startWBXML($multipart = false) /** * Output the Wbxml header to the output stream. * + * A no-op if the header was already written (streaming Sync responses + * pre-send it via keep-alive handling before the regular startWBXML()). */ public function outputWbxmlHeader() { + if ($this->_headerSent) { + return; + } + $this->_headerSent = true; $this->_outByte(0x03); // WBXML 1.3 $this->_outMBUInt(0x01); // Public ID 1 $this->_outMBUInt(106); // UTF-8 @@ -269,6 +287,55 @@ public function swapOutputStream($stream) return $previous; } + /** + * Flush pending output to the underlying stream and, when writing to the + * SAPI output stream, on to the client. + * + * Used by streaming Sync responses so each exported message reaches + * clients with hard read timeouts (e.g. Gmail Android at ~30 seconds) + * while the remaining batch is still being assembled. A no-op in effect + * when the output stream is a memory or temp stream (tests, buffers). + */ + public function flushOutput() + { + if (isset($this->_stream->stream) + && is_resource($this->_stream->stream)) { + fflush($this->_stream->stream); + } + // The transport layer (Horde_Rpc_ActiveSync) removes PHP output + // buffers before streaming starts, but output_buffering=On or a + // non-Horde integrator may still have one active - flush() alone + // would not push its contents to the client. Only the topmost + // buffer can be flushed without ending it; clearing nested + // buffers remains the transport layer's responsibility. + if (ob_get_level()) { + @ob_flush(); + } + flush(); + } + + /** + * Emit a WBXML keep-alive no-op and flush it to the client. + * + * Writes a SWITCH_PAGE token targeting the code page that is already + * active. Token-stream WBXML parsers (including this package's own + * decoder) process it without any semantic effect. Streaming Sync + * responses use this to keep response body bytes flowing while + * long-running server work is in progress and no protocol content is + * available yet - e.g. while importing client-sent changes, which can + * take far longer than the hard ~30 second read timeout of some clients + * (Gmail Android). + * + * @see doc/sync-streaming.md + */ + public function keepAlive() + { + $this->outputWbxmlHeader(); + $this->_stream->add(chr(self::SWITCH_PAGE)); + $this->_stream->add(chr($this->_tagcp)); + $this->flushOutput(); + } + /** * Append a buffered stream to the current output. * diff --git a/test/unit/Horde/ActiveSync/Request/SyncStreamingTest.php b/test/unit/Horde/ActiveSync/Request/SyncStreamingTest.php new file mode 100644 index 00000000..7effdb76 --- /dev/null +++ b/test/unit/Horde/ActiveSync/Request/SyncStreamingTest.php @@ -0,0 +1,432 @@ + + * @license http://www.horde.org/licenses/gpl GPLv2 + * @copyright 2026 The Horde Project + * @package ActiveSync + */ + +namespace Horde\ActiveSync; + +use Horde_ActiveSync; +use Horde_ActiveSync_Connector_Importer; +use Horde_ActiveSync_Log_Logger; +use Horde_ActiveSync_Message_Base; +use Horde_ActiveSync_Request_Sync; +use Horde_ActiveSync_State_Base; +use Horde_ActiveSync_Wbxml; +use Horde_ActiveSync_Wbxml_Decoder; +use Horde_ActiveSync_Wbxml_Encoder; +use Horde_Log_Handler_Null; +use Horde_Stream_Temp; +use PHPUnit\Framework\Attributes\CoversClass; +use PHPUnit\Framework\TestCase; +use ReflectionClass; + +#[CoversClass(Horde_ActiveSync_Request_Sync::class)] +#[CoversClass(Horde_ActiveSync_Wbxml_Encoder::class)] +class SyncStreamingTest extends TestCase +{ + public function testStreamingWindowSizeCapsWindow() + { + $sync = $this->_syncRequestWithoutConstructor(); + + // Cap smaller than window wins. + $this->assertSame(10, $this->_invokeSyncMethod( + $sync, + '_streamingMaxWindowSize', + [50, true, 10] + )); + + // Window smaller than cap wins. + $this->assertSame(5, $this->_invokeSyncMethod( + $sync, + '_streamingMaxWindowSize', + [5, true, 10] + )); + + // No effect when streaming is off. + $this->assertSame(50, $this->_invokeSyncMethod( + $sync, + '_streamingMaxWindowSize', + [50, false, 10] + )); + + // Cap disabled. + $this->assertSame(50, $this->_invokeSyncMethod( + $sync, + '_streamingMaxWindowSize', + [50, true, 0] + )); + } + + public function testGuardExceeded() + { + $sync = $this->_syncRequestWithoutConstructor(); + + $this->assertTrue($this->_invokeSyncMethod( + $sync, + '_isGuardExceeded', + [microtime(true) - 20, 10] + )); + + $this->assertFalse($this->_invokeSyncMethod( + $sync, + '_isGuardExceeded', + [microtime(true) - 5, 10] + )); + + // Disabled guard never triggers. + $this->assertFalse($this->_invokeSyncMethod( + $sync, + '_isGuardExceeded', + [microtime(true) - 3600, 0] + )); + } + + public function testUseSyncCommandsBufferDisabledWhenStreaming() + { + $sync = $this->_syncRequestWithoutConstructor(); + + // Buffering would be used without streaming... + $this->assertTrue($this->_invokeSyncMethod( + $sync, + '_useSyncCommandsBuffer', + [25, 10, 50, 0, 100, false] + )); + + // ...but never when streaming is enabled. + $this->assertFalse($this->_invokeSyncMethod( + $sync, + '_useSyncCommandsBuffer', + [25, 10, 50, 0, 100, true] + )); + } + + public function testMoreAvailablePrecedesCommandsWhenStreaming() + { + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + + // Simulate the streaming folder output: truncation is known before + // the Commands section starts (count-capped window), so + // MOREAVAILABLE is emitted inline and Commands stream unbuffered. + $encoder->startWBXML(); + $encoder->startTag(Horde_ActiveSync::SYNC_SYNCHRONIZE); + $encoder->startTag(Horde_ActiveSync::SYNC_FOLDERS); + $encoder->startTag(Horde_ActiveSync::SYNC_FOLDER); + $encoder->startTag(Horde_ActiveSync::SYNC_MOREAVAILABLE, false, true); + $encoder->startTag(Horde_ActiveSync::SYNC_COMMANDS); + $encoder->startTag(Horde_ActiveSync::SYNC_ADD); + $encoder->startTag(Horde_ActiveSync::SYNC_SERVERENTRYID); + $encoder->content('1'); + $encoder->endTag(); + $encoder->endTag(); + $encoder->endTag(); + $encoder->endTag(); + $encoder->endTag(); + $encoder->endTag(); + + rewind($main); + $bytes = stream_get_contents($main); + + // MoreAvailable (0x14, empty tag) must precede Commands + // (0x16 | content flag 0x40 = 0x56) in the byte stream. + $moreAvailablePos = strpos($bytes, chr(0x14)); + $commandsPos = strpos($bytes, chr(0x56)); + $this->assertNotFalse($moreAvailablePos); + $this->assertNotFalse($commandsPos); + $this->assertLessThan($commandsPos, $moreAvailablePos); + } + + public function testFlushOutputKeepsStreamIntact() + { + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + + $encoder->startWBXML(); + $encoder->flushOutput(); + $encoder->startTag(Horde_ActiveSync::SYNC_SYNCHRONIZE); + $encoder->startTag(Horde_ActiveSync::SYNC_STATUS); + $encoder->content('1'); + $encoder->flushOutput(); + $encoder->endTag(); + $encoder->endTag(); + $encoder->flushOutput(); + + rewind($main); + $bytes = stream_get_contents($main); + + // WBXML header + Synchronize/Status/'1'/ends, nothing lost or + // duplicated by flushing. + $this->assertSame( + chr(0x03) . chr(0x01) . chr(106) . chr(0x00) + . chr(0x45) . chr(0x4e) . chr(0x03) . '1' . chr(0x00) + . chr(0x01) . chr(0x01), + $bytes + ); + } + + public function testFlushOutputSafeOnSwappedBufferStream() + { + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + $encoder->startWBXML(); + + $buffer = new Horde_Stream_Temp(); + $saved = $encoder->swapOutputStream($buffer); + $encoder->startTag(Horde_ActiveSync::SYNC_COMMANDS); + $encoder->startTag(Horde_ActiveSync::SYNC_ADD, false, true); + $encoder->flushOutput(); + $encoder->endTag(); + $bufferLength = $buffer->length(); + $encoder->swapOutputStream($saved); + + $this->assertGreaterThan(0, $bufferLength); + } + + public function testKeepAliveTokensAreTransparentToDecoder() + { + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + + $encoder->startWBXML(); + $encoder->keepAlive(); + $encoder->startTag(Horde_ActiveSync::SYNC_SYNCHRONIZE); + $encoder->startTag(Horde_ActiveSync::SYNC_STATUS); + $encoder->keepAlive(); + $encoder->content('1'); + $encoder->endTag(); + $encoder->keepAlive(); + $encoder->endTag(); + + rewind($main); + $bytes = stream_get_contents($main); + + // Keep-alive is a redundant SWITCH_PAGE to the active code page. + $this->assertSame(3, substr_count($bytes, chr(0x00) . chr(0x00))); + + // A WBXML token parser must consume the document unchanged. + rewind($main); + $decoder = new Horde_ActiveSync_Wbxml_Decoder($main); + $decoder->readWbxmlHeader(); + $this->assertNotFalse($decoder->getElementStartTag(Horde_ActiveSync::SYNC_SYNCHRONIZE)); + $this->assertNotFalse($decoder->getElementStartTag(Horde_ActiveSync::SYNC_STATUS)); + $this->assertSame('1', $decoder->getElementContent()); + $this->assertNotFalse($decoder->getElementEndTag()); + $this->assertNotFalse($decoder->getElementEndTag()); + } + + public function testWbxmlHeaderIsEmittedExactlyOnce() + { + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + + // keepAlive() before startWBXML() must pre-send the header; + // startWBXML() must not emit it a second time. + $encoder->keepAlive(); + $encoder->startWBXML(); + $encoder->startTag(Horde_ActiveSync::SYNC_SYNCHRONIZE); + $encoder->startTag(Horde_ActiveSync::SYNC_STATUS); + $encoder->content('1'); + $encoder->endTag(); + $encoder->endTag(); + + rewind($main); + $bytes = stream_get_contents($main); + + $header = chr(0x03) . chr(0x01) . chr(106) . chr(0x00); + $this->assertSame(0, strpos($bytes, $header)); + $this->assertSame(1, substr_count($bytes, $header)); + } + + public function testRunDeferredSyncCommandsImportsAndEmitsKeepAlives() + { + $sync = $this->_syncRequestWithoutConstructor(); + $ref = new ReflectionClass($sync); + + $appdata = $this->createMock(Horde_ActiveSync_Message_Base::class); + + $importer = $this->createMock(Horde_ActiveSync_Connector_Importer::class); + $importer->expects($this->once())->method('init'); + $importer->expects($this->exactly(2)) + ->method('importMessageChange') + ->willReturnOnConsecutiveCalls( + // MODIFY result: stat array. Draft email modifies carry + // conversation data (the backend replaces the IMAP message, + // so the client must receive a SYNC_MODIFY reply). + [ + 'id' => '100', + 'mod' => 1, + 'conversationid' => '1a2b', + 'conversationindex' => 1234567890, + ], + // ADD result: stat array with new server uid. + ['id' => '200', 'mod' => 1] + ); + + $as = $this->createMock(Horde_ActiveSync::class); + $as->method('getImporter')->willReturn($importer); + + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + + foreach ([ + '_activeSync' => $as, + '_device' => $this->createMock(\Horde_ActiveSync_Device::class), + '_state' => $this->createMock(Horde_ActiveSync_State_Base::class), + '_encoder' => $encoder, + '_logger' => new Horde_ActiveSync_Log_Logger(new Horde_Log_Handler_Null()), + // Interval 0: emit a keep-alive after every imported command. + '_keepAliveInterval' => 0, + '_deferredCommands' => [ + 'F1' => [ + 'commands' => [ + [ + 'type' => Horde_ActiveSync::SYNC_MODIFY, + 'serverid' => '100', + 'clientid' => false, + 'appdata' => $appdata, + ], + [ + 'type' => Horde_ActiveSync::SYNC_ADD, + 'serverid' => false, + 'clientid' => 'client-1', + 'appdata' => $appdata, + ], + ], + ], + ], + ] as $property => $value) { + $prop = $ref->getProperty($property); + $prop->setAccessible(true); + $prop->setValue($sync, $value); + } + + $collection = [ + 'id' => 'F1', + 'class' => Horde_ActiveSync::CLASS_EMAIL, + 'synckey' => '{uuid}5', + 'conflict' => Horde_ActiveSync::CONFLICT_OVERWRITE_PIM, + 'clientids' => [], + ]; + + $method = $ref->getMethod('_runDeferredSyncCommands'); + $method->setAccessible(true); + $collectionArgs = [&$collection]; + $method->invokeArgs($sync, $collectionArgs); + + // Import results recorded like the inline (non-streaming) path. + $this->assertTrue($collection['importedchanges']); + $this->assertSame(['100'], $collection['modifiedids']); + $this->assertSame(['client-1' => '200'], $collection['clientids']); + + // Conversation data from the MODIFY import must be preserved so + // the exporter emits a SYNC_MODIFY reply for the item (Gmail + // rejects Drafts up-sync responses without these replies). + $this->assertSame( + ['100' => ['1a2b', 1234567890]], + $collection['conversations'] + ); + + // Queue consumed. + $deferredProp = $ref->getProperty('_deferredCommands'); + $deferredProp->setAccessible(true); + $this->assertSame([], $deferredProp->getValue($sync)); + + // One keep-alive per imported command reached the output stream + // (interval 0 disables the throttle). + rewind($main); + $bytes = stream_get_contents($main); + $this->assertSame(2, substr_count($bytes, chr(0x00) . chr(0x00))); + } + + public function testKeepAliveThrottleSuppressesTokensWithinInterval() + { + $sync = $this->_syncRequestWithoutConstructor(); + $ref = new ReflectionClass($sync); + + $main = fopen('php://memory', 'wb+'); + $encoder = $this->_encoder($main); + + foreach ([ + '_encoder' => $encoder, + // Default-style interval: far longer than this test runs. + '_keepAliveInterval' => 15, + '_lastKeepAlive' => microtime(true), + ] as $property => $value) { + $prop = $ref->getProperty($property); + $prop->setAccessible(true); + $prop->setValue($sync, $value); + } + + $method = $ref->getMethod('_emitKeepAlive'); + $method->setAccessible(true); + $emitted = 0; + for ($i = 0; $i < 200; $i++) { + $emitted += $method->invoke($sync); + } + + // Interval not elapsed: no token may reach the stream. + $this->assertSame(0, $emitted); + rewind($main); + $this->assertSame('', stream_get_contents($main)); + + // Backdate the last keep-alive beyond the interval: exactly one + // token is emitted, then the throttle closes again. + $prop = $ref->getProperty('_lastKeepAlive'); + $prop->setAccessible(true); + $prop->setValue($sync, microtime(true) - 16); + $emitted = 0; + for ($i = 0; $i < 200; $i++) { + $emitted += $method->invoke($sync); + } + $this->assertSame(1, $emitted); + } + + public function testRunDeferredSyncCommandsNoopWithoutQueue() + { + $sync = $this->_syncRequestWithoutConstructor(); + $ref = new ReflectionClass($sync); + + $collection = ['id' => 'F1']; + $method = $ref->getMethod('_runDeferredSyncCommands'); + $method->setAccessible(true); + $collectionArgs = [&$collection]; + $method->invokeArgs($sync, $collectionArgs); + + $this->assertSame(['id' => 'F1'], $collection); + } + + protected function _encoder($stream) + { + $encoder = new Horde_ActiveSync_Wbxml_Encoder( + $stream, + Horde_ActiveSync_Wbxml::LOG_PROTOCOL + ); + $encoder->setLogger( + new Horde_ActiveSync_Log_Logger(new Horde_Log_Handler_Null()) + ); + + return $encoder; + } + + protected function _syncRequestWithoutConstructor() + { + $ref = new ReflectionClass(Horde_ActiveSync_Request_Sync::class); + return $ref->newInstanceWithoutConstructor(); + } + + protected function _invokeSyncMethod($object, $method, array $args = []) + { + $ref = new ReflectionClass($object); + $method = $ref->getMethod($method); + $method->setAccessible(true); + return $method->invokeArgs($object, $args); + } +}